From de86197b3fb19d9dc274408ae08f7967b468e456 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Thu, 9 Jul 2026 02:45:30 +0100 Subject: [PATCH 01/96] Delta-encode the QWP symbol dictionary Previously every QWP ingress message re-sent the entire symbol dictionary, so a connection with many distinct symbols paid to retransmit the whole dictionary on every message. The client now sends each symbol id to the server only once per connection. Memory mode: - The producer keeps a monotonic "sent" watermark and each frame carries only the ids above it (a delta section), instead of the full dictionary from id 0. - On reconnect or failover the fresh server has an empty dictionary, so the I/O thread replays the whole dictionary as a catch-up frame before any post-reconnect traffic, keeping the producer's monotonic baseline valid across the wire boundary. Store-and-forward (file mode): - Each slot persists its dictionary to a dot-prefixed side-file (PersistedSymbolDict) using write-ahead ordering: new symbols are appended before the referencing frame is published, so a recovered or orphan-drained slot on a fresh process can always rebuild the dictionary that a delta frame references. - The persistence does not fsync, matching the rest of store-and-forward, which is process-crash durable (the page cache survives) but not host-crash durable. A host crash that tears the dictionary is caught at replay by a guard that fails the send cleanly ("resend required") instead of transmitting a gapped frame that would corrupt the table. Catch-up split: - The reconnect/recovery catch-up splits across as many frames as the server's advertised batch cap requires, so a dictionary larger than the cap is re-registered without any single frame exceeding it. The frames carry contiguous id ranges and reassemble on the server exactly as the original per-frame deltas would. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../qwp/client/QwpWebSocketSender.java | 144 ++++++- .../client/sf/cursor/CursorSendEngine.java | 70 +++- .../sf/cursor/CursorWebSocketSendLoop.java | 308 +++++++++++++- .../client/sf/cursor/PersistedSymbolDict.java | 379 ++++++++++++++++++ .../qwp/client/DeltaDictCatchUpTest.java | 325 +++++++++++++++ .../qwp/client/DeltaDictRecoveryTest.java | 352 ++++++++++++++++ .../cutlass/qwp/client/ReconnectTest.java | 7 +- .../qwp/client/SelfSufficientFramesTest.java | 151 +++++-- .../sf/cursor/PersistedSymbolDictTest.java | 219 ++++++++++ .../qwp/websocket/TestWebSocketServer.java | 17 + 10 files changed, 1910 insertions(+), 62 deletions(-) create mode 100644 core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictCatchUpTest.java create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index d1744065..286596c4 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -46,6 +46,7 @@ import io.questdb.client.cutlass.qwp.client.sf.cursor.DefaultSenderConnectionListener; import io.questdb.client.cutlass.qwp.client.sf.cursor.DefaultSenderErrorHandler; import io.questdb.client.cutlass.qwp.client.sf.cursor.DefaultSenderProgressHandler; +import io.questdb.client.cutlass.qwp.client.sf.cursor.PersistedSymbolDict; import io.questdb.client.cutlass.qwp.client.sf.cursor.SenderConnectionDispatcher; import io.questdb.client.cutlass.qwp.client.sf.cursor.SenderErrorDispatcher; import io.questdb.client.cutlass.qwp.client.sf.cursor.SenderProgressDispatcher; @@ -221,6 +222,14 @@ public class QwpWebSocketSender implements Sender { private CursorSendEngine cursorEngine; private CursorWebSocketSendLoop cursorSendLoop; private boolean deferCommit; + // True when the sender emits incremental (delta) symbol dictionaries: each + // message carries only symbol ids not yet sent on the wire, rather than the + // full dictionary from id 0. Enabled only in memory-mode, where a reconnect + // replays from the in-process ring and the I/O thread re-registers the whole + // dictionary via a catch-up frame before replaying. File-mode store-and-forward + // keeps full self-sufficient frames (recovery/orphan-drain can replay mid-stream + // to a fresh server, where a partial delta would leave gaps). Set in setCursorEngine. + private boolean deltaDictEnabled; // User-supplied observer for background orphan-slot drainer events. // Volatile: written by setDrainerListener (any thread, before or after // startOrphanDrainers) and read at pool-creation time. Null -> drainers @@ -313,6 +322,12 @@ public class QwpWebSocketSender implements Sender { // beginRound(true) call. roundSeq=1 is the first round; CONNECTED in the // first round indicates the initial connect. private long roundSeq; + // Highest global symbol id the producer has baked into a frame so far, or -1. + // Lifetime-monotonic in delta mode -- it is NOT reset on reconnect, because + // the I/O thread re-registers the full dictionary via a catch-up frame before + // replaying, so the producer's delta baseline stays valid across the wire + // boundary. Used only when deltaDictEnabled; ignored in full-dict mode. + private int sentMaxSymbolId = -1; // When true, auto-flush sends messages with FLAG_DEFER_COMMIT and only // explicit flush() triggers the server-side commit. Enables accumulating // arbitrarily large datasets that exceed the server's recv buffer. @@ -2215,6 +2230,18 @@ public void setCursorEngine(CursorSendEngine engine, boolean takeOwnership) { } this.cursorEngine = engine; this.ownsCursorEngine = takeOwnership && engine != null; + // Delta encoding is available in memory-mode (in-process catch-up) and in + // file-mode when the persisted dictionary opened (recovery / orphan-drain + // rebuild the dictionary from it). Otherwise fall back to full self- + // sufficient frames. See CursorSendEngine.isDeltaDictEnabled. + this.deltaDictEnabled = engine != null && engine.isDeltaDictEnabled(); + // Recovery: repopulate the producer's global dictionary from the slot's + // persisted dictionary so newly ingested symbols continue from the + // recovered ids (rather than colliding with them at 0), and the delta + // baseline resumes where the crashed session left off. + if (deltaDictEnabled && engine.wasRecoveredFromDisk()) { + seedGlobalDictionaryFromPersisted(engine.getPersistedSymbolDict()); + } } /** @@ -3423,14 +3450,15 @@ private void flushPendingRows(boolean deferCommit) { } ensureActiveBufferReady(); - // Cursor SF requires every on-disk frame to be self-sufficient: - // recorded frames replay to fresh server connections (orphan-slot - // drainers and post-reconnect replay), so always emit the full - // symbol-dict delta from id=0 and the full column schema inline, - // never a back-reference the target server may not have seen. + // In full-dict mode every frame is self-sufficient: it carries the whole + // symbol dictionary from id 0 so orphan-drain / recovery replay to a fresh + // server never dangles a symbol id. In delta mode (memory-mode only) each + // frame carries only ids above sentMaxSymbolId; a reconnect re-registers + // the dictionary via an I/O-thread catch-up frame before replay, so the + // producer's monotonic baseline stays valid across the wire boundary. encoder.setDeferCommit(deferCommit); encoder.beginMessage(tableCount, globalSymbolDictionary, - /*confirmedMaxId=*/ -1, currentBatchMaxSymbolId); + symbolDeltaBaseline(), currentBatchMaxSymbolId); for (int i = 0, n = keys.size(); i < n; i++) { CharSequence tableName = keys.getQuick(i); if (tableName == null) { @@ -3456,10 +3484,18 @@ private void flushPendingRows(boolean deferCommit) { return; } + // Write-ahead: durably persist this frame's new symbols BEFORE it is + // published, so a recovered/orphan-drained slot can always rebuild the + // dictionary the (non-self-sufficient) delta frame references. No-op in + // memory mode and when the frame introduces no new symbols. + persistNewSymbolsBeforePublish(); activeBuffer.ensureCapacity(messageSize); activeBuffer.write(buffer.getBufferPtr(), messageSize); activeBuffer.incrementRowCount(); sealAndSwapBuffer(); + // The frame carrying ids up to currentBatchMaxSymbolId is now on the ring; + // advance the delta baseline so the next frame ships only newer ids. + advanceSentMaxSymbolId(); hasDeferredMessages = deferCommit; if (!deferCommit) { @@ -3514,8 +3550,12 @@ private void flushPendingRowsSplit(ObjList keys, boolean deferComm boolean deferThis = deferCommit || !isLast; encoder.setDeferCommit(deferThis); + // Each split frame emits the delta above sentMaxSymbolId; the first + // frame ships the whole batch's new ids and advances the baseline, so + // the remaining frames carry an empty delta and just reference ids the + // first frame already registered. encoder.beginMessage(1, globalSymbolDictionary, - /*confirmedMaxId=*/ -1, currentBatchMaxSymbolId); + symbolDeltaBaseline(), currentBatchMaxSymbolId); encoder.addTable(tableBuffer); int messageSize = encoder.finishMessage(); QwpBufferWriter buffer = encoder.getBuffer(); @@ -3528,11 +3568,18 @@ private void flushPendingRowsSplit(ObjList keys, boolean deferComm .put(", serverMaxBatchSize=").put(serverMaxBatchSize).put(']'); } + // Write-ahead persist before publish (see flushPendingRows). The + // first split frame carries the batch's new symbols; the rest are + // no-ops once the baseline has advanced past them. + persistNewSymbolsBeforePublish(); ensureActiveBufferReady(); activeBuffer.ensureCapacity(messageSize); activeBuffer.write(buffer.getBufferPtr(), messageSize); activeBuffer.incrementRowCount(); sealAndSwapBuffer(); + // Frame queued: advance so the next split frame's delta starts above + // the ids this one just registered. + advanceSentMaxSymbolId(); } encoder.setDeferCommit(false); @@ -3573,8 +3620,10 @@ private void sendCommitMessage() { LOG.debug("Sending commit message for deferred batch"); } encoder.setDeferCommit(false); + // A commit carries no rows and no new symbols; in delta mode its empty + // delta simply starts at the server's current dictionary size. encoder.beginMessage(0, globalSymbolDictionary, - /*confirmedMaxId=*/ -1, currentBatchMaxSymbolId); + symbolDeltaBaseline(), currentBatchMaxSymbolId); int messageSize = encoder.finishMessage(); QwpBufferWriter buffer = encoder.getBuffer(); ensureActiveBufferReady(); @@ -3586,15 +3635,84 @@ private void sendCommitMessage() { lastCommitBoundaryFsn = cursorEngine.publishedFsn(); } + /** + * Advances the delta baseline once a frame carrying the current batch's + * symbols has been queued onto the ring. No-op in full-dict mode. Only ever + * moves the baseline forward, so a batch that used no new symbols leaves it + * unchanged. + */ + private void advanceSentMaxSymbolId() { + if (deltaDictEnabled && currentBatchMaxSymbolId > sentMaxSymbolId) { + sentMaxSymbolId = currentBatchMaxSymbolId; + } + } + + /** + * Appends the symbols this frame introduces ({@code [sentMaxSymbolId+1 .. + * currentBatchMaxSymbolId]}) to the slot's persisted dictionary BEFORE the + * frame is published to the ring. This write-ahead ordering keeps the + * persisted dictionary a superset of every process-crash-recoverable frame's + * references, so recovery and orphan-drain can re-register it on a fresh + * server. Not fsync'd (see PersistedSymbolDict) -- a host crash that tears it + * is caught by the send loop's replay guard. No-op in memory mode (no + * persisted dictionary) and when the frame introduces no new symbols. + */ + private void persistNewSymbolsBeforePublish() { + if (!deltaDictEnabled || cursorEngine == null) { + return; + } + PersistedSymbolDict pd = cursorEngine.getPersistedSymbolDict(); + if (pd == null) { + return; + } + int from = sentMaxSymbolId + 1; + int to = currentBatchMaxSymbolId; + if (to < from) { + return; + } + for (int id = from; id <= to; id++) { + pd.appendSymbol(globalSymbolDictionary.getSymbol(id)); + } + } + private void resetSymbolDictStateForNewConnection() { - // The new server has an empty symbol dictionary, so the next batch - // must ship a delta starting at id 0. beginMessage() always passes - // confirmedMaxId = -1; resetting the batch watermark here keeps a - // stale value from suppressing re-emission of symbol ids the new - // server has never seen. + // Runs on the foreground (initial) connect only -- NOT on the I/O thread's + // reconnect/failover path. The per-batch watermark is drained state, so + // clearing it here is harmless. sentMaxSymbolId is deliberately left + // untouched: in delta mode the I/O thread re-registers the whole + // dictionary with a catch-up frame on reconnect, so the producer's + // monotonic baseline must survive the wire boundary; resetting it would + // desync the producer from the I/O thread's sent-dictionary count. currentBatchMaxSymbolId = -1; } + /** + * On recovery, repopulates the producer's {@link GlobalSymbolDictionary} from + * the slot's persisted dictionary (ids assigned in the same ascending order, + * so they match the recovered frames) and resumes the delta baseline at the + * recovered tip, so newly ingested symbols continue above the recovered ids. + */ + private void seedGlobalDictionaryFromPersisted(PersistedSymbolDict pd) { + if (pd == null || pd.size() == 0) { + return; + } + ObjList symbols = pd.readLoadedSymbols(); + for (int i = 0, n = symbols.size(); i < n; i++) { + globalSymbolDictionary.getOrAddSymbol(symbols.getQuick(i)); + } + sentMaxSymbolId = globalSymbolDictionary.size() - 1; + } + + /** + * The symbol id below which the server already holds every dictionary entry, + * used as {@code confirmedMaxId} when encoding a frame. In delta mode this is + * the producer's monotonic sent watermark; in full-dict mode it is -1 so every + * frame re-ships the dictionary from id 0. + */ + private int symbolDeltaBaseline() { + return deltaDictEnabled ? sentMaxSymbolId : -1; + } + private void rollbackRow() { if (currentTableBuffer != null) { currentTableBuffer.cancelCurrentRow(); diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java index 64ca75d0..85015a73 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java @@ -110,6 +110,13 @@ public final class CursorSendEngine implements QuietCloseable { // in the constructor, closed by {@link #close()}. The segment manager // writes through this on every tick where ackedFsn has advanced. private final AckWatermark watermark; + // Engine-owned per-slot symbol dictionary file (disk mode only; {@code null} + // in memory mode and if open() failed). Enables delta-encoded SF frames: + // recovery / orphan-drain load it to re-register the dictionary on the fresh + // server before replaying non-self-sufficient frames. Opened in the + // constructor, closed by {@link #close()}. When null in disk mode the engine + // reports delta encoding as unavailable and the sender keeps full-dict frames. + private final PersistedSymbolDict persistedSymbolDict; // close() is publicly callable from any thread (Sender.close from a user // thread, JVM shutdown hooks, test cleanup). volatile + synchronized // close() makes the check-and-set atomic and gives readers a fence. @@ -199,6 +206,7 @@ private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager man // reference instead of orphaning the mmap'd segments + fds. SegmentRing ringInProgress = null; AckWatermark watermarkInProgress = null; + PersistedSymbolDict persistedDictInProgress = null; try { // Disk mode: try to recover any *.sfa files left behind by a prior // session before deciding to start fresh. Without this the engine @@ -257,6 +265,10 @@ private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager man // mmap doesn't take down the engine -- we just fall // back to the bare lowestBase - 1 seed. watermarkInProgress = AckWatermark.open(sfDir); + // Load the persisted symbol dictionary so delta-encoded frames + // in this recovered slot can be re-registered on the fresh + // server before replay. Null on open failure -> delta disabled. + persistedDictInProgress = PersistedSymbolDict.open(sfDir); long baseSeed = lowestBase - 1; long watermarkFsn = watermarkInProgress != null ? watermarkInProgress.read() @@ -309,6 +321,10 @@ private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager man if (!memoryMode) { AckWatermark.removeOrphan(sfDir); watermarkInProgress = AckWatermark.open(sfDir); + // Same stale-side-file hygiene for the symbol dictionary: a + // fresh slot starts with an empty dictionary. + PersistedSymbolDict.removeOrphan(sfDir); + persistedDictInProgress = PersistedSymbolDict.open(sfDir); } MmapSegment initial; String initialPath = null; @@ -333,10 +349,11 @@ private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager man manager.start(); } manager.register(ringInProgress, sfDir, watermarkInProgress); - // All construction succeeded — commit the ring and - // watermark references. + // All construction succeeded — commit the ring, watermark and + // symbol-dictionary references. this.ring = ringInProgress; this.watermark = watermarkInProgress; + this.persistedSymbolDict = persistedDictInProgress; } catch (Throwable t) { // Stop an owned manager before freeing the ring and watermark it may // touch, then release the slot lock. Each cleanup is in its own @@ -362,6 +379,12 @@ private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager man } catch (Throwable ignored) { } } + if (persistedDictInProgress != null) { + try { + persistedDictInProgress.close(); + } catch (Throwable ignored) { + } + } if (acquiredLock != null) { try { acquiredLock.close(); @@ -541,6 +564,12 @@ public synchronized void close() { } catch (Throwable ignored) { } } + if (persistedSymbolDict != null) { + try { + persistedSymbolDict.close(); + } catch (Throwable ignored) { + } + } if (fullyDrained) { try { unlinkAllSegmentFiles(sfDir); @@ -550,6 +579,11 @@ public synchronized void close() { AckWatermark.removeOrphan(sfDir); } catch (Throwable ignored) { } + try { + // Slot fully drained: the dictionary has no frames behind it. + PersistedSymbolDict.removeOrphan(sfDir); + } catch (Throwable ignored) { + } } } finally { if (slotLock != null) { @@ -586,6 +620,38 @@ public long getTotalBackpressureStalls() { return backpressureStallCount.get(); } + /** + * True when this engine has no store-and-forward directory: the ring lives + * only in malloc'd memory, so it cannot be recovered after a crash and no + * orphan drainer ever replays it. Only in-process reconnect/failover replays + * its frames, which is what makes send-time symbol-dict catch-up (rather than + * fully self-sufficient frames) safe. + */ + public boolean isMemoryMode() { + return sfDir == null; + } + + /** + * Whether the sender may delta-encode symbol dictionaries on this engine. + * Always true in memory mode (the send loop keeps an in-process catch-up + * mirror). In disk mode it requires the persisted dictionary to have opened, + * since delta frames are not self-sufficient and recovery / orphan-drain must + * be able to rebuild the dictionary from disk. When false in disk mode the + * sender falls back to full self-sufficient frames. + */ + public boolean isDeltaDictEnabled() { + return sfDir == null || persistedSymbolDict != null; + } + + /** + * The engine's persisted symbol dictionary, or {@code null} in memory mode + * (and in disk mode if it failed to open). The producer appends new symbols + * to it; recovery / orphan-drain read its loaded entries to seed catch-up. + */ + public PersistedSymbolDict getPersistedSymbolDict() { + return persistedSymbolDict; + } + /** * Pass-through to {@link SegmentRing#nextSealedAfter(MmapSegment)}. */ diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java index 13a69f77..cbf948b0 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java @@ -36,9 +36,12 @@ import io.questdb.client.cutlass.qwp.client.QwpIngressRoleRejectedException; import io.questdb.client.cutlass.qwp.client.QwpRoleMismatchException; import io.questdb.client.cutlass.qwp.client.QwpVersionMismatchException; +import io.questdb.client.cutlass.qwp.client.NativeBufferWriter; import io.questdb.client.cutlass.qwp.client.WebSocketResponse; +import io.questdb.client.cutlass.qwp.protocol.QwpConstants; import io.questdb.client.cutlass.qwp.websocket.WebSocketCloseCode; import io.questdb.client.std.CharSequenceLongHashMap; +import io.questdb.client.std.MemoryTag; import io.questdb.client.std.QuietCloseable; import io.questdb.client.std.Unsafe; import org.jetbrains.annotations.TestOnly; @@ -177,6 +180,20 @@ public final class CursorWebSocketSendLoop implements QuietCloseable { private final long reconnectMaxDurationMillis; private final WebSocketResponse response = new WebSocketResponse(); private final ResponseHandler responseHandler = new ResponseHandler(); + // Delta symbol dictionary catch-up state (memory-mode only; see swapClient). + // deltaDictEnabled gates all of it. The loop mirrors, in sentDict*, every + // symbol it has ever sent -- the concatenated [len varint][utf8] bytes in + // global-id order (sentDictBytes*) plus the count (sentDictCount) -- so that + // on reconnect it can re-register the whole dictionary on the fresh server + // (which discards its dictionary on every disconnect) before replaying frames + // whose deltas start above id 0. All of this is touched only by the I/O thread. + private final boolean deltaDictEnabled; + private long sentDictBytesAddr; + private int sentDictBytesCapacity; + private int sentDictBytesLen; + private int sentDictCount; + // End position (native address) written by the last readVarintAt() call. + private long varintEnd; private final CountDownLatch shutdownLatch = new CountDownLatch(1); private final AtomicLong totalAcks = new AtomicLong(); // Counters for observability of the durable-ack path. Both are zero @@ -489,6 +506,23 @@ public CursorWebSocketSendLoop(WebSocketClient client, CursorSendEngine engine, } this.client = client; this.engine = engine; + this.deltaDictEnabled = engine.isDeltaDictEnabled(); + // Recovery / orphan-drain: the loop starts with a fresh in-memory mirror, + // so seed it from the slot's persisted dictionary. That way the very first + // connection re-registers the whole dictionary (via a catch-up frame) + // before replaying the recovered delta frames. + if (deltaDictEnabled) { + PersistedSymbolDict pd = engine.getPersistedSymbolDict(); + if (pd != null && pd.size() > 0) { + int len = pd.loadedEntriesLen(); + if (len > 0) { + ensureSentDictCapacity(len); + Unsafe.getUnsafe().copyMemory(pd.loadedEntriesAddr(), sentDictBytesAddr, len); + sentDictBytesLen = len; + } + sentDictCount = pd.size(); + } + } this.fsnAtZero = fsnAtZero; this.parkNanos = parkNanos; this.reconnectFactory = reconnectFactory; @@ -1669,6 +1703,14 @@ private void ioLoop() { // best-effort } } + // The symbol-dict mirror is I/O-thread-owned; free it here, on the + // owning thread's exit path, after the last send that could touch it. + if (sentDictBytesAddr != 0) { + Unsafe.free(sentDictBytesAddr, sentDictBytesCapacity, MemoryTag.NATIVE_DEFAULT); + sentDictBytesAddr = 0; + sentDictBytesCapacity = 0; + sentDictBytesLen = 0; + } shutdownLatch.countDown(); // Failed-stop hand-off (see delegateEngineClose): the owner could // not free the engine safely while this thread was alive, so the @@ -1849,8 +1891,6 @@ private void swapClient(WebSocketClient newClient) { // past the tail instead of replaying into it. tryRetireOrphanTail(); long replayStart = engine.ackedFsn() + 1L; - this.fsnAtZero = replayStart; - this.nextWireSeq = 0L; // Snapshot publishedFsn at swap time — frames at FSN ≤ this value // were already on the wire before the drop and will be replayed. // trySendOne resets replayTargetFsn to -1 once we cross the boundary. @@ -1862,9 +1902,240 @@ private void swapClient(WebSocketClient newClient) { // carrying stale state across the wire boundary would either // double-trim or starve the queue. clearDurableAckTracking(); + setWireBaselineWithCatchUp(replayStart); positionCursorAt(replayStart); } + /** + * Sets the wire-sequence baseline for a fresh connection and, when the symbol + * dictionary mirror is non-empty, emits a full-dictionary catch-up frame first + * so the fresh server (whose dictionary starts empty) can resolve the + * non-self-sufficient delta frames that replay next. + *

+ * The catch-up occupies wire seq 0, which maps to the already-acked FSN just + * below {@code replayStart} (a harmless re-ack); real replay frames then follow + * from wire seq 1. With nothing to catch up (fresh sender, or full-dict mode), + * or before a client exists (async initial connect), keep the plain 1:1 + * {@code fsnAtZero == replayStart} mapping; the catch-up then happens on the + * first real connection via swapClient. + */ + private void setWireBaselineWithCatchUp(long replayStart) { + if (client != null && deltaDictEnabled && sentDictCount > 0) { + this.nextWireSeq = 0L; + // The catch-up may span several frames when the dictionary exceeds the + // server's batch cap; each consumes a wire sequence (0 .. n-1) that maps + // to an already-acked FSN, so the first real frame still lands on + // replayStart. + int catchUpFrames = sendDictCatchUp(); + this.fsnAtZero = replayStart - catchUpFrames; + } else { + this.fsnAtZero = replayStart; + this.nextWireSeq = 0L; + } + } + + /** + * Copies the symbol-dictionary delta a just-sent frame carries into the + * loop-local mirror ({@link #sentDictBytesAddr}) so a future reconnect can + * re-register it. Frames are sent in FSN order carrying monotonically + * extending deltas, so a frame whose delta starts exactly at + * {@link #sentDictCount} extends the mirror; a replayed or empty-delta frame + * (nothing new) is skipped. Only ever called in delta mode. + * + * @param payloadAddr address of the QWP message (12-byte header first) + * @param payloadLen message length in bytes + */ + /** + * Returns the symbol-dictionary delta start id of a frame, or -1 when the + * frame carries no delta section. Used by the pre-send torn-dictionary guard. + */ + private int frameDeltaStart(long payloadAddr, int payloadLen) { + if (!isDeltaFrame(payloadAddr, payloadLen)) { + return -1; + } + return (int) readVarintAt(payloadAddr + QwpConstants.HEADER_SIZE, payloadAddr + payloadLen); + } + + // True only for a well-formed QWP frame this encoder produced that carries a + // delta symbol-dict section. The magic check keeps the dict logic from + // misreading non-QWP payloads (e.g. synthetic frames injected by tests) whose + // bytes happen to set the delta flag. + private static boolean isDeltaFrame(long payloadAddr, int payloadLen) { + if (payloadLen < QwpConstants.HEADER_SIZE + || Unsafe.getUnsafe().getInt(payloadAddr) != QwpConstants.MAGIC_MESSAGE) { + return false; + } + byte flags = Unsafe.getUnsafe().getByte(payloadAddr + QwpConstants.HEADER_OFFSET_FLAGS); + return (flags & QwpConstants.FLAG_DELTA_SYMBOL_DICT) != 0; + } + + private void accumulateSentDict(long payloadAddr, int payloadLen) { + long limit = payloadAddr + payloadLen; + if (!isDeltaFrame(payloadAddr, payloadLen)) { + return; + } + long p = payloadAddr + QwpConstants.HEADER_SIZE; + long deltaStart = readVarintAt(p, limit); + p = varintEnd; + long deltaCount = readVarintAt(p, limit); + p = varintEnd; + // Only a delta that extends exactly from our current tip introduces new + // symbols. deltaStart < sentDictCount is a replay/overlap we already hold; + // deltaStart > sentDictCount is rejected before send by the torn-dictionary + // guard in trySendOne, so it never reaches here. + if (deltaCount <= 0 || deltaStart != sentDictCount) { + return; + } + long regionStart = p; + for (long i = 0; i < deltaCount; i++) { + long len = readVarintAt(p, limit); + p = varintEnd + len; + if (p > limit) { + // Malformed -- never happens for frames we encoded; bail rather + // than corrupt the mirror. + return; + } + } + int regionBytes = (int) (p - regionStart); + ensureSentDictCapacity(sentDictBytesLen + regionBytes); + Unsafe.getUnsafe().copyMemory(regionStart, sentDictBytesAddr + sentDictBytesLen, regionBytes); + sentDictBytesLen += regionBytes; + sentDictCount += (int) deltaCount; + } + + private void ensureSentDictCapacity(int required) { + if (sentDictBytesCapacity >= required) { + return; + } + int newCap = Math.max(sentDictBytesCapacity * 2, Math.max(4096, required)); + sentDictBytesAddr = Unsafe.realloc(sentDictBytesAddr, sentDictBytesCapacity, newCap, MemoryTag.NATIVE_DEFAULT); + sentDictBytesCapacity = newCap; + } + + private long readVarintAt(long p, long limit) { + long value = 0; + int shift = 0; + long cur = p; + while (cur < limit) { + byte b = Unsafe.getUnsafe().getByte(cur++); + value |= (long) (b & 0x7F) << shift; + if ((b & 0x80) == 0) { + break; + } + shift += 7; + } + varintEnd = cur; + return value; + } + + /** + * Builds a table-less catch-up frame carrying the full symbol dictionary + * ({@code deltaStart=0, deltaCount=sentDictCount}) and sends it to the freshly + * connected server, then consumes wire seq 0 for it. {@code sendBinary} copies + * the bytes synchronously, so the temporary frame buffer is freed immediately. + * On send failure it fails the loop, which recycles and retries on the next + * connection. + */ + /** + * Re-registers the whole symbol dictionary on a fresh connection, split into + * as many table-less frames as the server's advertised batch cap requires so + * no single frame exceeds it (a large dictionary would otherwise be rejected). + * Each chunk carries a contiguous id range {@code [start .. start+count)}, in + * order, so the server accumulates them exactly as it would the original + * per-frame deltas. Returns the number of frames sent (each consumed a wire + * sequence), so the caller can align {@code fsnAtZero}. Stops early and fails + * the loop on a send error or an entry too large for the cap. + */ + private int sendDictCatchUp() { + int cap = client.getServerMaxBatchSize(); + // Symbol-bytes budget per frame, leaving room for the 12-byte header and + // the two delta-section varints. cap <= 0 means the server advertised no + // limit -> send the whole dictionary in one frame. + int budget = cap > 0 ? Math.max(1, cap - QwpConstants.HEADER_SIZE - 16) : Integer.MAX_VALUE; + int framesSent = 0; + int chunkStartId = 0; + long chunkStartAddr = sentDictBytesAddr; + int chunkSymbols = 0; + long chunkBytes = 0; + long p = sentDictBytesAddr; + long limit = sentDictBytesAddr + sentDictBytesLen; + while (p < limit) { + long entryStart = p; + long len = readVarintAt(p, limit); // entry length prefix; varintEnd -> just past prefix + long entryEnd = varintEnd + len; // just past [len varint][utf8 bytes] + long entryBytes = entryEnd - entryStart; + if (entryBytes > budget) { + fail(new LineSenderException( + "symbol dictionary entry too large for the server batch cap during catch-up [" + + "entryBytes=" + entryBytes + ", budget=" + budget + ']')); + return framesSent; + } + if (chunkSymbols > 0 && chunkBytes + entryBytes > budget) { + if (!sendCatchUpChunk(chunkStartId, chunkSymbols, chunkStartAddr, (int) chunkBytes)) { + return framesSent; + } + framesSent++; + chunkStartId += chunkSymbols; + chunkStartAddr = entryStart; + chunkSymbols = 0; + chunkBytes = 0; + } + chunkSymbols++; + chunkBytes += entryBytes; + p = entryEnd; + } + if (chunkSymbols > 0 && sendCatchUpChunk(chunkStartId, chunkSymbols, chunkStartAddr, (int) chunkBytes)) { + framesSent++; + } + return framesSent; + } + + /** + * Sends one table-less catch-up frame carrying dictionary ids + * {@code [deltaStart .. deltaStart+deltaCount)}. Returns false and fails the + * loop on a send error. + */ + private boolean sendCatchUpChunk(int deltaStart, int deltaCount, long symbolsAddr, int symbolsLen) { + int payloadLen = NativeBufferWriter.varintSize(deltaStart) + + NativeBufferWriter.varintSize(deltaCount) + + symbolsLen; + int frameLen = QwpConstants.HEADER_SIZE + payloadLen; + long frame = Unsafe.malloc(frameLen, MemoryTag.NATIVE_DEFAULT); + try { + Unsafe.getUnsafe().putByte(frame, (byte) 'Q'); + Unsafe.getUnsafe().putByte(frame + 1, (byte) 'W'); + Unsafe.getUnsafe().putByte(frame + 2, (byte) 'P'); + Unsafe.getUnsafe().putByte(frame + 3, (byte) '1'); + Unsafe.getUnsafe().putByte(frame + 4, (byte) client.getServerQwpVersion()); + Unsafe.getUnsafe().putByte(frame + QwpConstants.HEADER_OFFSET_FLAGS, + (byte) (QwpConstants.FLAG_GORILLA | QwpConstants.FLAG_DELTA_SYMBOL_DICT)); + Unsafe.getUnsafe().putShort(frame + 6, (short) 0); // tableCount + Unsafe.getUnsafe().putInt(frame + 8, payloadLen); + long q = writeVarintAt(frame + QwpConstants.HEADER_SIZE, deltaStart); + q = writeVarintAt(q, deltaCount); + Unsafe.getUnsafe().copyMemory(symbolsAddr, q, symbolsLen); + client.sendBinary(frame, frameLen); + } catch (Throwable t) { + fail(t); + return false; + } finally { + Unsafe.free(frame, frameLen, MemoryTag.NATIVE_DEFAULT); + } + nextWireSeq++; // this catch-up chunk consumed a wire sequence + lastFrameOrPingNanos = System.nanoTime(); + totalFramesSent.incrementAndGet(); + return true; + } + + private static long writeVarintAt(long addr, long value) { + while (value > 0x7F) { + Unsafe.getUnsafe().putByte(addr++, (byte) ((value & 0x7F) | 0x80)); + value >>>= 7; + } + Unsafe.getUnsafe().putByte(addr++, (byte) value); + return addr; + } + private boolean tryReceiveAcks() { boolean any = false; try { @@ -1949,12 +2220,38 @@ private boolean trySendOne() { if (frameEnd > pub) { return false; // payload not fully published yet } + if (deltaDictEnabled) { + // Torn-dictionary guard. In normal operation a delta frame's start id + // never exceeds the dictionary coverage established so far (replayed + // frames overlap the catch-up dict; fresh frames extend it + // contiguously). A gap here means the persisted dictionary was torn -- + // almost always by a host/power crash, which leaves segment frames on + // disk but loses recently-written dictionary entries (SF, like the rest + // of the store, is process-crash durable but not host-crash durable). + // Sending the frame would corrupt the table (the server would null-pad + // the missing ids), so fail terminally instead; the unreplayable data + // must be resent. + int deltaStart = frameDeltaStart(base + sendOffset + MmapSegment.FRAME_HEADER_SIZE, payloadLen); + if (deltaStart > sentDictCount) { + recordFatal(new LineSenderException( + "recovered store-and-forward symbol dictionary is incomplete (likely a host crash): " + + "frame delta start " + deltaStart + " exceeds recovered dictionary size " + + sentDictCount + "; cannot replay without corrupting data -- resend required")); + return false; + } + } try { client.sendBinary(base + sendOffset + MmapSegment.FRAME_HEADER_SIZE, payloadLen); } catch (Throwable t) { fail(t); return false; } + if (deltaDictEnabled) { + // Mirror the symbols this frame introduced so a later reconnect can + // rebuild the whole dictionary. Idempotent on replay: a frame whose + // delta we already hold advances nothing. + accumulateSentDict(base + sendOffset + MmapSegment.FRAME_HEADER_SIZE, payloadLen); + } lastFrameOrPingNanos = System.nanoTime(); sendOffset = frameEnd; long fsnSent = fsnAtZero + nextWireSeq; @@ -1984,8 +2281,11 @@ void positionCursorForStart() { // starts past it. Zero wire cost, no recycle. tryRetireOrphanTail(); long replayStart = engine.ackedFsn() + 1L; - this.fsnAtZero = replayStart; - this.nextWireSeq = 0L; + // Recovery / orphan-drain seed the dictionary mirror, so the initial + // connection may also need a catch-up (client is non-null in the + // sync-start and drainer paths; null in async-initial, where swapClient + // handles it on the first connect). + setWireBaselineWithCatchUp(replayStart); positionCursorAt(replayStart); } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java new file mode 100644 index 00000000..4c2674b9 --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java @@ -0,0 +1,379 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.qwp.client.sf.cursor; + +import io.questdb.client.std.Files; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.ObjList; +import io.questdb.client.std.QuietCloseable; +import io.questdb.client.std.Unsafe; +import io.questdb.client.std.str.Utf8s; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Append-only, per-slot persistence of the global symbol dictionary that a + * store-and-forward sender ships to the server with delta encoding. Lives at + * {@code /.symbol-dict} alongside the segment files, the slot lock and + * {@code .ack-watermark}. + *

+ * Delta-encoded SF frames are NOT self-sufficient: a frame carries only the + * symbols it introduces, so recovering (process restart) or draining (orphan + * adoption) a slot requires re-registering the whole dictionary on the fresh + * server before those frames replay. This file is that dictionary. Unlike + * {@link AckWatermark} -- a discardable optimization protected by a + * {@code max()} clamp -- this file is load-bearing: a surviving frame + * that references an id missing from it is unrecoverable. It is therefore held + * to a stronger durability contract. + *

+ * Layout (little-endian): + * 

+ *   offset 0: u32 magic = 'SYD1'
+ *   offset 4: u8  version = 1
+ *   offset 5: 3 bytes reserved (zero)
+ *   offset 8: entries, each [len: varint][utf8 bytes], in ascending global-id order
+ *
+ * Symbol id {@code i} is the {@code i}-th entry (ids are dense and assigned + * sequentially from 0), so no id needs to be stored. + *

+ * Durability / write-ahead ordering: the producer appends the symbols a + * frame introduces BEFORE that frame is published to the ring, but does NOT + * fsync -- matching the rest of store-and-forward, which is page-cache (not + * disk) durable. This ordering is sufficient for a process/JVM crash: the + * page cache survives, so both the dictionary and the frames survive and the + * dictionary is a superset of every recoverable frame's references. It is NOT + * sufficient for a host/power crash, where unflushed pages can be lost out + * of order and the dictionary may end up torn relative to the frames it serves -- + * exactly as the segment frames themselves may be lost on a host crash. A torn + * dictionary is caught at replay by the send loop's guard, which fails loudly + * (the unreplayable data must be resent) rather than corrupting the target table. + *

+ * A torn trailing entry from a crash mid-append is self-healing: {@link #open} + * stops parsing at the first incomplete entry and the next append overwrites it. + *

+ * Lifecycle: single-writer (the producer / user thread). Read once at + * {@link #open} to seed in-memory state on recovery or orphan-drain. Owner + * (the engine) closes it. Not thread-safe for concurrent writers. + */ +public final class PersistedSymbolDict implements QuietCloseable { + + /** + * Filename within the slot directory. Dot-prefixed so directory + * enumerators that filter by the {@code .sfa} suffix (segment recovery, + * OrphanScanner, trim) skip it automatically. + */ + public static final String FILE_NAME = ".symbol-dict"; + static final int FILE_MAGIC = 0x31445953; // 'SYD1' little-endian + static final int HEADER_SIZE = 8; + static final byte VERSION = 1; + // Guards against a hostile/corrupt varint length driving a huge allocation + // or a runaway parse. Symbols are short; this is a generous ceiling. + private static final int MAX_ENTRY_LEN = 1 << 20; + private static final Logger LOG = LoggerFactory.getLogger(PersistedSymbolDict.class); + private final int fd; + // In-memory copy of the entry region [len][utf8]... exactly as on disk, + // populated only when open() recovered existing entries (recovery / + // orphan-drain). Zero/empty for a freshly created file. Consumed once to + // seed the send loop's catch-up mirror and the producer's id map. + private final long loadedEntriesAddr; + private final int loadedEntriesLen; + private long appendOffset; + private boolean closed; + private long scratchAddr; + private int scratchCap; + private int size; + + private PersistedSymbolDict(int fd, long appendOffset, int size, long loadedEntriesAddr, int loadedEntriesLen) { + this.fd = fd; + this.appendOffset = appendOffset; + this.size = size; + this.loadedEntriesAddr = loadedEntriesAddr; + this.loadedEntriesLen = loadedEntriesLen; + } + + /** + * Opens (creating if absent) the dictionary file in {@code slotDir}. An + * existing file is parsed and its complete entries are loaded into memory + * (see {@link #loadedEntriesAddr()}); a missing or invalid file is (re)created + * with a fresh header. Returns {@code null} on any I/O failure -- the caller + * then falls back to full-dictionary (self-sufficient) frames for this slot, + * so a broken side-file degrades gracefully rather than aborting the sender. + */ + public static PersistedSymbolDict open(String slotDir) { + String filePath = slotDir + "/" + FILE_NAME; + long existing = Files.exists(filePath) ? Files.length(filePath) : -1L; + if (existing >= HEADER_SIZE) { + PersistedSymbolDict d = openExisting(filePath, existing); + if (d != null) { + return d; + } + // Fall through to a clean re-create: a header/parse failure on an + // existing file means it cannot be trusted for delta replay. + } + return openFresh(filePath); + } + + /** + * Best-effort removal of a stale dictionary file. Used at fresh-start (a + * stale dict with no segments behind it is meaningless) and at fully-drained + * close (the slot is empty, nothing references the dictionary any more), + * mirroring {@link AckWatermark#removeOrphan}. + */ + public static void removeOrphan(String slotDir) { + Files.remove(slotDir + "/" + FILE_NAME); + } + + /** + * Appends one symbol, extending the on-disk dictionary. The caller appends a + * frame's new symbols BEFORE publishing that frame, so the write ordering + * (dictionary entry before referencing frame) holds; no fsync is performed + * (see the class-level durability note). Assigns the next dense id implicitly + * (the entry's position). + */ + public void appendSymbol(CharSequence symbol) { + if (closed) { + return; + } + int utf8Len = Utf8s.utf8Bytes(symbol); + int varLen = varintSize(utf8Len); + int recLen = varLen + utf8Len; + ensureScratch(recLen); + long p = writeVarint(scratchAddr, utf8Len); + if (utf8Len > 0) { + Utf8s.strCpyUtf8(symbol, p, utf8Len); + } + long written = Files.write(fd, scratchAddr, recLen, appendOffset); + if (written != recLen) { + throw new IllegalStateException("short write to " + FILE_NAME + + " [expected=" + recLen + ", actual=" + written + ']'); + } + appendOffset += recLen; + size++; + } + + @Override + public void close() { + if (closed) { + return; + } + closed = true; + if (loadedEntriesAddr != 0L) { + Unsafe.free(loadedEntriesAddr, loadedEntriesLen, MemoryTag.NATIVE_DEFAULT); + } + if (scratchAddr != 0L) { + Unsafe.free(scratchAddr, scratchCap, MemoryTag.NATIVE_DEFAULT); + scratchAddr = 0L; + scratchCap = 0; + } + if (fd >= 0) { + Files.close(fd); + } + } + + /** + * Base address of the loaded entry region -- the concatenated + * {@code [len][utf8]} bytes of every recovered symbol in id order, exactly + * as a delta section carries them. Zero when nothing was recovered. + */ + public long loadedEntriesAddr() { + return loadedEntriesAddr; + } + + /** + * Byte length of {@link #loadedEntriesAddr()}. + */ + public int loadedEntriesLen() { + return loadedEntriesLen; + } + + /** + * Materialises the loaded entries as symbol strings in ascending-id order + * (entry {@code i} is symbol id {@code i}). Used once on recovery to + * repopulate the producer's global dictionary. Empty when nothing was + * recovered. + */ + public ObjList readLoadedSymbols() { + ObjList out = new ObjList<>(Math.max(size, 1)); + long p = loadedEntriesAddr; + long limit = p + loadedEntriesLen; + for (int i = 0; i < size && p < limit; i++) { + long len = 0; + int shift = 0; + while (p < limit) { + byte b = Unsafe.getUnsafe().getByte(p++); + len |= (long) (b & 0x7F) << shift; + if ((b & 0x80) == 0) { + break; + } + shift += 7; + } + if (p + len > limit) { + break; // defensive: torn tail (should not happen past parse in open) + } + out.add(Utf8s.stringFromUtf8Bytes(p, p + len)); + p += len; + } + return out; + } + + /** + * Number of symbols the dictionary holds (highest id + 1). + */ + public int size() { + return size; + } + + private static PersistedSymbolDict openExisting(String filePath, long fileLen) { + int fd = Files.openRW(filePath); + if (fd < 0) { + LOG.warn("symbol dict {} could not be opened (rc={}); recreating", filePath, fd); + return null; + } + long buf = 0L; + try { + int len = (int) fileLen; + buf = Unsafe.malloc(len, MemoryTag.NATIVE_DEFAULT); + long read = Files.read(fd, buf, len, 0); + if (read != len || Unsafe.getUnsafe().getInt(buf) != FILE_MAGIC) { + LOG.warn("symbol dict {} unreadable or bad magic; recreating", filePath); + Unsafe.free(buf, len, MemoryTag.NATIVE_DEFAULT); + Files.close(fd); + return null; + } + // Parse complete entries starting after the header; stop at the + // first torn/incomplete trailing entry. + int pos = HEADER_SIZE; + int count = 0; + while (pos < len) { + long[] vr = decodeVarint(buf, pos, len); + if (vr == null) { + break; // torn length varint + } + int entryLen = (int) vr[0]; + int next = (int) vr[1]; + if (entryLen < 0 || entryLen > MAX_ENTRY_LEN || (long) next + entryLen > len) { + break; // torn/oversized entry -- self-healing tail + } + pos = next + entryLen; + count++; + } + int entriesLen = pos - HEADER_SIZE; + long entriesAddr = 0L; + if (entriesLen > 0) { + entriesAddr = Unsafe.malloc(entriesLen, MemoryTag.NATIVE_DEFAULT); + Unsafe.getUnsafe().copyMemory(buf + HEADER_SIZE, entriesAddr, entriesLen); + } + Unsafe.free(buf, len, MemoryTag.NATIVE_DEFAULT); + buf = 0L; + // appendOffset lands just past the last complete entry, so the next + // append overwrites any torn trailing bytes. + return new PersistedSymbolDict(fd, HEADER_SIZE + entriesLen, count, entriesAddr, entriesLen); + } catch (Throwable t) { + if (buf != 0L) { + Unsafe.free(buf, (int) fileLen, MemoryTag.NATIVE_DEFAULT); + } + Files.close(fd); + LOG.warn("symbol dict {} recovery failed ({}); recreating", filePath, String.valueOf(t)); + return null; + } + } + + private static PersistedSymbolDict openFresh(String filePath) { + int fd = Files.openCleanRW(filePath); + if (fd < 0) { + LOG.warn("symbol dict {} could not be created (rc={}); proceeding without it", filePath, fd); + return null; + } + long hdr = Unsafe.malloc(HEADER_SIZE, MemoryTag.NATIVE_DEFAULT); + try { + Unsafe.getUnsafe().putInt(hdr, FILE_MAGIC); + Unsafe.getUnsafe().putByte(hdr + 4, VERSION); + Unsafe.getUnsafe().putByte(hdr + 5, (byte) 0); + Unsafe.getUnsafe().putByte(hdr + 6, (byte) 0); + Unsafe.getUnsafe().putByte(hdr + 7, (byte) 0); + long written = Files.write(fd, hdr, HEADER_SIZE, 0); + if (written != HEADER_SIZE) { + Files.close(fd); + Files.remove(filePath); + LOG.warn("symbol dict {} header write failed; proceeding without it", filePath); + return null; + } + } finally { + Unsafe.free(hdr, HEADER_SIZE, MemoryTag.NATIVE_DEFAULT); + } + return new PersistedSymbolDict(fd, HEADER_SIZE, 0, 0L, 0); + } + + /** + * Decodes an unsigned LEB128 varint from {@code buf[pos..limit)}. Returns + * {@code [value, newPos]} or {@code null} if the varint is truncated + * (torn tail). + */ + private static long[] decodeVarint(long buf, int pos, int limit) { + long value = 0; + int shift = 0; + int cur = pos; + while (cur < limit) { + byte b = Unsafe.getUnsafe().getByte(buf + cur); + cur++; + value |= (long) (b & 0x7F) << shift; + if ((b & 0x80) == 0) { + return new long[]{value, cur}; + } + shift += 7; + if (shift > 35) { + return null; // implausible for an entry length; treat as torn + } + } + return null; + } + + private static int varintSize(long value) { + int n = 1; + while (value > 0x7F) { + value >>>= 7; + n++; + } + return n; + } + + private static long writeVarint(long addr, long value) { + while (value > 0x7F) { + Unsafe.getUnsafe().putByte(addr++, (byte) ((value & 0x7F) | 0x80)); + value >>>= 7; + } + Unsafe.getUnsafe().putByte(addr++, (byte) value); + return addr; + } + + private void ensureScratch(int required) { + if (scratchCap >= required) { + return; + } + int newCap = Math.max(required, Math.max(256, scratchCap * 2)); + scratchAddr = Unsafe.realloc(scratchAddr, scratchCap, newCap, MemoryTag.NATIVE_DEFAULT); + scratchCap = newCap; + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictCatchUpTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictCatchUpTest.java new file mode 100644 index 00000000..5e9af978 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictCatchUpTest.java @@ -0,0 +1,325 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client; + +import io.questdb.client.Sender; +import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer; +import org.junit.Assert; +import org.junit.Test; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; + +/** + * Verifies the delta symbol-dictionary catch-up on reconnect (memory-mode). + *

+ * When a memory-mode sender reconnects, the server it lands on has an empty + * dictionary (the server discards it on every disconnect). Because the producer + * ships monotonic deltas -- each symbol id once -- a naive replay would leave the + * fresh server with a dictionary gap. The I/O thread prevents this by sending a + * full-dictionary catch-up frame before any post-reconnect traffic. This test + * reconstructs the server's per-connection dictionary from the captured wire + * bytes and asserts it stays complete and gap-free across the reconnect. + */ +public class DeltaDictCatchUpTest { + + @Test + public void testReconnectCatchUpRebuildsDictionary() throws Exception { + // Connection 1: send "alpha" (id 0), ACK it, then drop the socket so the + // sender reconnects. Connection 2 (fresh, empty dict): send "beta" (id 1). + // Without catch-up, connection 2's first data frame would carry + // deltaStart=1 and the fresh server would never learn id 0. + CatchUpHandler handler = new CatchUpHandler(); + try (TestWebSocketServer server = new TestWebSocketServer(handler)) { + int port = server.getPort(); + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + + try (Sender sender = Sender.fromConfig("ws::addr=localhost:" + port + ";")) { + sender.table("t").symbol("s", "alpha").longColumn("v", 1L).atNow(); + sender.flush(); + waitFor(() -> handler.dictFor(1).size() >= 1, 5_000); + + // Let the I/O loop observe the server-side close before the next + // batch, so batch 2 is what drives the reconnect + catch-up. + Thread.sleep(200); + + sender.table("t").symbol("s", "beta").longColumn("v", 2L).atNow(); + sender.flush(); + waitFor(() -> handler.connectionsAccepted.get() >= 2 + && handler.dictFor(2).size() >= 2, 5_000); + } + + // The fresh (2nd) connection's dictionary, rebuilt purely from the + // frames it received, must hold both symbols contiguously with no + // null gap -- exactly what the catch-up frame guarantees. + List conn2 = handler.dictFor(2); + Assert.assertTrue("2nd connection saw a catch-up frame with 0 tables", + handler.sawZeroTableFrameOnConn2); + Assert.assertEquals("2nd connection dictionary size", 2, conn2.size()); + Assert.assertEquals("alpha", conn2.get(0)); + Assert.assertEquals("beta", conn2.get(1)); + } + } + + @Test + public void testReconnectCatchUpSplitsLargeDictionaryAcrossFrames() throws Exception { + // Connection 1 registers 40 ten-character symbols (~440 dictionary bytes), + // then drops once the server has learned them all. On reconnect the fresh + // server has an empty dictionary, so the I/O thread must replay all 40 as a + // catch-up -- but the server advertises a 160-byte batch cap, so the whole + // dictionary cannot fit in a single frame. The catch-up therefore splits + // into several contiguous zero-table frames that the fresh server stitches + // back into a complete, gap-free dictionary. + final int symbolCount = 40; + SplitCatchUpHandler handler = new SplitCatchUpHandler(symbolCount); + try (TestWebSocketServer server = new TestWebSocketServer(handler)) { + server.setAdvertisedMaxBatchSize(160); // small cap forces the catch-up to split + int port = server.getPort(); + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + + try (Sender sender = Sender.fromConfig("ws::addr=localhost:" + port + ";")) { + // One row per flush so each frame stays under the 160-byte cap; the + // sent dictionary still accumulates all 40 symbols on connection 1. + for (int i = 0; i < symbolCount; i++) { + sender.table("t").symbol("s", symbolName(i)).longColumn("v", i).atNow(); + sender.flush(); + } + // Wait until connection 1 has learned every symbol, so the sender's + // sent-dictionary mirror (the catch-up source) holds all of them. + waitFor(() -> handler.dictFor(1).size() >= symbolCount, 10_000); + + // Let the I/O loop observe the server-side close before the next + // batch, so batch 2 drives the reconnect + split catch-up. + Thread.sleep(200); + + sender.table("t").symbol("s", symbolName(symbolCount)).longColumn("v", symbolCount).atNow(); + sender.flush(); + waitFor(() -> handler.connectionsAccepted.get() >= 2 + && handler.dictFor(2).size() >= symbolCount + 1, 10_000); + } + + // Connection 2's dictionary, rebuilt purely from the frames it received, + // must hold every symbol contiguously with no null gap -- the split + // catch-up frames reassemble exactly. + List conn2 = handler.dictFor(2); + Assert.assertEquals("2nd connection dictionary size", symbolCount + 1, conn2.size()); + for (int i = 0; i <= symbolCount; i++) { + Assert.assertEquals("symbol at id " + i, symbolName(i), conn2.get(i)); + } + // The catch-up had to span more than one zero-table frame to stay under + // the advertised cap -- that split is the behaviour under test. + Assert.assertTrue("catch-up split into multiple frames (saw " + + handler.zeroTableFramesOnConn2 + ")", + handler.zeroTableFramesOnConn2 >= 2); + } + } + + private static String symbolName(int i) { + // 10-char symbols so 40 of them clearly exceed the advertised 160-byte cap. + return "symbol" + (1000 + i); + } + + private static int readVarint(byte[] buf, int[] pos) { + int result = 0; + int shift = 0; + while (pos[0] < buf.length) { + int b = buf[pos[0]++] & 0xFF; + result |= (b & 0x7F) << shift; + if ((b & 0x80) == 0) return result; + shift += 7; + if (shift > 28) throw new IllegalStateException("varint too long"); + } + throw new IllegalStateException("varint truncated"); + } + + private static void waitFor(BoolCondition cond, long timeoutMillis) { + long deadline = System.currentTimeMillis() + timeoutMillis; + while (System.currentTimeMillis() < deadline) { + if (cond.test()) return; + try { + Thread.sleep(20); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + Assert.fail("interrupted"); + } + } + Assert.fail("waitFor timed out"); + } + + @FunctionalInterface + private interface BoolCondition { + boolean test(); + } + + /** + * Mirrors the server's per-connection delta-dictionary accumulation: the + * dictionary resets on every new connection, and each frame's delta section + * ({@code setQuick(deltaStart + i)}, null-padding to reach deltaStart) extends + * or overwrites it. A missing catch-up would show up here as a null gap. + */ + private static class CatchUpHandler implements TestWebSocketServer.WebSocketServerHandler { + final AtomicInteger connectionsAccepted = new AtomicInteger(); + volatile boolean sawZeroTableFrameOnConn2; + private final List> dictsByConn = new CopyOnWriteArrayList<>(); + private TestWebSocketServer.ClientHandler currentClient; + private final AtomicLong nextSeq = new AtomicLong(0); + + synchronized List dictFor(int connNumber) { + return connNumber <= dictsByConn.size() + ? dictsByConn.get(connNumber - 1) + : new ArrayList<>(); + } + + @Override + public synchronized void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] data) { + boolean newConnection = currentClient != client; + if (newConnection) { + currentClient = client; + connectionsAccepted.incrementAndGet(); + dictsByConn.add(new ArrayList<>()); // fresh dictionary per connection + } + int connNumber = dictsByConn.size(); + List dict = dictsByConn.get(connNumber - 1); + accumulate(data, dict); + if (connNumber == 2 && tableCount(data) == 0) { + sawZeroTableFrameOnConn2 = true; + } + try { + client.sendBinary(buildAck(nextSeq.getAndIncrement())); + // Drop the first connection right after ACKing its only frame, + // forcing the sender to reconnect onto a fresh dictionary. + if (connNumber == 1) { + Thread.sleep(50); + client.close(); + } + } catch (IOException | InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException(e); + } + } + + private static void accumulate(byte[] frame, List dict) { + final byte FLAG_DELTA_SYMBOL_DICT = 0x08; + if (frame.length < 12 || (frame[5] & FLAG_DELTA_SYMBOL_DICT) == 0) { + return; + } + int[] pos = {12}; // just past the 12-byte QWP header + int deltaStart = readVarint(frame, pos); + int deltaCount = readVarint(frame, pos); + while (dict.size() < deltaStart) { + dict.add(null); // null-pad, mirroring the server + } + for (int i = 0; i < deltaCount; i++) { + int len = readVarint(frame, pos); + String symbol = new String(frame, pos[0], len, StandardCharsets.UTF_8); + pos[0] += len; + int idx = deltaStart + i; + while (dict.size() <= idx) { + dict.add(null); + } + dict.set(idx, symbol); + } + } + + private static byte[] buildAck(long seq) { + byte[] buf = new byte[1 + 8 + 2]; + ByteBuffer bb = ByteBuffer.wrap(buf).order(ByteOrder.LITTLE_ENDIAN); + bb.put((byte) 0x00); + bb.putLong(seq); + bb.putShort((short) 0); + return buf; + } + + private static int tableCount(byte[] frame) { + return (frame[6] & 0xFF) | ((frame[7] & 0xFF) << 8); + } + } + + /** + * Like {@link CatchUpHandler}, but drops connection 1 only after it has + * learned the whole batch, and counts the zero-table catch-up frames on + * connection 2 so a test can assert the dictionary replay split across + * several frames to respect the advertised batch cap. + */ + private static class SplitCatchUpHandler implements TestWebSocketServer.WebSocketServerHandler { + final AtomicInteger connectionsAccepted = new AtomicInteger(); + volatile int zeroTableFramesOnConn2; + private final List> dictsByConn = new CopyOnWriteArrayList<>(); + private final int dropConn1AtDictSize; + private final AtomicLong nextSeq = new AtomicLong(0); + private boolean conn1Dropped; + private TestWebSocketServer.ClientHandler currentClient; + + SplitCatchUpHandler(int dropConn1AtDictSize) { + this.dropConn1AtDictSize = dropConn1AtDictSize; + } + + synchronized List dictFor(int connNumber) { + return connNumber <= dictsByConn.size() + ? dictsByConn.get(connNumber - 1) + : new ArrayList<>(); + } + + @Override + public synchronized void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] data) { + boolean newConnection = currentClient != client; + if (newConnection) { + currentClient = client; + connectionsAccepted.incrementAndGet(); + dictsByConn.add(new ArrayList<>()); // fresh dictionary per connection + } + int connNumber = dictsByConn.size(); + List dict = dictsByConn.get(connNumber - 1); + CatchUpHandler.accumulate(data, dict); + if (connNumber == 2 && CatchUpHandler.tableCount(data) == 0) { + zeroTableFramesOnConn2++; + } + try { + client.sendBinary(CatchUpHandler.buildAck(nextSeq.getAndIncrement())); + // Drop connection 1 only once it has learned the entire batch, so + // the sender's sent-dictionary mirror is complete and the reconnect + // catch-up must replay a dictionary larger than the batch cap. + if (connNumber == 1 && !conn1Dropped && dict.size() >= dropConn1AtDictSize) { + conn1Dropped = true; + Thread.sleep(50); + client.close(); + } + } catch (IOException | InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException(e); + } + } + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java new file mode 100644 index 00000000..88afc3ef --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java @@ -0,0 +1,352 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client; + +import io.questdb.client.Sender; +import io.questdb.client.cutlass.line.LineSenderException; +import io.questdb.client.std.Files; +import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer; +import io.questdb.client.test.tools.TestUtils; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.charset.StandardCharsets; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; + +/** + * End-to-end recovery for the delta symbol dictionary in store-and-forward mode. + *

+ * A file-mode sender writes delta-encoded SYMBOL frames (each frame carries only + * the ids it introduces) to a slot but never drains it -- simulating a crash. A + * fresh sender then recovers the slot and replays those non-self-sufficient + * frames to a brand-new server whose dictionary starts empty. Correctness hinges + * on the persisted {@code .symbol-dict}: the recovering sender loads it, the I/O + * thread re-registers the whole dictionary via a catch-up frame, and only then do + * the delta frames replay. This test reconstructs the server-side dictionary from + * the wire and asserts it comes out complete and gap-free. + */ +public class DeltaDictRecoveryTest { + + private static final int DISTINCT_SYMBOLS = 8; + private static final int ROWS = 40; + private String sfDir; + + @Before + public void setUp() { + sfDir = Paths.get(System.getProperty("java.io.tmpdir"), + "qdb-delta-recov-" + System.nanoTime()).toString(); + } + + @After + public void tearDown() { + if (sfDir != null) { + rmDirRec(sfDir); + } + } + + @Test + public void testRecoveredSlotReplaysDeltaFramesAgainstFreshServer() throws Exception { + // Phase 1: silent server (no acks). Sender 1 writes symbol rows and + // close-fast (no drain), leaving unacked delta frames + a persisted + // dictionary in the slot. + try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) { + int port = silent.getPort(); + silent.start(); + Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS)); + + String pad = TestUtils.repeat("x", 64); + String cfg = "ws::addr=localhost:" + port + + ";sf_dir=" + sfDir + + ";sf_max_bytes=4096" + + ";close_flush_timeout_millis=0;"; + try (Sender s1 = Sender.fromConfig(cfg)) { + for (int i = 0; i < ROWS; i++) { + s1.table("m") + .symbol("s", "sym-" + (i % DISTINCT_SYMBOLS)) + .stringColumn("p", pad) + .longColumn("v", i) + .atNow(); + s1.flush(); + } + } + } + + // Phase 2: fresh server that reconstructs its per-connection dictionary + // from the delta sections. Sender 2 recovers the slot and replays. + DictReconstructingHandler handler = new DictReconstructingHandler(); + try (TestWebSocketServer good = new TestWebSocketServer(handler)) { + int port = good.getPort(); + good.start(); + Assert.assertTrue(good.awaitStart(5, TimeUnit.SECONDS)); + + String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";"; + try (Sender ignored = Sender.fromConfig(cfg)) { + long deadline = System.currentTimeMillis() + 5_000; + while (System.currentTimeMillis() < deadline + && handler.maxDictSize() < DISTINCT_SYMBOLS) { + Thread.sleep(20); + } + } + + // The recovering sender must have re-registered the dictionary via a + // catch-up (0-table) frame before replaying delta frames. + Assert.assertTrue("recovery sent a full-dictionary catch-up frame", + handler.sawCatchUpFrame); + // The reconstructed dictionary must be complete and gap-free: exactly + // the DISTINCT_SYMBOLS symbols, no null padding left by a missing id. + List dict = handler.dictSnapshot(); + Assert.assertEquals("reconstructed dictionary size", DISTINCT_SYMBOLS, dict.size()); + for (int i = 0; i < DISTINCT_SYMBOLS; i++) { + Assert.assertEquals("dictionary id " + i, "sym-" + i, dict.get(i)); + } + } + } + + @Test + public void testTornDictionaryFailsCleanlyInsteadOfCorrupting() throws Exception { + // Phase 1: each row introduces a new symbol, so frame i carries deltaStart=i. + // Silent server + close-fast leaves all frames unacked in the slot. + try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) { + int port = silent.getPort(); + silent.start(); + Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS)); + String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + + ";close_flush_timeout_millis=0;"; + try (Sender s1 = Sender.fromConfig(cfg)) { + for (int i = 0; i < 6; i++) { + s1.table("m").symbol("s", "sym-" + i).longColumn("v", i).atNow(); + s1.flush(); + } + } + } + + // Simulate a host/power crash: the segment frames survive but the persisted + // dictionary is lost, and the ack watermark was left mid-stream. Truncate + // .symbol-dict to its 8-byte header (0 symbols) and stamp the watermark at + // FSN 2, so recovery replays from FSN 3 -- a frame with deltaStart=3. + java.nio.file.Path slot = Paths.get(sfDir, "default"); + java.nio.file.Path dict = slot.resolve(".symbol-dict"); + byte[] header = Arrays.copyOf(java.nio.file.Files.readAllBytes(dict), 8); + java.nio.file.Files.write(dict, header); + writeAckWatermark(slot.resolve(".ack-watermark"), 2); + + // Phase 2: recover against a fresh counting server. The replay guard must + // fire (frame deltaStart 3 > recovered dictionary size 0) and fail terminally + // rather than send a gapped frame that would corrupt the table. + CountingHandler handler = new CountingHandler(); + try (TestWebSocketServer good = new TestWebSocketServer(handler)) { + int port = good.getPort(); + good.start(); + Assert.assertTrue(good.awaitStart(5, TimeUnit.SECONDS)); + + String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";"; + LineSenderException terminal = null; + Sender s2 = Sender.fromConfig(cfg); + try { + Thread.sleep(1_000); // let the I/O loop attempt replay and hit the guard + } finally { + try { + s2.close(); + } catch (LineSenderException e) { + terminal = e; + } + } + Assert.assertEquals("no frame may be replayed to a fresh server with a torn dictionary", + 0, handler.frames.get()); + Assert.assertNotNull("a torn dictionary must surface a terminal error", terminal); + Assert.assertTrue(terminal.getMessage(), + terminal.getMessage().contains("symbol dictionary is incomplete")); + } + } + + private static void writeAckWatermark(java.nio.file.Path path, long fsn) throws IOException { + byte[] buf = new byte[16]; + ByteBuffer bb = ByteBuffer.wrap(buf).order(ByteOrder.LITTLE_ENDIAN); + bb.putInt(0x31574B41); // 'AKW1' + bb.putInt(0); // reserved + bb.putLong(fsn); + java.nio.file.Files.write(path, buf); + } + + private static int readVarint(byte[] buf, int[] pos) { + int result = 0; + int shift = 0; + while (pos[0] < buf.length) { + int b = buf[pos[0]++] & 0xFF; + result |= (b & 0x7F) << shift; + if ((b & 0x80) == 0) { + return result; + } + shift += 7; + if (shift > 28) { + throw new IllegalStateException("varint too long"); + } + } + throw new IllegalStateException("varint truncated"); + } + + private static void rmDirRec(String dir) { + if (!Files.exists(dir)) { + return; + } + long find = Files.findFirst(dir); + if (find > 0) { + try { + int rc = 1; + while (rc > 0) { + String name = Files.utf8ToString(Files.findName(find)); + if (name != null && !".".equals(name) && !"..".equals(name)) { + String child = dir + "/" + name; + if (!Files.remove(child)) { + rmDirRec(child); + } + } + rc = Files.findNext(find); + } + } finally { + Files.findClose(find); + } + } + Files.remove(dir); + } + + /** + * Reconstructs the per-connection symbol dictionary from delta sections, + * mirroring the server's {@code setQuick(deltaStart + i)} + null-padding. + */ + private static class DictReconstructingHandler implements TestWebSocketServer.WebSocketServerHandler { + volatile boolean sawCatchUpFrame; + private final List dict = new ArrayList<>(); + private final AtomicLong nextSeq = new AtomicLong(0); + private TestWebSocketServer.ClientHandler currentClient; + + synchronized List dictSnapshot() { + return new ArrayList<>(dict); + } + + synchronized int maxDictSize() { + return dict.size(); + } + + @Override + public synchronized void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] data) { + if (currentClient != client) { + currentClient = client; + dict.clear(); // fresh server dictionary per connection + } + accumulate(data); + if (tableCount(data) == 0 && hasDelta(data)) { + sawCatchUpFrame = true; + } + try { + client.sendBinary(buildAck(nextSeq.getAndIncrement())); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + private static byte[] buildAck(long seq) { + byte[] buf = new byte[1 + 8 + 2]; + ByteBuffer bb = ByteBuffer.wrap(buf).order(ByteOrder.LITTLE_ENDIAN); + bb.put((byte) 0x00); + bb.putLong(seq); + bb.putShort((short) 0); + return buf; + } + + private static boolean hasDelta(byte[] frame) { + return frame.length >= 12 && (frame[5] & 0x08) != 0; + } + + private static int tableCount(byte[] frame) { + return (frame[6] & 0xFF) | ((frame[7] & 0xFF) << 8); + } + + private void accumulate(byte[] frame) { + if (!hasDelta(frame)) { + return; + } + int[] pos = {12}; + int deltaStart = readVarint(frame, pos); + int deltaCount = readVarint(frame, pos); + while (dict.size() < deltaStart) { + dict.add(null); + } + for (int i = 0; i < deltaCount; i++) { + int len = readVarint(frame, pos); + String sym = new String(frame, pos[0], len, StandardCharsets.UTF_8); + pos[0] += len; + int idx = deltaStart + i; + while (dict.size() <= idx) { + dict.add(null); + } + dict.set(idx, sym); + } + } + } + + private static class SilentHandler implements TestWebSocketServer.WebSocketServerHandler { + @Override + public void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] data) { + // never acks -- sender leaves everything unacked in the slot + } + } + + /** Counts every binary frame it receives and acks it. */ + private static class CountingHandler implements TestWebSocketServer.WebSocketServerHandler { + final AtomicInteger frames = new AtomicInteger(); + private final AtomicLong nextSeq = new AtomicLong(0); + + @Override + public void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] data) { + frames.incrementAndGet(); + try { + client.sendBinary(buildAck(nextSeq.getAndIncrement())); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + private static byte[] buildAck(long seq) { + byte[] buf = new byte[1 + 8 + 2]; + ByteBuffer bb = ByteBuffer.wrap(buf).order(ByteOrder.LITTLE_ENDIAN); + bb.put((byte) 0x00); + bb.putLong(seq); + bb.putShort((short) 0); + return buf; + } + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/ReconnectTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/ReconnectTest.java index 36553388..7ba7a1fe 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/ReconnectTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/ReconnectTest.java @@ -52,9 +52,10 @@ * disconnect = sender broken, every subsequent batch threw. Reconnect * machinery now handles transient drops: detect, build a fresh client * via the registered factory, reset wire state, and reposition the replay - * cursor at {@code engine.ackedFsn() + 1}. Cursor frames are self-sufficient - * (every frame carries full schema + full symbol-dict delta), so post-reconnect - * replay needs no producer-side schema-reset signal. + * cursor at {@code engine.ackedFsn() + 1}. Every frame carries its full schema + * inline; the symbol dictionary is either shipped in full per frame (file-mode + * store-and-forward) or re-registered by an I/O-thread catch-up frame before + * replay (memory-mode), so post-reconnect replay needs no producer-side reset. *

* This commit covers the mechanics with a single-attempt retry; backoff, * per-outage time cap, and auth-failure detection follow. diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java index 45275528..b0a1b5a7 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java @@ -32,23 +32,26 @@ import java.io.IOException; import java.nio.ByteBuffer; import java.nio.ByteOrder; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Comparator; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; /** - * Pins down the "every frame on disk is self-sufficient" rule for the symbol - * dictionary. + * Pins down how the symbol dictionary is framed on the wire. *

- * The cursor SF path used to elide previously-sent symbols on subsequent - * batches over the same connection, emitting a delta-dict that carried only - * the new entries. That's wrong for SF: the bytes survive process restarts and - * replay against fresh server connections (post-reconnect, or via a background - * drainer adopting an orphan slot). A delta that references symbol ids the new - * server has never seen is unrecoverable. + * Both engine modes ship monotonic deltas -- each symbol id travels once, + * not the whole dictionary per message -- which is the bandwidth win this feature + * adds. The I/O thread re-registers the dictionary with a catch-up frame whenever + * it (re)connects, so a fresh server can resolve the non-self-sufficient delta + * frames that follow. *

- * Today every frame must carry a complete symbol-dict delta starting at id 0 - * (column schemas travel inline on the first batch too). This test asserts the - * symbol-dict invariant on the wire. + * The modes differ only in where the catch-up's dictionary comes from: memory + * mode keeps it in an in-process mirror; file-backed store-and-forward keeps it in + * a per-slot {@code .symbol-dict} file so a recovered or orphan-drained slot (a + * fresh process with no in-memory mirror) can rebuild it. This test asserts the + * monotonic wire framing in both modes and the presence of that dictionary file. */ public class SelfSufficientFramesTest { @@ -56,12 +59,72 @@ public class SelfSufficientFramesTest { private static final int DELTA_START_OFFSET = 12; @Test - public void testEverySymbolBatchIncludesFullDeltaFromZero() throws Exception { - // Send two batches against the same connection, each with a - // distinct symbol value. With the old schema-ref/delta encoding, - // batch 2 would emit deltaStart=1, deltaCount=1 — only the new - // symbol. With self-sufficient frames, batch 2 must emit - // deltaStart=0 covering BOTH symbols. + public void testFileModeShipsMonotonicDeltaAndPersistsDict() throws Exception { + // File-backed SF also ships monotonic deltas now: batch 2 carries only + // "beta" (deltaStart=1). The dictionary is durably kept in .symbol-dict + // so a recovered/orphan-drained slot can rebuild it. + Path sfDir = Files.createTempDirectory("qwp-sf-selfsufficient"); + CapturingHandler handler = new CapturingHandler(); + try (TestWebSocketServer server = new TestWebSocketServer(handler)) { + int port = server.getPort(); + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + + String config = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";"; + // The engine places slot files under sf_dir/ (default "default"). + Path dictFile = sfDir.resolve("default").resolve(".symbol-dict"); + try (Sender sender = Sender.fromConfig(config)) { + sender.table("foo").symbol("s", "alpha").longColumn("v", 1L).atNow(); + sender.flush(); + waitFor(() -> handler.batches.size() >= 1, 5_000); + + sender.table("foo").symbol("s", "beta").longColumn("v", 2L).atNow(); + sender.flush(); + waitFor(() -> handler.batches.size() >= 2, 5_000); + + // Check the persisted dictionary while the sender is live: a + // fully-drained close intentionally unlinks it (slot cleanup). + Assert.assertTrue("persisted dictionary file exists", Files.exists(dictFile)); + byte[] dict = Files.readAllBytes(dictFile); + Assert.assertTrue("dictionary retains alpha", containsUtf8(dict, "alpha")); + Assert.assertTrue("dictionary retains beta", containsUtf8(dict, "beta")); + } + + Assert.assertEquals("expected 2 captured batches", 2, handler.batches.size()); + byte[] b1 = handler.batches.get(0); + byte[] b2 = handler.batches.get(1); + + Assert.assertEquals("batch 1 deltaStart must be 0", + 0, readVarint(b1, DELTA_START_OFFSET)); + Assert.assertEquals("batch 1 deltaCount must be 1", 1, readVarint(b1, DELTA_START_OFFSET + 1)); + // batch 2 ships ONLY beta as a delta from id 1. + Assert.assertEquals("batch 2 deltaStart must be 1 (monotonic)", + 1, readVarint(b2, DELTA_START_OFFSET)); + Assert.assertEquals("batch 2 deltaCount must be 1 (only the new symbol)", + 1, readVarint(b2, DELTA_START_OFFSET + 1)); + } finally { + rmDir(sfDir); + } + } + + private static boolean containsUtf8(byte[] haystack, String needle) { + byte[] n = needle.getBytes(java.nio.charset.StandardCharsets.UTF_8); + outer: + for (int i = 0; i + n.length <= haystack.length; i++) { + for (int j = 0; j < n.length; j++) { + if (haystack[i + j] != n[j]) { + continue outer; + } + } + return true; + } + return false; + } + + @Test + public void testMemoryModeShipsMonotonicDelta() throws Exception { + // Memory-mode (no sf_dir): each symbol id ships once. Batch 2 carries + // only "beta" as a delta starting at id 1, not the whole dictionary. CapturingHandler handler = new CapturingHandler(); try (TestWebSocketServer server = new TestWebSocketServer(handler)) { int port = server.getPort(); @@ -82,29 +145,17 @@ public void testEverySymbolBatchIncludesFullDeltaFromZero() throws Exception { byte[] b1 = handler.batches.get(0); byte[] b2 = handler.batches.get(1); - // The deltaStart varint sits right after the 12-byte header. - // For self-sufficient frames it must be 0 (single byte 0x00) - // in BOTH batches — regardless of how many symbols the prior - // batch already shipped. - int deltaStart1 = readVarint(b1, DELTA_START_OFFSET); - int deltaStart2 = readVarint(b2, DELTA_START_OFFSET); - Assert.assertEquals("batch 1 deltaStart must be 0", 0, deltaStart1); - Assert.assertEquals("batch 2 deltaStart must be 0 (self-sufficient)", - 0, deltaStart2); - - // batch 2 must include >= 2 symbols in its delta dict (alpha - // from the prior batch + beta from this one). The varint at - // DELTA_START_OFFSET+1 is deltaCount. - int deltaCount2 = readVarint(b2, DELTA_START_OFFSET + 1); - Assert.assertTrue("batch 2 must redefine at least 2 symbols, got " + deltaCount2, - deltaCount2 >= 2); - - // Sanity: batch 2 should NOT be much smaller than batch 1 — - // with schema-ref/delta encoding it would have been; with - // self-sufficient frames the size is in the same ballpark. - Assert.assertTrue("batch 2 (" + b2.length + " bytes) must not be drastically smaller than batch 1 (" - + b1.length + ")", - b2.length >= b1.length / 2); + // Batch 1 introduces alpha at id 0. + Assert.assertEquals("batch 1 deltaStart must be 0", + 0, readVarint(b1, DELTA_START_OFFSET)); + Assert.assertEquals("batch 1 deltaCount must be 1", + 1, readVarint(b1, DELTA_START_OFFSET + 1)); + + // Batch 2 ships ONLY beta as a delta from id 1. + Assert.assertEquals("batch 2 deltaStart must be 1 (monotonic)", + 1, readVarint(b2, DELTA_START_OFFSET)); + Assert.assertEquals("batch 2 deltaCount must be 1 (only the new symbol)", + 1, readVarint(b2, DELTA_START_OFFSET + 1)); } } @@ -122,6 +173,25 @@ private static int readVarint(byte[] buf, int offset) { throw new IllegalStateException("varint truncated"); } + private static void rmDir(Path dir) { + try { + if (dir == null || !Files.exists(dir)) { + return; + } + Files.walk(dir) + .sorted(Comparator.reverseOrder()) + .forEach(p -> { + try { + Files.deleteIfExists(p); + } catch (IOException ignored) { + // best-effort + } + }); + } catch (IOException ignored) { + // best-effort + } + } + private static void waitFor(BoolCondition cond, long timeoutMillis) { long deadline = System.currentTimeMillis() + timeoutMillis; while (System.currentTimeMillis() < deadline) { @@ -157,6 +227,7 @@ public void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] dat } } + // Mirrors WebSocketResponse STATUS_OK layout: status u8 | sequence u64 | table_count u16 static byte[] buildAck(long seq) { byte[] buf = new byte[1 + 8 + 2]; ByteBuffer bb = ByteBuffer.wrap(buf).order(ByteOrder.LITTLE_ENDIAN); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java new file mode 100644 index 00000000..10148468 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java @@ -0,0 +1,219 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client.sf.cursor; + +import io.questdb.client.cutlass.qwp.client.sf.cursor.PersistedSymbolDict; +import io.questdb.client.std.ObjList; +import org.junit.Assert; +import org.junit.Test; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.Comparator; + +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; + +public class PersistedSymbolDictTest { + + @Test + public void testAppendPersistsAcrossReopen() throws Exception { + assertMemoryLeak(() -> { + Path dir = Files.createTempDirectory("qwp-symdict"); + try { + PersistedSymbolDict d = PersistedSymbolDict.open(dir.toString()); + Assert.assertNotNull(d); + try { + Assert.assertEquals(0, d.size()); + d.appendSymbol("AAPL"); + d.appendSymbol("GOOG"); + d.appendSymbol("MSFT"); + Assert.assertEquals(3, d.size()); + } finally { + d.close(); + } + + // Reopen: entries recovered in id order. + PersistedSymbolDict reopened = PersistedSymbolDict.open(dir.toString()); + Assert.assertNotNull(reopened); + try { + Assert.assertEquals(3, reopened.size()); + ObjList symbols = reopened.readLoadedSymbols(); + Assert.assertEquals(3, symbols.size()); + Assert.assertEquals("AAPL", symbols.getQuick(0)); + Assert.assertEquals("GOOG", symbols.getQuick(1)); + Assert.assertEquals("MSFT", symbols.getQuick(2)); + Assert.assertTrue(reopened.loadedEntriesLen() > 0); + + // Appending after recovery continues from the recovered tip. + reopened.appendSymbol("TSLA"); + Assert.assertEquals(4, reopened.size()); + } finally { + reopened.close(); + } + + PersistedSymbolDict third = PersistedSymbolDict.open(dir.toString()); + try { + Assert.assertEquals(4, third.size()); + Assert.assertEquals("TSLA", third.readLoadedSymbols().getQuick(3)); + } finally { + third.close(); + } + } finally { + rmDir(dir); + } + }); + } + + @Test + public void testBadMagicIsRecreatedEmpty() throws Exception { + assertMemoryLeak(() -> { + Path dir = Files.createTempDirectory("qwp-symdict"); + try { + // A file with the right size but garbage content (bad magic). + Path f = dir.resolve(".symbol-dict"); + Files.write(f, new byte[]{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}); + PersistedSymbolDict d = PersistedSymbolDict.open(dir.toString()); + Assert.assertNotNull(d); + try { + Assert.assertEquals("bad-magic file recreated empty", 0, d.size()); + d.appendSymbol("X"); + Assert.assertEquals(1, d.size()); + } finally { + d.close(); + } + } finally { + rmDir(dir); + } + }); + } + + @Test + public void testEmptySymbolRoundTrips() throws Exception { + assertMemoryLeak(() -> { + Path dir = Files.createTempDirectory("qwp-symdict"); + try { + PersistedSymbolDict d = PersistedSymbolDict.open(dir.toString()); + try { + d.appendSymbol(""); + d.appendSymbol("nonempty"); + } finally { + d.close(); + } + PersistedSymbolDict re = PersistedSymbolDict.open(dir.toString()); + try { + Assert.assertEquals(2, re.size()); + ObjList s = re.readLoadedSymbols(); + Assert.assertEquals("", s.getQuick(0)); + Assert.assertEquals("nonempty", s.getQuick(1)); + } finally { + re.close(); + } + } finally { + rmDir(dir); + } + }); + } + + @Test + public void testRemoveOrphanDeletesFile() throws Exception { + assertMemoryLeak(() -> { + Path dir = Files.createTempDirectory("qwp-symdict"); + try { + PersistedSymbolDict d = PersistedSymbolDict.open(dir.toString()); + d.appendSymbol("A"); + d.close(); + Path f = dir.resolve(".symbol-dict"); + Assert.assertTrue(Files.exists(f)); + PersistedSymbolDict.removeOrphan(dir.toString()); + Assert.assertFalse(Files.exists(f)); + } finally { + rmDir(dir); + } + }); + } + + @Test + public void testTornTrailingEntrySelfHeals() throws Exception { + assertMemoryLeak(() -> { + Path dir = Files.createTempDirectory("qwp-symdict"); + try { + // Write two complete entries, then a torn trailing record: a + // length prefix of 5 followed by only 2 bytes (crash mid-append). + PersistedSymbolDict d = PersistedSymbolDict.open(dir.toString()); + d.appendSymbol("one"); + d.appendSymbol("two"); + d.close(); + + Path f = dir.resolve(".symbol-dict"); + Files.write(f, new byte[]{(byte) 5, (byte) 'x', (byte) 'y'}, + StandardOpenOption.APPEND); + + // Reopen: the torn tail is ignored; only the two complete entries load. + PersistedSymbolDict re = PersistedSymbolDict.open(dir.toString()); + try { + Assert.assertEquals(2, re.size()); + ObjList s = re.readLoadedSymbols(); + Assert.assertEquals("one", s.getQuick(0)); + Assert.assertEquals("two", s.getQuick(1)); + // The next append overwrites the torn tail, keeping the file consistent. + re.appendSymbol("three"); + Assert.assertEquals(3, re.size()); + } finally { + re.close(); + } + + PersistedSymbolDict re2 = PersistedSymbolDict.open(dir.toString()); + try { + Assert.assertEquals(3, re2.size()); + Assert.assertEquals("three", re2.readLoadedSymbols().getQuick(2)); + } finally { + re2.close(); + } + } finally { + rmDir(dir); + } + }); + } + + private static void rmDir(Path dir) { + try { + if (dir == null || !Files.exists(dir)) { + return; + } + Files.walk(dir) + .sorted(Comparator.reverseOrder()) + .forEach(p -> { + try { + Files.deleteIfExists(p); + } catch (IOException ignored) { + } + }); + } catch (IOException ignored) { + } + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/websocket/TestWebSocketServer.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/websocket/TestWebSocketServer.java index 6d4c5ef0..5e009d67 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/websocket/TestWebSocketServer.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/websocket/TestWebSocketServer.java @@ -108,6 +108,11 @@ public class TestWebSocketServer implements Closeable { // 401, 403, 404, 426, 503, etc. that the failover loop should // classify per failover.md §6. private volatile String rejectingStatusReason; + // When > 0, 101 upgrade responses advertise this value as the + // X-QWP-Max-Batch-Size header, capping the QWP message size the client + // builds. Lets a test force the delta-dictionary catch-up to split across + // several frames. Live-updatable via setAdvertisedMaxBatchSize(). + private volatile int advertisedMaxBatchSize; public TestWebSocketServer(WebSocketServerHandler handler) throws IOException { this(handler, false); @@ -221,6 +226,14 @@ public int liveConnectionCount() { return liveConnections.get(); } + /** + * Advertises {@code X-QWP-Max-Batch-Size: } on subsequent + * handshakes (live update). Pass {@code 0} to stop advertising a cap. + */ + public void setAdvertisedMaxBatchSize(int maxBatchSize) { + this.advertisedMaxBatchSize = maxBatchSize; + } + /** * Replaces the advertised role for subsequent handshakes (live update). */ @@ -603,6 +616,10 @@ private boolean performHandshake() throws IOException { if (role != null) { sb.append("X-QuestDB-Role: ").append(role).append("\r\n"); } + int maxBatch = advertisedMaxBatchSize; + if (maxBatch > 0) { + sb.append("X-QWP-Max-Batch-Size: ").append(maxBatch).append("\r\n"); + } sb.append("\r\n"); out.write(sb.toString().getBytes(StandardCharsets.US_ASCII)); out.flush(); From 30922301ef77d53ab68dab3da1d8f4e9dc4db1f3 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Thu, 9 Jul 2026 17:10:41 +0100 Subject: [PATCH 02/96] Contain QWP dict catch-up send failures in the reconnect loop The symbol-dictionary catch-up called fail() on a send error, but the catch-up runs inside connectLoop (via swapClient) and, on the initial connect, on the caller thread (via start() -> positionCursorForStart). Calling fail() there re-entered connectLoop. On a reconnect this corrupted the wire mapping: the outer setWireBaselineWithCatchUp overwrote fsnAtZero while nextWireSeq kept the nested attempt's value, so a later ACK translated through engine.acknowledge(fsnAtZero + wireSeq) and trimmed un-acked frames from the store-and-forward log -- silent data loss. A flapping connection recursed connectLoop until the stack overflowed into a terminal, turning a transient outage into a hard failure (breaking Invariant B). On the initial connect the same fail() ran connectLoop on the caller thread and blocked Sender construction forever. sendDictCatchUp and sendCatchUpChunk now throw CatchUpSendException instead of calling fail(). connectLoop's own retry catch handles the swapClient path (one non-re-entrant reconnect with backoff); trySendOne's orphan-retire re-anchor turns it into a fresh fail() from the I/O loop body; start() drops the dead client so the I/O thread reconnects and re-sends the catch-up off the caller thread. A single dictionary entry too large for the server batch cap is non-retriable, so it latches a terminal (recordFatal) rather than looping -- also removing the oversized-entry reconnect livelock. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../sf/cursor/CursorWebSocketSendLoop.java | 109 ++++++++++++++---- 1 file changed, 84 insertions(+), 25 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java index cbf948b0..25ebcadd 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java @@ -1030,7 +1030,28 @@ public synchronized void start() { // walks back to the lowest unacked frame so sealed-segment data // actually reaches the wire — without it, start() would skip // straight to the active and orphan everything in sealed. - positionCursorForStart(); + try { + positionCursorForStart(); + } catch (CatchUpSendException e) { + // A recovered sender re-registers its dictionary with a catch-up on + // the very first connect. Here that runs on the CALLER thread (sync + // start), so we must NOT let it drive connectLoop -- that would block + // Sender construction forever on a transient outage. Drop the dead + // client instead: the I/O thread then reconnects via + // attemptInitialConnect -> swapClient and re-sends the catch-up off + // this thread. If the failure was already terminal (recordFatal set + // running=false, e.g. an entry too large for the batch cap), the I/O + // thread simply winds down and checkError() surfaces it. + WebSocketClient dead = client; + client = null; + if (dead != null) { + try { + dead.close(); + } catch (Throwable ignored) { + // best-effort + } + } + } Thread t = new Thread(this::ioLoop, "qdb-cursor-ws-io"); t.setDaemon(true); try { @@ -2028,14 +2049,6 @@ private long readVarintAt(long p, long limit) { return value; } - /** - * Builds a table-less catch-up frame carrying the full symbol dictionary - * ({@code deltaStart=0, deltaCount=sentDictCount}) and sends it to the freshly - * connected server, then consumes wire seq 0 for it. {@code sendBinary} copies - * the bytes synchronously, so the temporary frame buffer is freed immediately. - * On send failure it fails the loop, which recycles and retries on the next - * connection. - */ /** * Re-registers the whole symbol dictionary on a fresh connection, split into * as many table-less frames as the server's advertised batch cap requires so @@ -2043,8 +2056,10 @@ private long readVarintAt(long p, long limit) { * Each chunk carries a contiguous id range {@code [start .. start+count)}, in * order, so the server accumulates them exactly as it would the original * per-frame deltas. Returns the number of frames sent (each consumed a wire - * sequence), so the caller can align {@code fsnAtZero}. Stops early and fails - * the loop on a send error or an entry too large for the cap. + * sequence), so the caller can align {@code fsnAtZero}. Throws {@link + * CatchUpSendException} on a send error (retriable -- the caller reconnects); + * a single entry too large for the cap is non-retriable, so it latches a + * terminal before throwing. */ private int sendDictCatchUp() { int cap = client.getServerMaxBatchSize(); @@ -2065,15 +2080,19 @@ private int sendDictCatchUp() { long entryEnd = varintEnd + len; // just past [len varint][utf8 bytes] long entryBytes = entryEnd - entryStart; if (entryBytes > budget) { - fail(new LineSenderException( + // Non-retriable: the entry will not shrink and the same cluster + // re-advertises the same cap, so reconnecting would livelock. + // Latch a terminal (the data must be resent after the cap is + // raised) rather than calling fail() -- which, from inside the + // catch-up, would re-enter connectLoop (see CatchUpSendException). + LineSenderException err = new LineSenderException( "symbol dictionary entry too large for the server batch cap during catch-up [" - + "entryBytes=" + entryBytes + ", budget=" + budget + ']')); - return framesSent; + + "entryBytes=" + entryBytes + ", budget=" + budget + ']'); + recordFatal(err); + throw new CatchUpSendException(err); } if (chunkSymbols > 0 && chunkBytes + entryBytes > budget) { - if (!sendCatchUpChunk(chunkStartId, chunkSymbols, chunkStartAddr, (int) chunkBytes)) { - return framesSent; - } + sendCatchUpChunk(chunkStartId, chunkSymbols, chunkStartAddr, (int) chunkBytes); framesSent++; chunkStartId += chunkSymbols; chunkStartAddr = entryStart; @@ -2084,7 +2103,8 @@ private int sendDictCatchUp() { chunkBytes += entryBytes; p = entryEnd; } - if (chunkSymbols > 0 && sendCatchUpChunk(chunkStartId, chunkSymbols, chunkStartAddr, (int) chunkBytes)) { + if (chunkSymbols > 0) { + sendCatchUpChunk(chunkStartId, chunkSymbols, chunkStartAddr, (int) chunkBytes); framesSent++; } return framesSent; @@ -2092,10 +2112,12 @@ private int sendDictCatchUp() { /** * Sends one table-less catch-up frame carrying dictionary ids - * {@code [deltaStart .. deltaStart+deltaCount)}. Returns false and fails the - * loop on a send error. + * {@code [deltaStart .. deltaStart+deltaCount)}. Throws {@link + * CatchUpSendException} on a send error instead of calling {@link #fail} + * (see that type for why the catch-up must not re-enter the reconnect loop); + * the caller turns it into a single, non-re-entrant reconnect. */ - private boolean sendCatchUpChunk(int deltaStart, int deltaCount, long symbolsAddr, int symbolsLen) { + private void sendCatchUpChunk(int deltaStart, int deltaCount, long symbolsAddr, int symbolsLen) { int payloadLen = NativeBufferWriter.varintSize(deltaStart) + NativeBufferWriter.varintSize(deltaCount) + symbolsLen; @@ -2116,15 +2138,20 @@ private boolean sendCatchUpChunk(int deltaStart, int deltaCount, long symbolsAdd Unsafe.getUnsafe().copyMemory(symbolsAddr, q, symbolsLen); client.sendBinary(frame, frameLen); } catch (Throwable t) { - fail(t); - return false; + // Do NOT fail() here -- see CatchUpSendException. Signal the failure + // up so exactly one non-re-entrant reconnect follows. A JVM Error is + // never a transient reconnect case; let it propagate as-is so the + // I/O loop latches it terminal rather than looping on it. + if (t instanceof Error) { + throw (Error) t; + } + throw new CatchUpSendException(t); } finally { Unsafe.free(frame, frameLen, MemoryTag.NATIVE_DEFAULT); } nextWireSeq++; // this catch-up chunk consumed a wire sequence lastFrameOrPingNanos = System.nanoTime(); totalFramesSent.incrementAndGet(); - return true; } private static long writeVarintAt(long addr, long value) { @@ -2169,7 +2196,15 @@ private boolean trySendOne() { // Nothing sent on this connection yet: re-anchor in place past // the retired tail. The wireSeq<->FSN mapping is untouched // because no wire sequence has been consumed. - positionCursorForStart(); + try { + positionCursorForStart(); + } catch (CatchUpSendException e) { + // Re-anchor's catch-up send failed. fail() here is a fresh, + // non-re-entrant connectLoop entry from the I/O loop body -- + // the same recovery a normal trySendOne send failure takes. + fail(e.getCause()); + return false; + } return true; } // Frames were already sent on this connection: the linear @@ -2336,6 +2371,30 @@ public interface ReconnectFactory { WebSocketClient reconnect() throws Exception; } + /** + * Signals that a symbol-dictionary catch-up frame could not be sent on the + * current connection. Thrown by {@link #sendDictCatchUp}/{@link + * #sendCatchUpChunk} instead of calling {@link #fail}: the catch-up runs + * inside {@link #connectLoop} (via {@link #swapClient}) and, on the initial + * connect, inside {@link #start()} / {@link #trySendOne} on the caller + * thread. Calling {@code fail()} from there would re-enter {@code + * connectLoop} -- corrupting the {@code fsnAtZero}/{@code nextWireSeq} wire + * mapping (a subsequent ACK then trims un-acked frames) and growing the + * stack until it overflows into a terminal, breaking the "retry a transient + * outage forever" invariant -- or run {@code connectLoop} on the caller + * thread and block {@code Sender} construction indefinitely. Each catch + * site instead turns it into ONE non-re-entrant reconnect: {@code + * connectLoop}'s own retry catch (swapClient path), a fresh {@code fail()} + * from the I/O loop body (trySendOne path), or dropping the dead client so + * the I/O thread reconnects (start path). A JVM {@code Error} is never + * wrapped -- it must stay terminal. + */ + private static final class CatchUpSendException extends RuntimeException { + CatchUpSendException(Throwable cause) { + super(cause); + } + } + /** * One slot in the pendingDurable FIFO. Holds a wireSeq plus the per-table * (name, seqTxn) pairs from its OK frame. Empty entries (tableCount = 0) From 9f3f7bdc8b8f1109781d6f92196d283068d15eb9 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Thu, 9 Jul 2026 17:24:13 +0100 Subject: [PATCH 03/96] Resume the QWP symbol-dict write-ahead from the durable size persistNewSymbolsBeforePublish keyed the append range off sentMaxSymbolId+1. That watermark only advances after the whole frame is published, whereas PersistedSymbolDict.size() advances per persisted entry. If a mid-batch appendSymbol threw (a short write on a full disk), the symbols before the failing one were already durable but the frame was not published, so sentMaxSymbolId stayed put. A retry then re-keyed from sentMaxSymbolId+1 and re-appended that already-persisted prefix, duplicating entries and breaking the dense id->symbol mapping recovery relies on (entry i must be symbol id i) -- a torn dictionary that re-registers the wrong symbols on the fresh server, or diverges the producer's watermark from the I/O thread's mirror. Resume from pd.size() instead: it is exactly the count already durable, so the retry continues past the persisted prefix (the next append overwrites any torn trailing bytes) without duplicating. In the happy path pd.size() equals sentMaxSymbolId+1, so behaviour is unchanged. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../qwp/client/QwpWebSocketSender.java | 20 ++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index 286596c4..216794ff 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -3665,12 +3665,22 @@ private void persistNewSymbolsBeforePublish() { if (pd == null) { return; } - int from = sentMaxSymbolId + 1; + // Resume from the dictionary's own durable size, NOT sentMaxSymbolId+1. + // appendSymbol advances pd.size() per persisted entry, whereas + // sentMaxSymbolId only advances after the WHOLE frame is published (via + // advanceSentMaxSymbolId, after activeBuffer.write). If a prior + // appendSymbol threw mid-batch (a short write -- disk full/quota), the + // frame was not published and sentMaxSymbolId stayed put, while the + // symbols before the failing one are already on disk. Keying the resume + // point off sentMaxSymbolId+1 would then re-append that persisted prefix + // on the retry, duplicating entries and corrupting the dense id->symbol + // mapping recovery relies on (position i must be symbol id i). pd.size() + // resumes exactly past what is already durable -- the next appendSymbol + // overwrites any torn trailing bytes at appendOffset -- so the + // write-ahead is idempotent under retry. In the happy path pd.size() + // equals sentMaxSymbolId+1, so behaviour is unchanged. int to = currentBatchMaxSymbolId; - if (to < from) { - return; - } - for (int id = from; id <= to; id++) { + for (int id = pd.size(); id <= to; id++) { pd.appendSymbol(globalSymbolDictionary.getSymbol(id)); } } From 09cd706af158f97ecb1299bd14024f52a5a6cb56 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Thu, 9 Jul 2026 17:24:23 +0100 Subject: [PATCH 04/96] Guard the oversized catch-up entry terminal Add a regression test: a dictionary entry larger than the reconnect server's per-chunk catch-up budget must latch a clean terminal, not reconnect-loop. Connection 1 advertises no cap so a ~200-byte symbol registers into the sent-dictionary mirror; the handler then shrinks the advertised cap and drops the socket, so the reconnect's catch-up cannot re-ship the entry. The test asserts the surfaced terminal names the catch-up path ("... during catch-up"). Reverting the fix (entry-too-large calling fail() again) fails this test with a StackOverflowError on the I/O thread -- the catch-up re-entering connectLoop -- confirming the guard bites both ways. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../qwp/client/DeltaDictCatchUpTest.java | 102 ++++++++++++++++++ 1 file changed, 102 insertions(+) diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictCatchUpTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictCatchUpTest.java index 5e9af978..1af55bc3 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictCatchUpTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictCatchUpTest.java @@ -25,7 +25,9 @@ package io.questdb.client.test.cutlass.qwp.client; import io.questdb.client.Sender; +import io.questdb.client.cutlass.line.LineSenderException; import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer; +import io.questdb.client.test.tools.TestUtils; import org.junit.Assert; import org.junit.Test; @@ -92,6 +94,65 @@ public void testReconnectCatchUpRebuildsDictionary() throws Exception { } } + @Test + public void testCatchUpEntryTooLargeForCapFailsTerminally() throws Exception { + // A dictionary entry that exceeds the reconnect server's per-chunk budget + // (cap - HEADER_SIZE - 16) cannot be shipped as a catch-up chunk. + // sendDictCatchUp must latch a clean terminal ("... during catch-up") + // rather than call fail(): pre-fix the oversized entry drove an endless + // reconnect loop (the entry never shrinks and the same cluster + // re-advertises the same cap) and re-entered connectLoop from the catch-up. + // + // Connection 1 advertises no cap, so the ~202-byte symbol registers and + // enters the sent-dictionary mirror. The handler then shrinks the + // advertised cap to 160 (catch-up budget 132) and drops the socket, so the + // reconnect's catch-up cannot re-ship the symbol. (One fixed cap can't do + // this: the client refuses to SEND a single-table frame over the cap, and + // that data frame is always larger than the bare catch-up entry.) + CapShrinkHandler handler = new CapShrinkHandler(); + try (TestWebSocketServer server = new TestWebSocketServer(handler)) { + handler.setServer(server); + int port = server.getPort(); + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + + String bigSymbol = TestUtils.repeat("x", 200); // ~202-byte dict entry + LineSenderException terminal = null; + Sender sender = Sender.fromConfig("ws::addr=localhost:" + port + ";"); + try { + sender.table("t").symbol("s", bigSymbol).longColumn("v", 1L).atNow(); + sender.flush(); + // The terminal latches on the I/O thread once the reconnect's + // catch-up hits the oversized entry; it surfaces to the producer + // on a subsequent flush. Poll a bounded time for it. The polling + // rows use a small symbol that fits the shrunk cap, so the + // producer-side cap check never fires and flush() surfaces the + // I/O thread's catch-up terminal via checkError. + long deadline = System.currentTimeMillis() + 10_000; + while (System.currentTimeMillis() < deadline && terminal == null) { + try { + sender.table("t").symbol("s", "y").longColumn("v", 2L).atNow(); + sender.flush(); + Thread.sleep(20); + } catch (LineSenderException e) { + terminal = e; + } + } + } finally { + try { + sender.close(); + } catch (LineSenderException e) { + if (terminal == null) { + terminal = e; + } + } + } + Assert.assertNotNull("an oversized catch-up entry must surface a terminal", terminal); + Assert.assertTrue("terminal must come from the catch-up path, got: " + terminal.getMessage(), + terminal.getMessage().contains("during catch-up")); + } + } + @Test public void testReconnectCatchUpSplitsLargeDictionaryAcrossFrames() throws Exception { // Connection 1 registers 40 ten-character symbols (~440 dictionary bytes), @@ -267,6 +328,47 @@ private static int tableCount(byte[] frame) { } } + /** + * ACKs connection 1's frames with no advertised cap (so an oversized symbol + * registers), then -- once connection 1 has sent something -- shrinks the + * advertised batch cap and drops the socket. The reconnect (connection 2) + * therefore advertises a cap whose catch-up budget is too small for the + * symbol, exercising the oversized-entry terminal in sendDictCatchUp. + */ + private static class CapShrinkHandler implements TestWebSocketServer.WebSocketServerHandler { + final AtomicInteger connectionsAccepted = new AtomicInteger(); + private final AtomicLong nextSeq = new AtomicLong(0); + private TestWebSocketServer.ClientHandler currentClient; + private volatile TestWebSocketServer server; + + void setServer(TestWebSocketServer server) { + this.server = server; + } + + @Override + public synchronized void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] data) { + if (currentClient != client) { + currentClient = client; + connectionsAccepted.incrementAndGet(); + } + try { + client.sendBinary(CatchUpHandler.buildAck(nextSeq.getAndIncrement())); + if (connectionsAccepted.get() == 1) { + // Connection 1 registered the big symbol. Shrink the cap so the + // reconnect's catch-up budget can't fit it, then drop to force + // the reconnect. Setting the cap before the close (not from the + // test thread after it) removes the set-vs-reconnect race. + server.setAdvertisedMaxBatchSize(160); // catch-up budget = 132 + Thread.sleep(50); + client.close(); + } + } catch (IOException | InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException(e); + } + } + } + /** * Like {@link CatchUpHandler}, but drops connection 1 only after it has * learned the whole batch, and counts the zero-table catch-up frames on From 18dd545381d0f5c8a0a5fb547a26447b1fd32362 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Thu, 9 Jul 2026 18:25:36 +0100 Subject: [PATCH 05/96] Persist the QWP symbol-dict batch in a single write persistNewSymbolsBeforePublish appended each new symbol with its own PersistedSymbolDict.appendSymbol call, and each appendSymbol issues one positioned write. A high-cardinality batch -- one new symbol per row, which is exactly the store-and-forward workload delta encoding targets -- therefore stalled the producer thread with up to one pwrite syscall per row per flush. Add PersistedSymbolDict.appendSymbols(dict, from, to): it encodes the whole [from..to] entry region into scratch once and issues a single positioned write, so a flush that introduces N symbols costs one syscall instead of N. It keeps appendSymbol's durability and idempotency contract -- no fsync, and a short write throws without advancing size, so a retry keyed off size() re-encodes and overwrites at the same offset. PersistedSymbolDictTest.testAppendSymbolsBatchWritesDenseRange checks the batched write produces the same dense, id-ordered file (including an empty symbol mid-range), that an empty range is a no-op, and that a follow-on batch keyed off the recovered size continues without a gap or duplicate. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../qwp/client/QwpWebSocketSender.java | 34 +++++------ .../client/sf/cursor/PersistedSymbolDict.java | 41 +++++++++++++ .../sf/cursor/PersistedSymbolDictTest.java | 59 +++++++++++++++++++ 3 files changed, 117 insertions(+), 17 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index 216794ff..54be9ea0 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -3665,24 +3665,24 @@ private void persistNewSymbolsBeforePublish() { if (pd == null) { return; } - // Resume from the dictionary's own durable size, NOT sentMaxSymbolId+1. - // appendSymbol advances pd.size() per persisted entry, whereas + // Persist [pd.size() .. currentBatchMaxSymbolId] as ONE write. + // + // Resume from the dictionary's own durable size, NOT sentMaxSymbolId+1: + // appendSymbols advances pd.size() only after a full write, whereas // sentMaxSymbolId only advances after the WHOLE frame is published (via - // advanceSentMaxSymbolId, after activeBuffer.write). If a prior - // appendSymbol threw mid-batch (a short write -- disk full/quota), the - // frame was not published and sentMaxSymbolId stayed put, while the - // symbols before the failing one are already on disk. Keying the resume - // point off sentMaxSymbolId+1 would then re-append that persisted prefix - // on the retry, duplicating entries and corrupting the dense id->symbol - // mapping recovery relies on (position i must be symbol id i). pd.size() - // resumes exactly past what is already durable -- the next appendSymbol - // overwrites any torn trailing bytes at appendOffset -- so the - // write-ahead is idempotent under retry. In the happy path pd.size() - // equals sentMaxSymbolId+1, so behaviour is unchanged. - int to = currentBatchMaxSymbolId; - for (int id = pd.size(); id <= to; id++) { - pd.appendSymbol(globalSymbolDictionary.getSymbol(id)); - } + // advanceSentMaxSymbolId, after activeBuffer.write). If a prior persist + // threw (a short write -- disk full/quota), the frame was not published + // and sentMaxSymbolId stayed put, while the symbols before the failing + // one are already on disk. Keying the resume point off sentMaxSymbolId+1 + // would then re-append that persisted prefix on the retry, duplicating + // entries and corrupting the dense id->symbol mapping recovery relies on + // (position i must be symbol id i). pd.size() resumes exactly past what is + // already durable (the write overwrites any torn trailing bytes at + // appendOffset), so the write-ahead is idempotent under retry. In the + // happy path pd.size() equals sentMaxSymbolId+1, so the range -- and the + // behaviour -- are unchanged; the single positioned write just replaces + // the previous one-syscall-per-new-symbol loop. + pd.appendSymbols(globalSymbolDictionary, pd.size(), currentBatchMaxSymbolId); } private void resetSymbolDictStateForNewConnection() { diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java index 4c2674b9..92a0cc78 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java @@ -24,6 +24,7 @@ package io.questdb.client.cutlass.qwp.client.sf.cursor; +import io.questdb.client.cutlass.qwp.client.GlobalSymbolDictionary; import io.questdb.client.std.Files; import io.questdb.client.std.MemoryTag; import io.questdb.client.std.ObjList; @@ -173,6 +174,46 @@ public void appendSymbol(CharSequence symbol) { size++; } + /** + * Appends the dense id range {@code [from .. to]} in a SINGLE write. Encodes + * the whole {@code [len varint][utf8]...} region into scratch first, then + * issues one positioned write -- versus one {@code pwrite(2)} per symbol via + * {@link #appendSymbol}. That per-symbol syscall count is the hot-path cost + * on a high-cardinality batch (one new symbol per row), which is exactly the + * store-and-forward workload delta encoding targets. Callers pass the + * dictionary and the range so the ids resolve to their symbol strings. + *

+ * Same durability and idempotency contract as {@link #appendSymbol}: no + * fsync, and a short write throws WITHOUT advancing {@code size}/{@code + * appendOffset}, so a retry keyed off {@link #size()} re-encodes the same + * range and overwrites at the same offset. No-op when the range is empty or + * the dictionary is closed. + */ + public void appendSymbols(GlobalSymbolDictionary dict, int from, int to) { + if (closed || to < from) { + return; + } + int len = 0; + for (int id = from; id <= to; id++) { + CharSequence symbol = dict.getSymbol(id); + int utf8Len = Utf8s.utf8Bytes(symbol); + int recLen = varintSize(utf8Len) + utf8Len; + ensureScratch(len + recLen); + long p = writeVarint(scratchAddr + len, utf8Len); + if (utf8Len > 0) { + Utf8s.strCpyUtf8(symbol, p, utf8Len); + } + len += recLen; + } + long written = Files.write(fd, scratchAddr, len, appendOffset); + if (written != len) { + throw new IllegalStateException("short write to " + FILE_NAME + + " [expected=" + len + ", actual=" + written + ']'); + } + appendOffset += len; + size += to - from + 1; + } + @Override public void close() { if (closed) { diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java index 10148468..73549689 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java @@ -24,6 +24,7 @@ package io.questdb.client.test.cutlass.qwp.client.sf.cursor; +import io.questdb.client.cutlass.qwp.client.GlobalSymbolDictionary; import io.questdb.client.cutlass.qwp.client.sf.cursor.PersistedSymbolDict; import io.questdb.client.std.ObjList; import org.junit.Assert; @@ -89,6 +90,64 @@ public void testAppendPersistsAcrossReopen() throws Exception { }); } + @Test + public void testAppendSymbolsBatchWritesDenseRange() throws Exception { + // appendSymbols persists a whole id range in one write (the hot-path + // syscall reduction). It must produce the same dense, id-ordered file a + // per-symbol loop would, including an empty symbol mid-range, and a second + // batched call keyed off size() must continue densely (the resume-from- + // durable-size contract the producer relies on). + assertMemoryLeak(() -> { + Path dir = Files.createTempDirectory("qwp-symdict"); + try { + GlobalSymbolDictionary dict = new GlobalSymbolDictionary(); + dict.getOrAddSymbol("AAPL"); // id 0 + dict.getOrAddSymbol(""); // id 1 -- empty symbol mid-range + dict.getOrAddSymbol("MSFT"); // id 2 + dict.getOrAddSymbol("TSLA"); // id 3 + + PersistedSymbolDict d = PersistedSymbolDict.open(dir.toString()); + try { + d.appendSymbols(dict, 0, 3); // one write for all four ids + Assert.assertEquals(4, d.size()); + d.appendSymbols(dict, 4, 3); // empty range (to < from) is a no-op + Assert.assertEquals(4, d.size()); + } finally { + d.close(); + } + + PersistedSymbolDict reopened = PersistedSymbolDict.open(dir.toString()); + try { + Assert.assertEquals(4, reopened.size()); + ObjList symbols = reopened.readLoadedSymbols(); + Assert.assertEquals(4, symbols.size()); + Assert.assertEquals("AAPL", symbols.getQuick(0)); + Assert.assertEquals("", symbols.getQuick(1)); + Assert.assertEquals("MSFT", symbols.getQuick(2)); + Assert.assertEquals("TSLA", symbols.getQuick(3)); + + // A follow-on batch keyed off the recovered size continues + // the dense sequence without a gap or duplicate. + dict.getOrAddSymbol("NVDA"); // id 4 + reopened.appendSymbols(dict, reopened.size(), 4); + Assert.assertEquals(5, reopened.size()); + } finally { + reopened.close(); + } + + PersistedSymbolDict third = PersistedSymbolDict.open(dir.toString()); + try { + Assert.assertEquals(5, third.size()); + Assert.assertEquals("NVDA", third.readLoadedSymbols().getQuick(4)); + } finally { + third.close(); + } + } finally { + rmDir(dir); + } + }); + } + @Test public void testBadMagicIsRecreatedEmpty() throws Exception { assertMemoryLeak(() -> { From 2a1a2d636b4fa31dc264d923f9f285b0189cf0f7 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Thu, 9 Jul 2026 18:25:49 +0100 Subject: [PATCH 06/96] Free the recovery-seeded dict mirror in close() On recovery / orphan-drain the CursorWebSocketSendLoop constructor seeds a native mirror (sentDictBytesAddr) from the slot's persisted dictionary so the first connection can re-register it. That mirror is freed only on ioLoop's exit path, so a loop that is constructed but never runs -- start() never called, or Thread.start() failing before the loop runs, or a close() racing an unstarted loop -- leaked it. close() already safety-nets the client for that same "loop never started" case; the mirror was missed. close() now frees the mirror when the loop never ran (ioThread was null on entry). It does NOT free it when the loop ran: ioLoop's exit owns the free there, and on the failed-stop path the thread may still be mid-send, so touching the mirror would race; a duplicate close observes a zero address and skips. CursorWebSocketSendLoopMirrorLeakTest populates a recoverable slot, then leak-checks constructing an engine + loop over it and closing WITHOUT start(). Reverting the free fails it with a 4096-byte NATIVE_DEFAULT leak. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../sf/cursor/CursorWebSocketSendLoop.java | 17 +++ ...CursorWebSocketSendLoopMirrorLeakTest.java | 141 ++++++++++++++++++ 2 files changed, 158 insertions(+) create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopMirrorLeakTest.java diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java index 25ebcadd..2e6b826c 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java @@ -788,6 +788,12 @@ public synchronized void close() { // after — the latch await is only skipped when the loop never ran. running = false; Thread t = ioThread; + // The symbol-dict mirror (sentDictBytesAddr) is I/O-thread-owned and gets + // freed on ioLoop's exit path. When t == null the loop never ran (start() + // was never called, or t.start() failed before committing ioThread), so + // that free never happens and a seeded (recovery / orphan-drain) mirror + // would leak. Capture that here; free it below, after client teardown. + boolean loopNeverRan = t == null; if (t != null) { LockSupport.unpark(t); // Only await the shutdown latch if the I/O thread actually ran. @@ -846,6 +852,17 @@ public synchronized void close() { } client = null; } + // Free the I/O-thread-owned symbol-dict mirror ONLY when the loop never + // ran (see loopNeverRan). If it ran, ioLoop's exit already freed it -- and + // on the failed-stop path (interrupted latch await, ioThread left set) the + // thread may still be mid-send, so touching the mirror here would race. + // A duplicate close observes sentDictBytesAddr == 0 and skips. + if (loopNeverRan && sentDictBytesAddr != 0) { + Unsafe.free(sentDictBytesAddr, sentDictBytesCapacity, MemoryTag.NATIVE_DEFAULT); + sentDictBytesAddr = 0; + sentDictBytesCapacity = 0; + sentDictBytesLen = 0; + } } /** diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopMirrorLeakTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopMirrorLeakTest.java new file mode 100644 index 00000000..19626ba4 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopMirrorLeakTest.java @@ -0,0 +1,141 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client.sf.cursor; + +import io.questdb.client.Sender; +import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine; +import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorWebSocketSendLoop; +import io.questdb.client.cutlass.qwp.client.sf.cursor.PersistedSymbolDict; +import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer; +import org.junit.Assert; +import org.junit.Test; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Comparator; +import java.util.concurrent.TimeUnit; + +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; + +/** + * Guards the recovery-seeded symbol-dictionary mirror against leaking when the + * I/O loop is constructed but never run. + *

+ * On recovery / orphan-drain the {@link CursorWebSocketSendLoop} constructor + * seeds a native mirror ({@code sentDictBytesAddr}) from the slot's persisted + * dictionary so the first connection can re-register it. That mirror is freed on + * the I/O thread's exit path -- so if the loop is closed WITHOUT ever starting + * (start() never called, or Thread.start() failing before the loop runs), the + * free never happens. {@code close()} must free it in that case. + */ +public class CursorWebSocketSendLoopMirrorLeakTest { + + private static final int DISTINCT_SYMBOLS = 8; + private static final int ROWS = 40; + + @Test + public void testSeededMirrorFreedWhenLoopClosedWithoutStart() throws Exception { + Path sfDir = Files.createTempDirectory("qwp-mirror-leak"); + try { + // Populate a slot with delta frames + a non-empty .symbol-dict, then + // abandon it (silent server, close-fast) -- outside assertMemoryLeak, + // because a full Sender+server round trip is not net-zero on its own. + populateRecoverableSlot(sfDir); + + Path slot = sfDir.resolve("default"); + Assert.assertTrue("populate must leave a persisted dictionary", + Files.exists(slot.resolve(".symbol-dict"))); + + // Only the recovery construct + close is leak-checked: the engine + // recovers (loading the dict), the loop ctor seeds the mirror from it, + // and close() -- with NO start() -- must free every native allocation. + // Pre-fix the seeded mirror leaks here and this assertion fails. + assertMemoryLeak(() -> { + try (CursorSendEngine engine = new CursorSendEngine(slot.toString(), 4096)) { + PersistedSymbolDict pd = engine.getPersistedSymbolDict(); + Assert.assertNotNull("disk-mode engine must open a persisted dict", pd); + Assert.assertTrue("recovery must load the persisted symbols (seeds the mirror)", + pd.size() > 0 && pd.loadedEntriesLen() > 0); + + CursorWebSocketSendLoop loop = new CursorWebSocketSendLoop( + null, engine, 0, 1_000_000L, + () -> { + throw new IOException("no reconnect in this test"); + }, + 0, 0, 1); + // Close without start(): the ctor-seeded mirror is this + // thread's to free, since the I/O loop never ran. + loop.close(); + } + }); + } finally { + rmDir(sfDir); + } + } + + private static void populateRecoverableSlot(Path sfDir) throws Exception { + try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) { + int port = silent.getPort(); + silent.start(); + Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS)); + String cfg = "ws::addr=localhost:" + port + + ";sf_dir=" + sfDir + + ";sf_max_bytes=4096" + + ";close_flush_timeout_millis=0;"; + try (Sender s1 = Sender.fromConfig(cfg)) { + for (int i = 0; i < ROWS; i++) { + s1.table("m") + .symbol("s", "sym-" + (i % DISTINCT_SYMBOLS)) + .longColumn("v", i) + .atNow(); + s1.flush(); + } + } + } + } + + private static void rmDir(Path dir) throws IOException { + if (dir == null || !Files.exists(dir)) { + return; + } + Files.walk(dir) + .sorted(Comparator.reverseOrder()) + .forEach(p -> { + try { + Files.deleteIfExists(p); + } catch (IOException ignored) { + // best-effort + } + }); + } + + private static class SilentHandler implements TestWebSocketServer.WebSocketServerHandler { + @Override + public void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] data) { + // never acks -- the sender leaves everything unacked in the slot + } + } +} From e130da922fd35e981be682ea9609dbfb774f746d Mon Sep 17 00:00:00 2001 From: glasstiger Date: Thu, 9 Jul 2026 18:42:03 +0100 Subject: [PATCH 07/96] Pin the recovery test on the catch-up frame testRecoveredSlotReplaysDeltaFramesAgainstFreshServer never acked in phase 1, so recovery replayed from the very first frame -- whose delta already starts at id 0. The replayed frames were thus self-sufficient from 0, and the reconstructed-dictionary assertions passed whether or not the seeded catch-up carried the right symbols (or any at all). Only the sawCatchUpFrame existence check was load-bearing. Stamp the ack watermark at FSN DISTINCT_SYMBOLS-1 between the phases so recovery replays from the first frame past the symbol-introducing cycle: a frame with deltaStart=DISTINCT_SYMBOLS carrying no new symbols. The early ids it references now exist only in the persisted dictionary, so the reconstructed dictionary is complete solely because the catch-up re-registered them. Verified both ways: with a catch-up that sends a table-less frame but no symbols, the pre-change test still passes (the head frames carry the dictionary) while the stamped test fails at "dictionary id 0 expected sym-0 but was null". Co-Authored-By: Claude Opus 4.8 (1M context) --- .../cutlass/qwp/client/DeltaDictRecoveryTest.java | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java index 88afc3ef..46c2a66b 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java @@ -104,6 +104,20 @@ public void testRecoveredSlotReplaysDeltaFramesAgainstFreshServer() throws Excep } } + // Ack a prefix so recovery does NOT replay from the self-sufficient head. + // Rows 0..DISTINCT_SYMBOLS-1 register all the symbols, so stamping the + // watermark at FSN DISTINCT_SYMBOLS-1 makes recovery replay from FSN + // DISTINCT_SYMBOLS onward -- frames whose delta starts at + // DISTINCT_SYMBOLS and carries NO new symbols (rows past the first cycle + // reuse existing ids). The early ids those frames reference then exist + // ONLY in the persisted dictionary, so the reconstructed dictionary below + // is complete solely because the catch-up frame re-registered them. That + // pins the content assertions to the catch-up: without it (or with a + // broken one) the fresh server would null-pad ids 0..DISTINCT_SYMBOLS-1 + // and the per-id checks would fail. + java.nio.file.Path slot = Paths.get(sfDir, "default"); + writeAckWatermark(slot.resolve(".ack-watermark"), DISTINCT_SYMBOLS - 1); + // Phase 2: fresh server that reconstructs its per-connection dictionary // from the delta sections. Sender 2 recovers the slot and replays. DictReconstructingHandler handler = new DictReconstructingHandler(); From 2b78b0e5e0dd80ad7170cc16cea2e731e75a7331 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Thu, 9 Jul 2026 18:42:13 +0100 Subject: [PATCH 08/96] Cover the disk-mode full-dict fallback When a disk-mode slot's .symbol-dict cannot be opened, the engine reports delta encoding as unavailable and the sender must fall back to self-sufficient frames -- every batch re-ships the whole dictionary from id 0 -- because a recovered slot would have no dictionary to rebuild non-self-sufficient deltas from. Nothing exercised that path. Add a test that plants a directory where the dictionary file belongs, so openRW / openCleanRW fail and open() returns null. It then asserts both batches ship deltaStart=0 and that batch 2 re-ships the whole dictionary (deltaCount=2), rather than the monotonic delta (deltaStart=1, deltaCount=1) the enabled path emits. Verified it bites: forcing isDeltaDictEnabled() to stay true regresses batch 2 to deltaStart=1 and the test fails. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../qwp/client/SelfSufficientFramesTest.java | 56 +++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java index b0a1b5a7..10bd1e6b 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java @@ -107,6 +107,62 @@ public void testFileModeShipsMonotonicDeltaAndPersistsDict() throws Exception { } } + @Test + public void testDiskModeFallsBackToFullDictWhenPersistedDictUnopenable() throws Exception { + // When the per-slot .symbol-dict cannot be opened in disk mode, + // isDeltaDictEnabled() is false and the sender must fall back to + // self-sufficient frames: every batch re-ships the WHOLE dictionary from + // id 0. A recovered / orphan-drained slot then has no dictionary to + // rebuild deltas from, so a monotonic delta would dangle ids on the fresh + // server -- the full-dict frame is the safe degradation. Force the open + // failure by planting a DIRECTORY where the dictionary file belongs: + // openRW / openCleanRW on a directory fails, so open() returns null. + Path sfDir = Files.createTempDirectory("qwp-sf-fallback"); + Path dictPath = sfDir.resolve("default").resolve(".symbol-dict"); + Files.createDirectories(dictPath); // a directory, not a file + Files.createFile(dictPath.resolve("blocker")); // non-empty: cannot be unlinked/rmdir'd + CapturingHandler handler = new CapturingHandler(); + try (TestWebSocketServer server = new TestWebSocketServer(handler)) { + int port = server.getPort(); + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + + String config = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";"; + try (Sender sender = Sender.fromConfig(config)) { + sender.table("foo").symbol("s", "alpha").longColumn("v", 1L).atNow(); + sender.flush(); + waitFor(() -> handler.batches.size() >= 1, 5_000); + + sender.table("foo").symbol("s", "beta").longColumn("v", 2L).atNow(); + sender.flush(); + waitFor(() -> handler.batches.size() >= 2, 5_000); + } + + // The planted directory is untouched -- the dictionary never opened, + // so delta encoding stayed disabled. + Assert.assertTrue("planted .symbol-dict directory must remain (open failed)", + Files.isDirectory(dictPath)); + + Assert.assertEquals("expected 2 captured batches", 2, handler.batches.size()); + byte[] b1 = handler.batches.get(0); + byte[] b2 = handler.batches.get(1); + + // Full-dict fallback: BOTH batches start at id 0, and batch 2 re-ships + // the WHOLE dictionary (alpha + beta), NOT a monotonic delta (which + // would be deltaStart=1, deltaCount=1 as in the test above). + Assert.assertEquals("batch 1 deltaStart must be 0", + 0, readVarint(b1, DELTA_START_OFFSET)); + Assert.assertEquals("batch 1 deltaCount must be 1", + 1, readVarint(b1, DELTA_START_OFFSET + 1)); + Assert.assertEquals("batch 2 deltaStart must be 0 (full-dict fallback, not monotonic)", + 0, readVarint(b2, DELTA_START_OFFSET)); + Assert.assertEquals("batch 2 deltaCount must be 2 (whole dictionary re-shipped)", + 2, readVarint(b2, DELTA_START_OFFSET + 1)); + } finally { + rmDir(sfDir); + } + } + private static boolean containsUtf8(byte[] haystack, String needle) { byte[] n = needle.getBytes(java.nio.charset.StandardCharsets.UTF_8); outer: From 282ac09fbc3ffe920435964780506816d02f9899 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Thu, 9 Jul 2026 18:52:59 +0100 Subject: [PATCH 09/96] Truncate the torn tail when reopening the symbol dict openExisting parsed complete entries and set appendOffset past the last one, but left the file at its full length. A crash mid-append leaves a torn trailing record; if the next append after recovery is SHORTER than that torn tail, it overwrites only the tail's prefix and leaves residue beyond its own end. A later recovery can then mis-parse that residue as a ghost symbol, shifting every subsequent dense id -- so the "self-healing tail" guarantee was not actually airtight. open() now truncates the file to the end of the last complete entry (ftruncate) so nothing survives past appendOffset. Best-effort: a failed truncate falls back to the prior overwrite-from-appendOffset behaviour. testTornTrailingEntrySelfHeals now asserts the file returns to its clean length after the reopen; reverting the truncate fails it (19 vs 16 bytes). Co-Authored-By: Claude Opus 4.8 (1M context) --- .../qwp/client/sf/cursor/PersistedSymbolDict.java | 14 ++++++++++++-- .../client/sf/cursor/PersistedSymbolDictTest.java | 9 ++++++++- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java index 92a0cc78..15179e89 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java @@ -328,8 +328,18 @@ private static PersistedSymbolDict openExisting(String filePath, long fileLen) { } Unsafe.free(buf, len, MemoryTag.NATIVE_DEFAULT); buf = 0L; - // appendOffset lands just past the last complete entry, so the next - // append overwrites any torn trailing bytes. + // Physically drop any torn/stale trailing bytes (a crash mid-append) + // so a LATER, shorter append cannot leave residue past its own end + // that a future recovery mis-parses as a ghost symbol -- which would + // shift every subsequent dense id. Without this, appendOffset already + // lands just past the last complete entry, so the next append + // overwrites FROM there, but only truncation guarantees nothing + // survives BEYOND it. Best-effort: a failed truncate just forgoes the + // hardening and falls back to the overwrite-from-appendOffset behaviour. + long validLen = HEADER_SIZE + entriesLen; + if (validLen < len) { + Files.truncate(fd, validLen); + } return new PersistedSymbolDict(fd, HEADER_SIZE + entriesLen, count, entriesAddr, entriesLen); } catch (Throwable t) { if (buf != 0L) { diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java index 73549689..df6bab21 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java @@ -229,17 +229,24 @@ public void testTornTrailingEntrySelfHeals() throws Exception { d.close(); Path f = dir.resolve(".symbol-dict"); + long cleanLen = Files.size(f); // header + "one" + "two", no tail Files.write(f, new byte[]{(byte) 5, (byte) 'x', (byte) 'y'}, StandardOpenOption.APPEND); + Assert.assertEquals("torn tail present before reopen", cleanLen + 3, Files.size(f)); // Reopen: the torn tail is ignored; only the two complete entries load. PersistedSymbolDict re = PersistedSymbolDict.open(dir.toString()); try { + // open() physically truncates the torn tail: the file returns + // to its clean length, so a later SHORTER append can never + // leave residue past its end that a future recovery mis-parses + // as a ghost symbol. + Assert.assertEquals("torn tail physically dropped by open", cleanLen, Files.size(f)); Assert.assertEquals(2, re.size()); ObjList s = re.readLoadedSymbols(); Assert.assertEquals("one", s.getQuick(0)); Assert.assertEquals("two", s.getQuick(1)); - // The next append overwrites the torn tail, keeping the file consistent. + // The next append continues from the truncated tail cleanly. re.appendSymbol("three"); Assert.assertEquals(3, re.size()); } finally { From 754eb3d642ce991755bef93dc833b0be114fa7a0 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Thu, 9 Jul 2026 18:53:09 +0100 Subject: [PATCH 10/96] Guard the sent-dict mirror against int overflow The I/O thread's lifetime-monotonic symbol-dictionary mirror is sized with int math: accumulateSentDict passed sentDictBytesLen + regionBytes (an int sum) to ensureSentDictCapacity, and the grow step doubled capacity*2, also int. On a pathological, very-high-cardinality connection the sum overflows negative -- so the capacity check passes and copyMemory scribbles past the buffer (silent heap corruption) -- and capacity*2 overflows negative near 1 GB, degrading the doubling to exact-fit reallocs. Reaching this needs ~200M+ distinct symbols on one connection, far past any real workload, but the failure mode is silent corruption. ensureSentDictCapacity now takes a long, the caller passes a long sum, and the method throws a LineSenderException above an int-addressable ceiling (Integer.MAX_VALUE - 8) instead of overflowing, growing in long math clamped to that ceiling. Defensive only -- not reachable at realistic symbol cardinality, so there is no scale test. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../sf/cursor/CursorWebSocketSendLoop.java | 29 +++++++++++++++---- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java index 2e6b826c..3c7a0421 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java @@ -133,6 +133,12 @@ public final class CursorWebSocketSendLoop implements QuietCloseable { */ public static final long DEFAULT_POISON_MIN_ESCALATION_WINDOW_MILLIS = 5_000L; private static final Logger LOG = LoggerFactory.getLogger(CursorWebSocketSendLoop.class); + // Hard ceiling for the lifetime-monotonic sent-dictionary mirror. The mirror + // fields are int, so it cannot exceed Integer.MAX_VALUE bytes; reaching even + // this needs ~200M+ distinct symbols on a single connection, far past any real + // workload. The guard exists so that pathological growth fails loudly instead + // of overflowing the int capacity math into a heap-corrupting copyMemory. + private static final int MAX_SENT_DICT_BYTES = Integer.MAX_VALUE - 8; /** * Throttle "reconnect attempt N failed" WARN logs to one per 5 s. */ @@ -2035,19 +2041,32 @@ private void accumulateSentDict(long payloadAddr, int payloadLen) { } } int regionBytes = (int) (p - regionStart); - ensureSentDictCapacity(sentDictBytesLen + regionBytes); + // long sum: sentDictBytesLen + regionBytes can exceed Integer.MAX_VALUE on + // a very-high-cardinality lifetime connection; ensureSentDictCapacity then + // fails loudly rather than overflowing to a negative int (which would make + // the capacity check pass and copyMemory scribble past the buffer). + ensureSentDictCapacity((long) sentDictBytesLen + regionBytes); Unsafe.getUnsafe().copyMemory(regionStart, sentDictBytesAddr + sentDictBytesLen, regionBytes); sentDictBytesLen += regionBytes; sentDictCount += (int) deltaCount; } - private void ensureSentDictCapacity(int required) { + private void ensureSentDictCapacity(long required) { if (sentDictBytesCapacity >= required) { return; } - int newCap = Math.max(sentDictBytesCapacity * 2, Math.max(4096, required)); - sentDictBytesAddr = Unsafe.realloc(sentDictBytesAddr, sentDictBytesCapacity, newCap, MemoryTag.NATIVE_DEFAULT); - sentDictBytesCapacity = newCap; + if (required > MAX_SENT_DICT_BYTES) { + throw new LineSenderException("symbol dictionary mirror exceeds the maximum size [" + + "required=" + required + ", max=" + MAX_SENT_DICT_BYTES + ']'); + } + // Grow in long to avoid the capacity*2 int overflow (negative) that would + // otherwise degrade the doubling near 1 GB; clamp to the int ceiling. + long newCap = Math.max((long) sentDictBytesCapacity * 2, Math.max(4096L, required)); + if (newCap > MAX_SENT_DICT_BYTES) { + newCap = MAX_SENT_DICT_BYTES; + } + sentDictBytesAddr = Unsafe.realloc(sentDictBytesAddr, sentDictBytesCapacity, (int) newCap, MemoryTag.NATIVE_DEFAULT); + sentDictBytesCapacity = (int) newCap; } private long readVarintAt(long p, long limit) { From 4f44fb3b8d880e416044b41cf1ebde0e348503e7 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Thu, 9 Jul 2026 22:09:25 +0100 Subject: [PATCH 11/96] Parse the delta header once per frame on send trySendOne decoded a frame's delta header twice: the pre-send torn-dictionary guard called frameDeltaStart (magic/flags check + start-id varint), then post-send accumulateSentDict re-ran isDeltaFrame and re-read the start id before reading deltaCount. Both run on every delta frame on the I/O send path. Decode the start id once in the guard, hoist the frame address into a local, and pass the start id into accumulateSentDict, which now locates deltaCount just past the canonical start-id encoding (via NativeBufferWriter.varintSize) instead of re-parsing the header. The non-delta-frame case is carried by the same start id (-1), so the post- send mirror update runs exactly when it did before. Also move the accumulateSentDict javadoc onto accumulateSentDict: it had drifted above frameDeltaStart (which kept its own doc), leaving accumulateSentDict undocumented. The per-entry region walk (to size the mirror copy) remains; eliminating it needs a wire-level deltaBytes field, a server-side change out of scope for this client fix. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../sf/cursor/CursorWebSocketSendLoop.java | 51 +++++++++++-------- 1 file changed, 29 insertions(+), 22 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java index 3c7a0421..9f41e1e6 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java @@ -1978,17 +1978,6 @@ private void setWireBaselineWithCatchUp(long replayStart) { } } - /** - * Copies the symbol-dictionary delta a just-sent frame carries into the - * loop-local mirror ({@link #sentDictBytesAddr}) so a future reconnect can - * re-register it. Frames are sent in FSN order carrying monotonically - * extending deltas, so a frame whose delta starts exactly at - * {@link #sentDictCount} extends the mirror; a replayed or empty-delta frame - * (nothing new) is skipped. Only ever called in delta mode. - * - * @param payloadAddr address of the QWP message (12-byte header first) - * @param payloadLen message length in bytes - */ /** * Returns the symbol-dictionary delta start id of a frame, or -1 when the * frame carries no delta section. Used by the pre-send torn-dictionary guard. @@ -2013,14 +2002,27 @@ private static boolean isDeltaFrame(long payloadAddr, int payloadLen) { return (flags & QwpConstants.FLAG_DELTA_SYMBOL_DICT) != 0; } - private void accumulateSentDict(long payloadAddr, int payloadLen) { + /** + * Copies the symbol-dictionary delta a just-sent frame carries into the + * loop-local mirror ({@link #sentDictBytesAddr}) so a future reconnect can + * re-register it. Frames are sent in FSN order carrying monotonically + * extending deltas, so a frame whose delta starts exactly at + * {@link #sentDictCount} extends the mirror; a replayed or empty-delta frame + * (nothing new) is skipped. Only ever called in delta mode, for a frame the + * pre-send guard already classified as a delta frame. + * + * @param payloadAddr address of the QWP message (12-byte header first) + * @param payloadLen message length in bytes + * @param deltaStart the frame's delta start id, already decoded by the + * pre-send guard ({@link #frameDeltaStart}) -- passed in so + * the magic/flags and start-id varint are not re-parsed + */ + private void accumulateSentDict(long payloadAddr, int payloadLen, int deltaStart) { long limit = payloadAddr + payloadLen; - if (!isDeltaFrame(payloadAddr, payloadLen)) { - return; - } - long p = payloadAddr + QwpConstants.HEADER_SIZE; - long deltaStart = readVarintAt(p, limit); - p = varintEnd; + // deltaStart is known (the guard decoded it); locate deltaCount just past + // its canonical LEB128 encoding rather than re-reading the header and the + // start-id varint. + long p = payloadAddr + QwpConstants.HEADER_SIZE + NativeBufferWriter.varintSize(deltaStart); long deltaCount = readVarintAt(p, limit); p = varintEnd; // Only a delta that extends exactly from our current tip introduces new @@ -2291,6 +2293,11 @@ private boolean trySendOne() { if (frameEnd > pub) { return false; // payload not fully published yet } + long frameAddr = base + sendOffset + MmapSegment.FRAME_HEADER_SIZE; + // -1 unless this is a delta frame; the guard decodes it once here and + // accumulateSentDict reuses it post-send, so the delta header is parsed + // once per frame rather than twice. + int deltaStart = -1; if (deltaDictEnabled) { // Torn-dictionary guard. In normal operation a delta frame's start id // never exceeds the dictionary coverage established so far (replayed @@ -2302,7 +2309,7 @@ private boolean trySendOne() { // Sending the frame would corrupt the table (the server would null-pad // the missing ids), so fail terminally instead; the unreplayable data // must be resent. - int deltaStart = frameDeltaStart(base + sendOffset + MmapSegment.FRAME_HEADER_SIZE, payloadLen); + deltaStart = frameDeltaStart(frameAddr, payloadLen); if (deltaStart > sentDictCount) { recordFatal(new LineSenderException( "recovered store-and-forward symbol dictionary is incomplete (likely a host crash): " @@ -2312,16 +2319,16 @@ private boolean trySendOne() { } } try { - client.sendBinary(base + sendOffset + MmapSegment.FRAME_HEADER_SIZE, payloadLen); + client.sendBinary(frameAddr, payloadLen); } catch (Throwable t) { fail(t); return false; } - if (deltaDictEnabled) { + if (deltaStart >= 0) { // Mirror the symbols this frame introduced so a later reconnect can // rebuild the whole dictionary. Idempotent on replay: a frame whose // delta we already hold advances nothing. - accumulateSentDict(base + sendOffset + MmapSegment.FRAME_HEADER_SIZE, payloadLen); + accumulateSentDict(frameAddr, payloadLen, deltaStart); } lastFrameOrPingNanos = System.nanoTime(); sendOffset = frameEnd; From 0f9d88f1c31befe03c4702cd08965585f6b96724 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Thu, 9 Jul 2026 22:09:35 +0100 Subject: [PATCH 12/96] Fix stale self-sufficient-frame comments Several comments predated file-mode delta encoding and claimed every cursor frame is self-sufficient with a "symbol-dict delta from id 0". That is now only the fallback: in delta mode (memory mode, and file mode when the persisted dictionary opened) frames carry monotonic deltas that are NOT self-sufficient, and the fresh server's dictionary is re-established by an I/O-thread catch-up frame before replay. The worst offender was the deltaDictEnabled field doc ("Enabled only in memory-mode ... File-mode keeps full self-sufficient frames"), which directly contradicted the feature. Corrected it plus the two ensureConnected call-site comments, the append-failed-path comment, and the wasRecoveredFromDisk field doc (schema stays self-sufficient per frame; the dictionary does not). No behavior change. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../qwp/client/QwpWebSocketSender.java | 31 ++++++++++--------- .../client/sf/cursor/CursorSendEngine.java | 6 ++-- 2 files changed, 20 insertions(+), 17 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index 54be9ea0..77e85392 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -224,11 +224,14 @@ public class QwpWebSocketSender implements Sender { private boolean deferCommit; // True when the sender emits incremental (delta) symbol dictionaries: each // message carries only symbol ids not yet sent on the wire, rather than the - // full dictionary from id 0. Enabled only in memory-mode, where a reconnect - // replays from the in-process ring and the I/O thread re-registers the whole - // dictionary via a catch-up frame before replaying. File-mode store-and-forward - // keeps full self-sufficient frames (recovery/orphan-drain can replay mid-stream - // to a fresh server, where a partial delta would leave gaps). Set in setCursorEngine. + // full dictionary from id 0. Enabled in memory-mode (a reconnect replays from + // the in-process ring) and in file-mode store-and-forward when the per-slot + // persisted dictionary opened. In both, the I/O thread re-registers the whole + // dictionary via a catch-up frame before replaying, so a non-self-sufficient + // delta frame never dangles an id on a fresh server. Falls back to full + // self-sufficient frames only when the persisted dictionary is unavailable in + // file-mode (recovery/orphan-drain would then have nothing to rebuild the + // deltas from). Set in setCursorEngine. private boolean deltaDictEnabled; // User-supplied observer for background orphan-slot drainer events. // Volatile: written by setDrainerListener (any thread, before or after @@ -3386,18 +3389,18 @@ private void ensureConnected() { host, port, client.getServerQwpVersion(), serverMaxBatchSize, effectiveAutoFlushBytes); } else { // Async mode: I/O thread will drive the connect. Encoder uses - // its default version (V1). The symbol-dict watermark still gets - // reset for consistency with the sync path; the post-connect replay - // path does not need a producer-side reset signal because every - // cursor frame is self-sufficient. + // its default version (V1). The per-batch symbol-dict watermark still + // gets reset for consistency with the sync path; the post-connect + // replay path needs no producer-side reset signal (see below). Endpoint ep = endpoints.get(0); LOG.info("Async initial connect deferred to I/O thread [firstHost={}, firstPort={}, endpointCount={}]", ep.host, ep.port, endpoints.size()); } - // Server starts fresh on each connection, so reset the symbol-dict - // watermark. Cursor frames are self-sufficient (every frame carries its - // full inline schema + a symbol-dict delta from id 0), so post-reconnect - // replay needs no producer-side reset signal. + // Server starts fresh on each connection, so reset the per-batch + // symbol-dict watermark. Every frame still carries its full inline schema, + // and the fresh server's dictionary is re-established either by a full-dict + // frame (full-dict mode) or by an I/O-thread catch-up frame before replay + // (delta mode), so post-reconnect replay needs no producer-side reset signal. resetSymbolDictStateForNewConnection(); connectionError.set(null); @@ -3784,7 +3787,7 @@ private void sealAndSwapBuffer() { // back to it; flushPendingRows aborts its post-enqueue state // updates after this throw, so the source rows stay intact and the // next batch re-emits the same rows along with the full inline - // schema and symbol-dict delta from id 0. + // schema and the symbol-dict delta the batch requires. if (toSend.isSending()) { toSend.markRecycled(); } else if (toSend.isSealed()) { diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java index 85015a73..3bc0a185 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java @@ -84,9 +84,9 @@ public final class CursorSendEngine implements QuietCloseable { private final SlotLock slotLock; // True when the constructor recovered an existing on-disk slot rather // than starting fresh. Diagnostic accessor for tests and observability; - // cursor frames are self-sufficient (every frame carries full schema + - // full symbol-dict delta), so producer-side schema reset on recovery - // is not required. + // every frame carries its full inline schema, so producer-side schema reset + // on recovery is not required (the symbol dictionary, which delta frames do + // NOT carry in full, is re-registered by an I/O-thread catch-up instead). private final boolean wasRecoveredFromDisk; // FSN of the last commit-bearing (non-FLAG_DEFER_COMMIT) frame found in a // ring recovered from disk, or -1 for fresh/memory rings and recovered From d89f14f6186e4ca8d96ce1cf250836af3fe7406d Mon Sep 17 00:00:00 2001 From: glasstiger Date: Thu, 9 Jul 2026 22:35:19 +0100 Subject: [PATCH 13/96] Make the delta-dict tests deterministic and leak-checked Two robustness fixes to the delta symbol-dictionary tests. Deterministic synchronization (replaces fixed sleeps): - DeltaDictCatchUpTest waited a fixed 200 ms for the server to close connection 1 before sending batch 2. On a loaded machine that could under-wait and let batch 2 race into connection 1's pre-close window, changing which connection the catch-up lands on. The handler now sets a conn1Closed flag after it closes the socket, and the test waits on that. - DeltaDictRecoveryTest's torn-dictionary test slept a fixed 1 s to let the replay guard fire before close(). It now polls flush() for the latched terminal (close() remains the fallback), so it captures the terminal as soon as it fires -- the run dropped from ~1 s to ~0.3 s. Leak checks: the Sender-based tests allocate native memory (the send-loop mirror, persisted-dict buffers, segment mmaps) but were not wrapped in assertMemoryLeak, unlike the rest of the suite. Wrap all eight methods across the three classes; every one is balanced (they already cleaned up via try-with-resources -- the wrapper now guards against future leaks). Co-Authored-By: Claude Opus 4.8 (1M context) --- .../qwp/client/DeltaDictCatchUpTest.java | 227 +++++++++-------- .../qwp/client/DeltaDictRecoveryTest.java | 238 ++++++++++-------- .../qwp/client/SelfSufficientFramesTest.java | 220 ++++++++-------- 3 files changed, 367 insertions(+), 318 deletions(-) diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictCatchUpTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictCatchUpTest.java index 1af55bc3..0d2ff9f7 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictCatchUpTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictCatchUpTest.java @@ -42,6 +42,8 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; + /** * Verifies the delta symbol-dictionary catch-up on reconnect (memory-mode). *

@@ -61,37 +63,40 @@ public void testReconnectCatchUpRebuildsDictionary() throws Exception { // sender reconnects. Connection 2 (fresh, empty dict): send "beta" (id 1). // Without catch-up, connection 2's first data frame would carry // deltaStart=1 and the fresh server would never learn id 0. - CatchUpHandler handler = new CatchUpHandler(); - try (TestWebSocketServer server = new TestWebSocketServer(handler)) { - int port = server.getPort(); - server.start(); - Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); - - try (Sender sender = Sender.fromConfig("ws::addr=localhost:" + port + ";")) { - sender.table("t").symbol("s", "alpha").longColumn("v", 1L).atNow(); - sender.flush(); - waitFor(() -> handler.dictFor(1).size() >= 1, 5_000); - - // Let the I/O loop observe the server-side close before the next - // batch, so batch 2 is what drives the reconnect + catch-up. - Thread.sleep(200); - - sender.table("t").symbol("s", "beta").longColumn("v", 2L).atNow(); - sender.flush(); - waitFor(() -> handler.connectionsAccepted.get() >= 2 - && handler.dictFor(2).size() >= 2, 5_000); - } + assertMemoryLeak(() -> { + CatchUpHandler handler = new CatchUpHandler(); + try (TestWebSocketServer server = new TestWebSocketServer(handler)) { + int port = server.getPort(); + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + + try (Sender sender = Sender.fromConfig("ws::addr=localhost:" + port + ";")) { + sender.table("t").symbol("s", "alpha").longColumn("v", 1L).atNow(); + sender.flush(); + waitFor(() -> handler.dictFor(1).size() >= 1, 5_000); - // The fresh (2nd) connection's dictionary, rebuilt purely from the - // frames it received, must hold both symbols contiguously with no - // null gap -- exactly what the catch-up frame guarantees. - List conn2 = handler.dictFor(2); - Assert.assertTrue("2nd connection saw a catch-up frame with 0 tables", - handler.sawZeroTableFrameOnConn2); - Assert.assertEquals("2nd connection dictionary size", 2, conn2.size()); - Assert.assertEquals("alpha", conn2.get(0)); - Assert.assertEquals("beta", conn2.get(1)); - } + // Wait until the server has actually closed connection 1 before + // sending batch 2, so batch 2 cannot race into connection 1 and + // must drive the reconnect + catch-up. + waitFor(() -> handler.conn1Closed, 5_000); + + sender.table("t").symbol("s", "beta").longColumn("v", 2L).atNow(); + sender.flush(); + waitFor(() -> handler.connectionsAccepted.get() >= 2 + && handler.dictFor(2).size() >= 2, 5_000); + } + + // The fresh (2nd) connection's dictionary, rebuilt purely from the + // frames it received, must hold both symbols contiguously with no + // null gap -- exactly what the catch-up frame guarantees. + List conn2 = handler.dictFor(2); + Assert.assertTrue("2nd connection saw a catch-up frame with 0 tables", + handler.sawZeroTableFrameOnConn2); + Assert.assertEquals("2nd connection dictionary size", 2, conn2.size()); + Assert.assertEquals("alpha", conn2.get(0)); + Assert.assertEquals("beta", conn2.get(1)); + } + }); } @Test @@ -109,48 +114,50 @@ public void testCatchUpEntryTooLargeForCapFailsTerminally() throws Exception { // reconnect's catch-up cannot re-ship the symbol. (One fixed cap can't do // this: the client refuses to SEND a single-table frame over the cap, and // that data frame is always larger than the bare catch-up entry.) - CapShrinkHandler handler = new CapShrinkHandler(); - try (TestWebSocketServer server = new TestWebSocketServer(handler)) { - handler.setServer(server); - int port = server.getPort(); - server.start(); - Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); - - String bigSymbol = TestUtils.repeat("x", 200); // ~202-byte dict entry - LineSenderException terminal = null; - Sender sender = Sender.fromConfig("ws::addr=localhost:" + port + ";"); - try { - sender.table("t").symbol("s", bigSymbol).longColumn("v", 1L).atNow(); - sender.flush(); - // The terminal latches on the I/O thread once the reconnect's - // catch-up hits the oversized entry; it surfaces to the producer - // on a subsequent flush. Poll a bounded time for it. The polling - // rows use a small symbol that fits the shrunk cap, so the - // producer-side cap check never fires and flush() surfaces the - // I/O thread's catch-up terminal via checkError. - long deadline = System.currentTimeMillis() + 10_000; - while (System.currentTimeMillis() < deadline && terminal == null) { + assertMemoryLeak(() -> { + CapShrinkHandler handler = new CapShrinkHandler(); + try (TestWebSocketServer server = new TestWebSocketServer(handler)) { + handler.setServer(server); + int port = server.getPort(); + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + + String bigSymbol = TestUtils.repeat("x", 200); // ~202-byte dict entry + LineSenderException terminal = null; + Sender sender = Sender.fromConfig("ws::addr=localhost:" + port + ";"); + try { + sender.table("t").symbol("s", bigSymbol).longColumn("v", 1L).atNow(); + sender.flush(); + // The terminal latches on the I/O thread once the reconnect's + // catch-up hits the oversized entry; it surfaces to the producer + // on a subsequent flush. Poll a bounded time for it. The polling + // rows use a small symbol that fits the shrunk cap, so the + // producer-side cap check never fires and flush() surfaces the + // I/O thread's catch-up terminal via checkError. + long deadline = System.currentTimeMillis() + 10_000; + while (System.currentTimeMillis() < deadline && terminal == null) { + try { + sender.table("t").symbol("s", "y").longColumn("v", 2L).atNow(); + sender.flush(); + Thread.sleep(20); + } catch (LineSenderException e) { + terminal = e; + } + } + } finally { try { - sender.table("t").symbol("s", "y").longColumn("v", 2L).atNow(); - sender.flush(); - Thread.sleep(20); + sender.close(); } catch (LineSenderException e) { - terminal = e; - } - } - } finally { - try { - sender.close(); - } catch (LineSenderException e) { - if (terminal == null) { - terminal = e; + if (terminal == null) { + terminal = e; + } } } + Assert.assertNotNull("an oversized catch-up entry must surface a terminal", terminal); + Assert.assertTrue("terminal must come from the catch-up path, got: " + terminal.getMessage(), + terminal.getMessage().contains("during catch-up")); } - Assert.assertNotNull("an oversized catch-up entry must surface a terminal", terminal); - Assert.assertTrue("terminal must come from the catch-up path, got: " + terminal.getMessage(), - terminal.getMessage().contains("during catch-up")); - } + }); } @Test @@ -163,48 +170,50 @@ public void testReconnectCatchUpSplitsLargeDictionaryAcrossFrames() throws Excep // into several contiguous zero-table frames that the fresh server stitches // back into a complete, gap-free dictionary. final int symbolCount = 40; - SplitCatchUpHandler handler = new SplitCatchUpHandler(symbolCount); - try (TestWebSocketServer server = new TestWebSocketServer(handler)) { - server.setAdvertisedMaxBatchSize(160); // small cap forces the catch-up to split - int port = server.getPort(); - server.start(); - Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); - - try (Sender sender = Sender.fromConfig("ws::addr=localhost:" + port + ";")) { - // One row per flush so each frame stays under the 160-byte cap; the - // sent dictionary still accumulates all 40 symbols on connection 1. - for (int i = 0; i < symbolCount; i++) { - sender.table("t").symbol("s", symbolName(i)).longColumn("v", i).atNow(); + assertMemoryLeak(() -> { + SplitCatchUpHandler handler = new SplitCatchUpHandler(symbolCount); + try (TestWebSocketServer server = new TestWebSocketServer(handler)) { + server.setAdvertisedMaxBatchSize(160); // small cap forces the catch-up to split + int port = server.getPort(); + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + + try (Sender sender = Sender.fromConfig("ws::addr=localhost:" + port + ";")) { + // One row per flush so each frame stays under the 160-byte cap; the + // sent dictionary still accumulates all 40 symbols on connection 1. + for (int i = 0; i < symbolCount; i++) { + sender.table("t").symbol("s", symbolName(i)).longColumn("v", i).atNow(); + sender.flush(); + } + // Wait until connection 1 has learned every symbol, so the sender's + // sent-dictionary mirror (the catch-up source) holds all of them. + waitFor(() -> handler.dictFor(1).size() >= symbolCount, 10_000); + + // Wait until the server has actually closed connection 1 before + // sending batch 2, so batch 2 drives the reconnect + split catch-up. + waitFor(() -> handler.conn1Closed, 5_000); + + sender.table("t").symbol("s", symbolName(symbolCount)).longColumn("v", symbolCount).atNow(); sender.flush(); + waitFor(() -> handler.connectionsAccepted.get() >= 2 + && handler.dictFor(2).size() >= symbolCount + 1, 10_000); } - // Wait until connection 1 has learned every symbol, so the sender's - // sent-dictionary mirror (the catch-up source) holds all of them. - waitFor(() -> handler.dictFor(1).size() >= symbolCount, 10_000); - - // Let the I/O loop observe the server-side close before the next - // batch, so batch 2 drives the reconnect + split catch-up. - Thread.sleep(200); - - sender.table("t").symbol("s", symbolName(symbolCount)).longColumn("v", symbolCount).atNow(); - sender.flush(); - waitFor(() -> handler.connectionsAccepted.get() >= 2 - && handler.dictFor(2).size() >= symbolCount + 1, 10_000); - } - // Connection 2's dictionary, rebuilt purely from the frames it received, - // must hold every symbol contiguously with no null gap -- the split - // catch-up frames reassemble exactly. - List conn2 = handler.dictFor(2); - Assert.assertEquals("2nd connection dictionary size", symbolCount + 1, conn2.size()); - for (int i = 0; i <= symbolCount; i++) { - Assert.assertEquals("symbol at id " + i, symbolName(i), conn2.get(i)); + // Connection 2's dictionary, rebuilt purely from the frames it received, + // must hold every symbol contiguously with no null gap -- the split + // catch-up frames reassemble exactly. + List conn2 = handler.dictFor(2); + Assert.assertEquals("2nd connection dictionary size", symbolCount + 1, conn2.size()); + for (int i = 0; i <= symbolCount; i++) { + Assert.assertEquals("symbol at id " + i, symbolName(i), conn2.get(i)); + } + // The catch-up had to span more than one zero-table frame to stay under + // the advertised cap -- that split is the behaviour under test. + Assert.assertTrue("catch-up split into multiple frames (saw " + + handler.zeroTableFramesOnConn2 + ")", + handler.zeroTableFramesOnConn2 >= 2); } - // The catch-up had to span more than one zero-table frame to stay under - // the advertised cap -- that split is the behaviour under test. - Assert.assertTrue("catch-up split into multiple frames (saw " - + handler.zeroTableFramesOnConn2 + ")", - handler.zeroTableFramesOnConn2 >= 2); - } + }); } private static String symbolName(int i) { @@ -252,6 +261,10 @@ private interface BoolCondition { */ private static class CatchUpHandler implements TestWebSocketServer.WebSocketServerHandler { final AtomicInteger connectionsAccepted = new AtomicInteger(); + // Set once the server has closed connection 1. A test waits on this + // (rather than a fixed sleep) before sending batch 2, so batch 2 cannot + // race into connection 1's pre-close window and must land on the reconnect. + volatile boolean conn1Closed; volatile boolean sawZeroTableFrameOnConn2; private final List> dictsByConn = new CopyOnWriteArrayList<>(); private TestWebSocketServer.ClientHandler currentClient; @@ -284,6 +297,7 @@ public synchronized void onBinaryMessage(TestWebSocketServer.ClientHandler clien if (connNumber == 1) { Thread.sleep(50); client.close(); + conn1Closed = true; } } catch (IOException | InterruptedException e) { Thread.currentThread().interrupt(); @@ -377,6 +391,8 @@ public synchronized void onBinaryMessage(TestWebSocketServer.ClientHandler clien */ private static class SplitCatchUpHandler implements TestWebSocketServer.WebSocketServerHandler { final AtomicInteger connectionsAccepted = new AtomicInteger(); + // Set once the server has closed connection 1 (see CatchUpHandler.conn1Closed). + volatile boolean conn1Closed; volatile int zeroTableFramesOnConn2; private final List> dictsByConn = new CopyOnWriteArrayList<>(); private final int dropConn1AtDictSize; @@ -417,6 +433,7 @@ public synchronized void onBinaryMessage(TestWebSocketServer.ClientHandler clien conn1Dropped = true; Thread.sleep(50); client.close(); + conn1Closed = true; } } catch (IOException | InterruptedException e) { Thread.currentThread().interrupt(); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java index 46c2a66b..8d1bd24a 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java @@ -46,6 +46,8 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; + /** * End-to-end recovery for the delta symbol dictionary in store-and-forward mode. *

@@ -79,131 +81,149 @@ public void tearDown() { @Test public void testRecoveredSlotReplaysDeltaFramesAgainstFreshServer() throws Exception { - // Phase 1: silent server (no acks). Sender 1 writes symbol rows and - // close-fast (no drain), leaving unacked delta frames + a persisted - // dictionary in the slot. - try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) { - int port = silent.getPort(); - silent.start(); - Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS)); - - String pad = TestUtils.repeat("x", 64); - String cfg = "ws::addr=localhost:" + port - + ";sf_dir=" + sfDir - + ";sf_max_bytes=4096" - + ";close_flush_timeout_millis=0;"; - try (Sender s1 = Sender.fromConfig(cfg)) { - for (int i = 0; i < ROWS; i++) { - s1.table("m") - .symbol("s", "sym-" + (i % DISTINCT_SYMBOLS)) - .stringColumn("p", pad) - .longColumn("v", i) - .atNow(); - s1.flush(); + assertMemoryLeak(() -> { + // Phase 1: silent server (no acks). Sender 1 writes symbol rows and + // close-fast (no drain), leaving unacked delta frames + a persisted + // dictionary in the slot. + try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) { + int port = silent.getPort(); + silent.start(); + Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS)); + + String pad = TestUtils.repeat("x", 64); + String cfg = "ws::addr=localhost:" + port + + ";sf_dir=" + sfDir + + ";sf_max_bytes=4096" + + ";close_flush_timeout_millis=0;"; + try (Sender s1 = Sender.fromConfig(cfg)) { + for (int i = 0; i < ROWS; i++) { + s1.table("m") + .symbol("s", "sym-" + (i % DISTINCT_SYMBOLS)) + .stringColumn("p", pad) + .longColumn("v", i) + .atNow(); + s1.flush(); + } } } - } - // Ack a prefix so recovery does NOT replay from the self-sufficient head. - // Rows 0..DISTINCT_SYMBOLS-1 register all the symbols, so stamping the - // watermark at FSN DISTINCT_SYMBOLS-1 makes recovery replay from FSN - // DISTINCT_SYMBOLS onward -- frames whose delta starts at - // DISTINCT_SYMBOLS and carries NO new symbols (rows past the first cycle - // reuse existing ids). The early ids those frames reference then exist - // ONLY in the persisted dictionary, so the reconstructed dictionary below - // is complete solely because the catch-up frame re-registered them. That - // pins the content assertions to the catch-up: without it (or with a - // broken one) the fresh server would null-pad ids 0..DISTINCT_SYMBOLS-1 - // and the per-id checks would fail. - java.nio.file.Path slot = Paths.get(sfDir, "default"); - writeAckWatermark(slot.resolve(".ack-watermark"), DISTINCT_SYMBOLS - 1); - - // Phase 2: fresh server that reconstructs its per-connection dictionary - // from the delta sections. Sender 2 recovers the slot and replays. - DictReconstructingHandler handler = new DictReconstructingHandler(); - try (TestWebSocketServer good = new TestWebSocketServer(handler)) { - int port = good.getPort(); - good.start(); - Assert.assertTrue(good.awaitStart(5, TimeUnit.SECONDS)); - - String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";"; - try (Sender ignored = Sender.fromConfig(cfg)) { - long deadline = System.currentTimeMillis() + 5_000; - while (System.currentTimeMillis() < deadline - && handler.maxDictSize() < DISTINCT_SYMBOLS) { - Thread.sleep(20); + // Ack a prefix so recovery does NOT replay from the self-sufficient head. + // Rows 0..DISTINCT_SYMBOLS-1 register all the symbols, so stamping the + // watermark at FSN DISTINCT_SYMBOLS-1 makes recovery replay from FSN + // DISTINCT_SYMBOLS onward -- frames whose delta starts at + // DISTINCT_SYMBOLS and carries NO new symbols (rows past the first cycle + // reuse existing ids). The early ids those frames reference then exist + // ONLY in the persisted dictionary, so the reconstructed dictionary below + // is complete solely because the catch-up frame re-registered them. That + // pins the content assertions to the catch-up: without it (or with a + // broken one) the fresh server would null-pad ids 0..DISTINCT_SYMBOLS-1 + // and the per-id checks would fail. + java.nio.file.Path slot = Paths.get(sfDir, "default"); + writeAckWatermark(slot.resolve(".ack-watermark"), DISTINCT_SYMBOLS - 1); + + // Phase 2: fresh server that reconstructs its per-connection dictionary + // from the delta sections. Sender 2 recovers the slot and replays. + DictReconstructingHandler handler = new DictReconstructingHandler(); + try (TestWebSocketServer good = new TestWebSocketServer(handler)) { + int port = good.getPort(); + good.start(); + Assert.assertTrue(good.awaitStart(5, TimeUnit.SECONDS)); + + String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";"; + try (Sender ignored = Sender.fromConfig(cfg)) { + long deadline = System.currentTimeMillis() + 5_000; + while (System.currentTimeMillis() < deadline + && handler.maxDictSize() < DISTINCT_SYMBOLS) { + Thread.sleep(20); + } } - } - // The recovering sender must have re-registered the dictionary via a - // catch-up (0-table) frame before replaying delta frames. - Assert.assertTrue("recovery sent a full-dictionary catch-up frame", - handler.sawCatchUpFrame); - // The reconstructed dictionary must be complete and gap-free: exactly - // the DISTINCT_SYMBOLS symbols, no null padding left by a missing id. - List dict = handler.dictSnapshot(); - Assert.assertEquals("reconstructed dictionary size", DISTINCT_SYMBOLS, dict.size()); - for (int i = 0; i < DISTINCT_SYMBOLS; i++) { - Assert.assertEquals("dictionary id " + i, "sym-" + i, dict.get(i)); + // The recovering sender must have re-registered the dictionary via a + // catch-up (0-table) frame before replaying delta frames. + Assert.assertTrue("recovery sent a full-dictionary catch-up frame", + handler.sawCatchUpFrame); + // The reconstructed dictionary must be complete and gap-free: exactly + // the DISTINCT_SYMBOLS symbols, no null padding left by a missing id. + List dict = handler.dictSnapshot(); + Assert.assertEquals("reconstructed dictionary size", DISTINCT_SYMBOLS, dict.size()); + for (int i = 0; i < DISTINCT_SYMBOLS; i++) { + Assert.assertEquals("dictionary id " + i, "sym-" + i, dict.get(i)); + } } - } + }); } @Test public void testTornDictionaryFailsCleanlyInsteadOfCorrupting() throws Exception { - // Phase 1: each row introduces a new symbol, so frame i carries deltaStart=i. - // Silent server + close-fast leaves all frames unacked in the slot. - try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) { - int port = silent.getPort(); - silent.start(); - Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS)); - String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir - + ";close_flush_timeout_millis=0;"; - try (Sender s1 = Sender.fromConfig(cfg)) { - for (int i = 0; i < 6; i++) { - s1.table("m").symbol("s", "sym-" + i).longColumn("v", i).atNow(); - s1.flush(); + assertMemoryLeak(() -> { + // Phase 1: each row introduces a new symbol, so frame i carries deltaStart=i. + // Silent server + close-fast leaves all frames unacked in the slot. + try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) { + int port = silent.getPort(); + silent.start(); + Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS)); + String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + + ";close_flush_timeout_millis=0;"; + try (Sender s1 = Sender.fromConfig(cfg)) { + for (int i = 0; i < 6; i++) { + s1.table("m").symbol("s", "sym-" + i).longColumn("v", i).atNow(); + s1.flush(); + } } } - } - // Simulate a host/power crash: the segment frames survive but the persisted - // dictionary is lost, and the ack watermark was left mid-stream. Truncate - // .symbol-dict to its 8-byte header (0 symbols) and stamp the watermark at - // FSN 2, so recovery replays from FSN 3 -- a frame with deltaStart=3. - java.nio.file.Path slot = Paths.get(sfDir, "default"); - java.nio.file.Path dict = slot.resolve(".symbol-dict"); - byte[] header = Arrays.copyOf(java.nio.file.Files.readAllBytes(dict), 8); - java.nio.file.Files.write(dict, header); - writeAckWatermark(slot.resolve(".ack-watermark"), 2); - - // Phase 2: recover against a fresh counting server. The replay guard must - // fire (frame deltaStart 3 > recovered dictionary size 0) and fail terminally - // rather than send a gapped frame that would corrupt the table. - CountingHandler handler = new CountingHandler(); - try (TestWebSocketServer good = new TestWebSocketServer(handler)) { - int port = good.getPort(); - good.start(); - Assert.assertTrue(good.awaitStart(5, TimeUnit.SECONDS)); - - String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";"; - LineSenderException terminal = null; - Sender s2 = Sender.fromConfig(cfg); - try { - Thread.sleep(1_000); // let the I/O loop attempt replay and hit the guard - } finally { + // Simulate a host/power crash: the segment frames survive but the persisted + // dictionary is lost, and the ack watermark was left mid-stream. Truncate + // .symbol-dict to its 8-byte header (0 symbols) and stamp the watermark at + // FSN 2, so recovery replays from FSN 3 -- a frame with deltaStart=3. + java.nio.file.Path slot = Paths.get(sfDir, "default"); + java.nio.file.Path dict = slot.resolve(".symbol-dict"); + byte[] header = Arrays.copyOf(java.nio.file.Files.readAllBytes(dict), 8); + java.nio.file.Files.write(dict, header); + writeAckWatermark(slot.resolve(".ack-watermark"), 2); + + // Phase 2: recover against a fresh counting server. The replay guard must + // fire (frame deltaStart 3 > recovered dictionary size 0) and fail terminally + // rather than send a gapped frame that would corrupt the table. + CountingHandler handler = new CountingHandler(); + try (TestWebSocketServer good = new TestWebSocketServer(handler)) { + int port = good.getPort(); + good.start(); + Assert.assertTrue(good.awaitStart(5, TimeUnit.SECONDS)); + + String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";"; + LineSenderException terminal = null; + Sender s2 = Sender.fromConfig(cfg); try { - s2.close(); - } catch (LineSenderException e) { - terminal = e; + // Poll for the replay guard to fire (it recordFatal's on the I/O + // thread); flush() surfaces the latched terminal to the producer. + // A bounded poll replaces a fixed sleep and captures it as soon as + // it fires; close() below is the fallback if it surfaces only there. + long deadline = System.currentTimeMillis() + 10_000; + while (System.currentTimeMillis() < deadline && terminal == null) { + try { + s2.flush(); + Thread.sleep(20); + } catch (LineSenderException e) { + terminal = e; + } + } + } finally { + try { + s2.close(); + } catch (LineSenderException e) { + if (terminal == null) { + terminal = e; + } + } } + Assert.assertEquals("no frame may be replayed to a fresh server with a torn dictionary", + 0, handler.frames.get()); + Assert.assertNotNull("a torn dictionary must surface a terminal error", terminal); + Assert.assertTrue(terminal.getMessage(), + terminal.getMessage().contains("symbol dictionary is incomplete")); } - Assert.assertEquals("no frame may be replayed to a fresh server with a torn dictionary", - 0, handler.frames.get()); - Assert.assertNotNull("a torn dictionary must surface a terminal error", terminal); - Assert.assertTrue(terminal.getMessage(), - terminal.getMessage().contains("symbol dictionary is incomplete")); - } + }); } private static void writeAckWatermark(java.nio.file.Path path, long fsn) throws IOException { diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java index 10bd1e6b..af30da83 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java @@ -38,6 +38,8 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; + /** * Pins down how the symbol dictionary is framed on the wire. *

@@ -64,44 +66,48 @@ public void testFileModeShipsMonotonicDeltaAndPersistsDict() throws Exception { // "beta" (deltaStart=1). The dictionary is durably kept in .symbol-dict // so a recovered/orphan-drained slot can rebuild it. Path sfDir = Files.createTempDirectory("qwp-sf-selfsufficient"); - CapturingHandler handler = new CapturingHandler(); - try (TestWebSocketServer server = new TestWebSocketServer(handler)) { - int port = server.getPort(); - server.start(); - Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); - - String config = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";"; - // The engine places slot files under sf_dir/ (default "default"). - Path dictFile = sfDir.resolve("default").resolve(".symbol-dict"); - try (Sender sender = Sender.fromConfig(config)) { - sender.table("foo").symbol("s", "alpha").longColumn("v", 1L).atNow(); - sender.flush(); - waitFor(() -> handler.batches.size() >= 1, 5_000); - - sender.table("foo").symbol("s", "beta").longColumn("v", 2L).atNow(); - sender.flush(); - waitFor(() -> handler.batches.size() >= 2, 5_000); - - // Check the persisted dictionary while the sender is live: a - // fully-drained close intentionally unlinks it (slot cleanup). - Assert.assertTrue("persisted dictionary file exists", Files.exists(dictFile)); - byte[] dict = Files.readAllBytes(dictFile); - Assert.assertTrue("dictionary retains alpha", containsUtf8(dict, "alpha")); - Assert.assertTrue("dictionary retains beta", containsUtf8(dict, "beta")); - } + try { + assertMemoryLeak(() -> { + CapturingHandler handler = new CapturingHandler(); + try (TestWebSocketServer server = new TestWebSocketServer(handler)) { + int port = server.getPort(); + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + + String config = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";"; + // The engine places slot files under sf_dir/ (default "default"). + Path dictFile = sfDir.resolve("default").resolve(".symbol-dict"); + try (Sender sender = Sender.fromConfig(config)) { + sender.table("foo").symbol("s", "alpha").longColumn("v", 1L).atNow(); + sender.flush(); + waitFor(() -> handler.batches.size() >= 1, 5_000); + + sender.table("foo").symbol("s", "beta").longColumn("v", 2L).atNow(); + sender.flush(); + waitFor(() -> handler.batches.size() >= 2, 5_000); + + // Check the persisted dictionary while the sender is live: a + // fully-drained close intentionally unlinks it (slot cleanup). + Assert.assertTrue("persisted dictionary file exists", Files.exists(dictFile)); + byte[] dict = Files.readAllBytes(dictFile); + Assert.assertTrue("dictionary retains alpha", containsUtf8(dict, "alpha")); + Assert.assertTrue("dictionary retains beta", containsUtf8(dict, "beta")); + } - Assert.assertEquals("expected 2 captured batches", 2, handler.batches.size()); - byte[] b1 = handler.batches.get(0); - byte[] b2 = handler.batches.get(1); - - Assert.assertEquals("batch 1 deltaStart must be 0", - 0, readVarint(b1, DELTA_START_OFFSET)); - Assert.assertEquals("batch 1 deltaCount must be 1", 1, readVarint(b1, DELTA_START_OFFSET + 1)); - // batch 2 ships ONLY beta as a delta from id 1. - Assert.assertEquals("batch 2 deltaStart must be 1 (monotonic)", - 1, readVarint(b2, DELTA_START_OFFSET)); - Assert.assertEquals("batch 2 deltaCount must be 1 (only the new symbol)", - 1, readVarint(b2, DELTA_START_OFFSET + 1)); + Assert.assertEquals("expected 2 captured batches", 2, handler.batches.size()); + byte[] b1 = handler.batches.get(0); + byte[] b2 = handler.batches.get(1); + + Assert.assertEquals("batch 1 deltaStart must be 0", + 0, readVarint(b1, DELTA_START_OFFSET)); + Assert.assertEquals("batch 1 deltaCount must be 1", 1, readVarint(b1, DELTA_START_OFFSET + 1)); + // batch 2 ships ONLY beta as a delta from id 1. + Assert.assertEquals("batch 2 deltaStart must be 1 (monotonic)", + 1, readVarint(b2, DELTA_START_OFFSET)); + Assert.assertEquals("batch 2 deltaCount must be 1 (only the new symbol)", + 1, readVarint(b2, DELTA_START_OFFSET + 1)); + } + }); } finally { rmDir(sfDir); } @@ -121,43 +127,47 @@ public void testDiskModeFallsBackToFullDictWhenPersistedDictUnopenable() throws Path dictPath = sfDir.resolve("default").resolve(".symbol-dict"); Files.createDirectories(dictPath); // a directory, not a file Files.createFile(dictPath.resolve("blocker")); // non-empty: cannot be unlinked/rmdir'd - CapturingHandler handler = new CapturingHandler(); - try (TestWebSocketServer server = new TestWebSocketServer(handler)) { - int port = server.getPort(); - server.start(); - Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); - - String config = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";"; - try (Sender sender = Sender.fromConfig(config)) { - sender.table("foo").symbol("s", "alpha").longColumn("v", 1L).atNow(); - sender.flush(); - waitFor(() -> handler.batches.size() >= 1, 5_000); - - sender.table("foo").symbol("s", "beta").longColumn("v", 2L).atNow(); - sender.flush(); - waitFor(() -> handler.batches.size() >= 2, 5_000); - } + try { + assertMemoryLeak(() -> { + CapturingHandler handler = new CapturingHandler(); + try (TestWebSocketServer server = new TestWebSocketServer(handler)) { + int port = server.getPort(); + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + + String config = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";"; + try (Sender sender = Sender.fromConfig(config)) { + sender.table("foo").symbol("s", "alpha").longColumn("v", 1L).atNow(); + sender.flush(); + waitFor(() -> handler.batches.size() >= 1, 5_000); + + sender.table("foo").symbol("s", "beta").longColumn("v", 2L).atNow(); + sender.flush(); + waitFor(() -> handler.batches.size() >= 2, 5_000); + } - // The planted directory is untouched -- the dictionary never opened, - // so delta encoding stayed disabled. - Assert.assertTrue("planted .symbol-dict directory must remain (open failed)", - Files.isDirectory(dictPath)); - - Assert.assertEquals("expected 2 captured batches", 2, handler.batches.size()); - byte[] b1 = handler.batches.get(0); - byte[] b2 = handler.batches.get(1); - - // Full-dict fallback: BOTH batches start at id 0, and batch 2 re-ships - // the WHOLE dictionary (alpha + beta), NOT a monotonic delta (which - // would be deltaStart=1, deltaCount=1 as in the test above). - Assert.assertEquals("batch 1 deltaStart must be 0", - 0, readVarint(b1, DELTA_START_OFFSET)); - Assert.assertEquals("batch 1 deltaCount must be 1", - 1, readVarint(b1, DELTA_START_OFFSET + 1)); - Assert.assertEquals("batch 2 deltaStart must be 0 (full-dict fallback, not monotonic)", - 0, readVarint(b2, DELTA_START_OFFSET)); - Assert.assertEquals("batch 2 deltaCount must be 2 (whole dictionary re-shipped)", - 2, readVarint(b2, DELTA_START_OFFSET + 1)); + // The planted directory is untouched -- the dictionary never + // opened, so delta encoding stayed disabled. + Assert.assertTrue("planted .symbol-dict directory must remain (open failed)", + Files.isDirectory(dictPath)); + + Assert.assertEquals("expected 2 captured batches", 2, handler.batches.size()); + byte[] b1 = handler.batches.get(0); + byte[] b2 = handler.batches.get(1); + + // Full-dict fallback: BOTH batches start at id 0, and batch 2 + // re-ships the WHOLE dictionary (alpha + beta), NOT a monotonic + // delta (which would be deltaStart=1, deltaCount=1 as above). + Assert.assertEquals("batch 1 deltaStart must be 0", + 0, readVarint(b1, DELTA_START_OFFSET)); + Assert.assertEquals("batch 1 deltaCount must be 1", + 1, readVarint(b1, DELTA_START_OFFSET + 1)); + Assert.assertEquals("batch 2 deltaStart must be 0 (full-dict fallback, not monotonic)", + 0, readVarint(b2, DELTA_START_OFFSET)); + Assert.assertEquals("batch 2 deltaCount must be 2 (whole dictionary re-shipped)", + 2, readVarint(b2, DELTA_START_OFFSET + 1)); + } + }); } finally { rmDir(sfDir); } @@ -181,38 +191,40 @@ private static boolean containsUtf8(byte[] haystack, String needle) { public void testMemoryModeShipsMonotonicDelta() throws Exception { // Memory-mode (no sf_dir): each symbol id ships once. Batch 2 carries // only "beta" as a delta starting at id 1, not the whole dictionary. - CapturingHandler handler = new CapturingHandler(); - try (TestWebSocketServer server = new TestWebSocketServer(handler)) { - int port = server.getPort(); - server.start(); - Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); - - try (Sender sender = Sender.fromConfig("ws::addr=localhost:" + port + ";")) { - sender.table("foo").symbol("s", "alpha").longColumn("v", 1L).atNow(); - sender.flush(); - waitFor(() -> handler.batches.size() >= 1, 5_000); - - sender.table("foo").symbol("s", "beta").longColumn("v", 2L).atNow(); - sender.flush(); - waitFor(() -> handler.batches.size() >= 2, 5_000); - } + assertMemoryLeak(() -> { + CapturingHandler handler = new CapturingHandler(); + try (TestWebSocketServer server = new TestWebSocketServer(handler)) { + int port = server.getPort(); + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); - Assert.assertEquals("expected 2 captured batches", 2, handler.batches.size()); - byte[] b1 = handler.batches.get(0); - byte[] b2 = handler.batches.get(1); - - // Batch 1 introduces alpha at id 0. - Assert.assertEquals("batch 1 deltaStart must be 0", - 0, readVarint(b1, DELTA_START_OFFSET)); - Assert.assertEquals("batch 1 deltaCount must be 1", - 1, readVarint(b1, DELTA_START_OFFSET + 1)); - - // Batch 2 ships ONLY beta as a delta from id 1. - Assert.assertEquals("batch 2 deltaStart must be 1 (monotonic)", - 1, readVarint(b2, DELTA_START_OFFSET)); - Assert.assertEquals("batch 2 deltaCount must be 1 (only the new symbol)", - 1, readVarint(b2, DELTA_START_OFFSET + 1)); - } + try (Sender sender = Sender.fromConfig("ws::addr=localhost:" + port + ";")) { + sender.table("foo").symbol("s", "alpha").longColumn("v", 1L).atNow(); + sender.flush(); + waitFor(() -> handler.batches.size() >= 1, 5_000); + + sender.table("foo").symbol("s", "beta").longColumn("v", 2L).atNow(); + sender.flush(); + waitFor(() -> handler.batches.size() >= 2, 5_000); + } + + Assert.assertEquals("expected 2 captured batches", 2, handler.batches.size()); + byte[] b1 = handler.batches.get(0); + byte[] b2 = handler.batches.get(1); + + // Batch 1 introduces alpha at id 0. + Assert.assertEquals("batch 1 deltaStart must be 0", + 0, readVarint(b1, DELTA_START_OFFSET)); + Assert.assertEquals("batch 1 deltaCount must be 1", + 1, readVarint(b1, DELTA_START_OFFSET + 1)); + + // Batch 2 ships ONLY beta as a delta from id 1. + Assert.assertEquals("batch 2 deltaStart must be 1 (monotonic)", + 1, readVarint(b2, DELTA_START_OFFSET)); + Assert.assertEquals("batch 2 deltaCount must be 1 (only the new symbol)", + 1, readVarint(b2, DELTA_START_OFFSET + 1)); + } + }); } private static int readVarint(byte[] buf, int offset) { From 27aad9edc4d4254ff5ce87bf66c75735cc9d8d0b Mon Sep 17 00:00:00 2001 From: glasstiger Date: Thu, 9 Jul 2026 22:38:08 +0100 Subject: [PATCH 14/96] Cover the split-batch delta contract flushPendingRowsSplit fires when one flush's encoded size exceeds the server's batch cap: it emits one frame per table. The first frame must carry the whole batch's symbol-dict delta and advance the baseline, and the remaining frames must carry an empty delta that only references ids the first frame already registered -- otherwise a fresh server would see dangling symbol ids. No test drove that producer-side split. Add a test that buffers two padded tables into one flush under a small advertised cap, so the batch splits, and asserts the first frame ships deltaStart=0/deltaCount=2 while the second ships deltaStart=2/deltaCount=0. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../qwp/client/SelfSufficientFramesTest.java | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java index af30da83..cb2b0118 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java @@ -227,6 +227,52 @@ public void testMemoryModeShipsMonotonicDelta() throws Exception { }); } + @Test + public void testSplitBatchShipsDeltaOnFirstFrameOnly() throws Exception { + // A single flush whose encoded size exceeds the server's batch cap is + // split into one frame per table (flushPendingRowsSplit). The FIRST split + // frame carries the whole batch's symbol-dict delta and advances the + // baseline; the remaining frames carry an EMPTY delta and only reference + // ids the first frame already registered. A regression that shipped each + // table's own symbols (wrong deltaStart) would dangle ids on the server. + assertMemoryLeak(() -> { + CapturingHandler handler = new CapturingHandler(); + try (TestWebSocketServer server = new TestWebSocketServer(handler)) { + server.setAdvertisedMaxBatchSize(150); // forces the two-table batch to split + int port = server.getPort(); + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + + // Padding inflates each table past half the cap, so the combined + // two-table message exceeds it while each single-table frame fits. + String pad = new String(new char[60]).replace('\0', 'x'); + try (Sender sender = Sender.fromConfig("ws::addr=localhost:" + port + ";")) { + // Buffer TWO tables (symbols "a" id 0, "b" id 1), then ONE flush. + sender.table("t1").symbol("s", "a").stringColumn("p", pad).longColumn("v", 1L).atNow(); + sender.table("t2").symbol("s", "b").stringColumn("p", pad).longColumn("v", 2L).atNow(); + sender.flush(); + waitFor(() -> handler.batches.size() >= 2, 5_000); + } + + Assert.assertEquals("the oversized two-table batch must split into 2 frames", + 2, handler.batches.size()); + byte[] f1 = handler.batches.get(0); + byte[] f2 = handler.batches.get(1); + + // First split frame ships the whole batch's dictionary (a + b). + Assert.assertEquals("first split frame deltaStart must be 0", + 0, readVarint(f1, DELTA_START_OFFSET)); + Assert.assertEquals("first split frame ships both new symbols", + 2, readVarint(f1, DELTA_START_OFFSET + 1)); + // Second split frame carries an empty delta above the advanced baseline. + Assert.assertEquals("second split frame deltaStart must be 2 (baseline advanced)", + 2, readVarint(f2, DELTA_START_OFFSET)); + Assert.assertEquals("second split frame carries no new symbols", + 0, readVarint(f2, DELTA_START_OFFSET + 1)); + } + }); + } + private static int readVarint(byte[] buf, int offset) { // Simple unsigned varint decode — sufficient for small values. int result = 0; From 034d8f30c780692c612dd6b93498eb4b36d2e8ad Mon Sep 17 00:00:00 2001 From: glasstiger Date: Thu, 9 Jul 2026 23:12:35 +0100 Subject: [PATCH 15/96] Accumulate the tail of a partial-overlap delta into the mirror accumulateSentDict dropped a frame entirely whenever deltaStart != sentDictCount. A delta that overlaps the mirror tip and extends past it (deltaStart < sentDictCount < deltaStart+deltaCount) was therefore discarded whole -- the new tail symbols never entered the mirror, which would leave a later reconnect catch-up incomplete and shift server-side ids. The producer only ever emits strictly contiguous, non-overlapping deltas, so this is currently unreachable, but it is load-bearing correctness resting on an invariant enforced elsewhere. Handle the overlap: skip the already-held prefix [deltaStart, sentDictCount) and copy only the new tail [sentDictCount, deltaStart+deltaCount). The steady-state case (deltaStart == sentDictCount) has skip == 0, so it is unchanged and free. A gap (deltaStart > sentDictCount, which the torn-dictionary guard rejects before send) now bails explicitly rather than implicitly. Also document that the I/O-thread mirror is a second, native copy of the dictionary (the producer's GlobalSymbolDictionary already holds the same symbols as Java Strings) -- so a memory-mode connection's steady-state dictionary footprint is ~2x the symbol set, an intentional cost of the reconnect-catch-up capability. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../sf/cursor/CursorWebSocketSendLoop.java | 44 ++++++++++++++++--- 1 file changed, 37 insertions(+), 7 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java index 9f41e1e6..3e14de02 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java @@ -193,6 +193,13 @@ public final class CursorWebSocketSendLoop implements QuietCloseable { // on reconnect it can re-register the whole dictionary on the fresh server // (which discards its dictionary on every disconnect) before replaying frames // whose deltas start above id 0. All of this is touched only by the I/O thread. + // Footprint note: this mirror is a SECOND copy of the dictionary -- the same + // symbols the producer's GlobalSymbolDictionary already holds as Java Strings -- + // kept as native UTF-8 bytes for the reconnect-catch-up capability. So a + // memory-mode connection's steady-state dictionary footprint is ~2x the symbol + // set. It is bounded by distinct-symbol count (not per-row) and never trimmed + // for the connection's lifetime (a reconnect may need the whole dictionary at + // any moment), so it cannot be dropped; it is an intentional cost of the feature. private final boolean deltaDictEnabled; private long sentDictBytesAddr; private int sentDictBytesCapacity; @@ -2025,15 +2032,38 @@ private void accumulateSentDict(long payloadAddr, int payloadLen, int deltaStart long p = payloadAddr + QwpConstants.HEADER_SIZE + NativeBufferWriter.varintSize(deltaStart); long deltaCount = readVarintAt(p, limit); p = varintEnd; - // Only a delta that extends exactly from our current tip introduces new - // symbols. deltaStart < sentDictCount is a replay/overlap we already hold; - // deltaStart > sentDictCount is rejected before send by the torn-dictionary - // guard in trySendOne, so it never reaches here. - if (deltaCount <= 0 || deltaStart != sentDictCount) { + // The mirror holds ids [0, sentDictCount). Accumulate ONLY the part of this + // frame's delta [deltaStart, deltaStart+deltaCount) that extends past the + // tip -- ids [sentDictCount, deltaStart+deltaCount). Cases: + // - deltaStart > sentDictCount: a gap. trySendOne's torn-dictionary guard + // rejects it before send, so it never reaches here; bail defensively + // rather than accumulate past a hole. + // - deltaEnd <= sentDictCount: a pure replay/overlap we already hold -- + // nothing new. + // - deltaStart <= sentDictCount < deltaEnd: extend the mirror by the tail. + // deltaStart == sentDictCount is the steady-state case (skip == 0). + // Handling the partial overlap explicitly -- rather than dropping the whole + // frame whenever deltaStart != sentDictCount -- keeps the mirror complete + // even if a future producer ever emits a delta that overlaps the tip; + // silently dropping the new tail would leave the reconnect catch-up + // incomplete and shift server-side ids. + long deltaEnd = deltaStart + deltaCount; + if (deltaCount <= 0 || deltaStart > sentDictCount || deltaEnd <= sentDictCount) { return; } + // Walk past the already-held prefix [deltaStart, sentDictCount), then copy + // the new tail [sentDictCount, deltaEnd). + int skip = sentDictCount - deltaStart; + for (int i = 0; i < skip; i++) { + long len = readVarintAt(p, limit); + p = varintEnd + len; + if (p > limit) { + return; // malformed -- bail rather than corrupt the mirror + } + } long regionStart = p; - for (long i = 0; i < deltaCount; i++) { + long newCount = deltaEnd - sentDictCount; + for (long i = 0; i < newCount; i++) { long len = readVarintAt(p, limit); p = varintEnd + len; if (p > limit) { @@ -2050,7 +2080,7 @@ private void accumulateSentDict(long payloadAddr, int payloadLen, int deltaStart ensureSentDictCapacity((long) sentDictBytesLen + regionBytes); Unsafe.getUnsafe().copyMemory(regionStart, sentDictBytesAddr + sentDictBytesLen, regionBytes); sentDictBytesLen += regionBytes; - sentDictCount += (int) deltaCount; + sentDictCount += (int) newCount; } private void ensureSentDictCapacity(long required) { From 41fddb11cd409af8a7452c46a220d702be909c80 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Thu, 9 Jul 2026 23:12:46 +0100 Subject: [PATCH 16/96] Guard against persisted-dict duplication on a failed publish Regression test for the write-ahead persist path: persistNewSymbolsBefore- Publish runs before the frame is published (sealAndSwapBuffer -> appendBlocking). If publish fails after the persist -- here PAYLOAD_TOO_LARGE (a frame bigger than the SF segment), a backpressure deadline in production -- the symbols are already on disk but sentMaxSymbolId is not advanced and the rows stay buffered, so a retry re-runs the persist. The fix keys the persist range off pd.size() (idempotent); this pins it. The test drives one new-symbol row whose padded frame exceeds a 1 KB segment, flushes it twice (both fail to publish), then asserts the persisted .symbol-dict holds the symbol exactly once. Reverting the fix to sentMaxSymbolId+1 fails it with size 2 -- the duplicate that shifts every later global id and silently misattributes symbol values on recovery. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../qwp/client/DeltaDictRecoveryTest.java | 65 +++++++++++++++++++ 1 file changed, 65 insertions(+) diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java index 8d1bd24a..143686b4 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java @@ -26,6 +26,7 @@ import io.questdb.client.Sender; import io.questdb.client.cutlass.line.LineSenderException; +import io.questdb.client.cutlass.qwp.client.sf.cursor.PersistedSymbolDict; import io.questdb.client.std.Files; import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer; import io.questdb.client.test.tools.TestUtils; @@ -226,6 +227,70 @@ public void testTornDictionaryFailsCleanlyInsteadOfCorrupting() throws Exception }); } + @Test + public void testFailedPublishDoesNotDuplicatePersistedSymbols() throws Exception { + // Regression: persistNewSymbolsBeforePublish is a write-ahead -- it runs + // BEFORE the frame is published (sealAndSwapBuffer -> appendBlocking). If + // publish fails (here PAYLOAD_TOO_LARGE, a frame bigger than the SF + // segment; a backpressure deadline in production), the frame's symbols are + // already on disk but sentMaxSymbolId is NOT advanced and the rows stay + // buffered -- so a retry re-runs the persist. Keying the persist range off + // pd.size() (not sentMaxSymbolId+1) makes it idempotent. Before that fix, + // the retry appended the symbol a SECOND time, breaking the dense + // id->position invariant; on recovery every later global id shifts by one + // and symbol column values are silently misattributed. + assertMemoryLeak(() -> { + try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) { + int port = silent.getPort(); + silent.start(); + Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS)); + + // Small segment; a heavily padded row's frame cannot fit, so + // appendBlocking throws PAYLOAD_TOO_LARGE deterministically -- no + // backpressure timing needed. The server never acks (SilentHandler). + String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";sf_max_bytes=1024;"; + String pad = TestUtils.repeat("x", 2000); // frame >> 1024-byte segment + Sender sender = Sender.fromConfig(cfg); + try { + // Buffer ONE new-symbol row, then flush it TWICE. Each flush + // runs the write-ahead persist and then fails to publish; the + // failed flush leaves the row buffered, so the second flush is + // the retry that (pre-fix) duplicated the persisted symbol. + sender.table("m").symbol("s", "s0").stringColumn("p", pad).longColumn("v", 1L).atNow(); + for (int attempt = 0; attempt < 2; attempt++) { + try { + sender.flush(); + Assert.fail("oversized frame must fail to publish"); + } catch (LineSenderException expected) { + // frame too large -- expected on every attempt + } + } + + // The persisted dictionary must hold "s0" EXACTLY ONCE. + // Pre-fix, the retry duplicated it (size == 2). + PersistedSymbolDict pd = PersistedSymbolDict.open(Paths.get(sfDir, "default").toString()); + Assert.assertNotNull(pd); + try { + Assert.assertEquals("failed-publish retry must not duplicate the persisted symbol", + 1, pd.size()); + Assert.assertEquals("s0", pd.readLoadedSymbols().getQuick(0)); + } finally { + pd.close(); + } + } finally { + try { + sender.close(); + } catch (LineSenderException ignored) { + // close() re-flushes the still-buffered oversized row and + // fails again (PAYLOAD_TOO_LARGE); expected here and not + // what we assert. close() still runs its resource cleanup, + // so no native memory leaks. + } + } + } + }); + } + private static void writeAckWatermark(java.nio.file.Path path, long fsn) throws IOException { byte[] buf = new byte[16]; ByteBuffer bb = ByteBuffer.wrap(buf).order(ByteOrder.LITTLE_ENDIAN); From 0ef96afbcf840540c8b45a7ba07c7a33cf750a1e Mon Sep 17 00:00:00 2001 From: glasstiger Date: Fri, 10 Jul 2026 00:43:35 +0100 Subject: [PATCH 17/96] Cover the reconnect catch-up ACK alignment setWireBaselineWithCatchUp anchors fsnAtZero = replayStart - catchUpFrames so every catch-up frame maps to an already-acked FSN. Dropping the - catchUpFrames term is silent data loss: a server ACK for a catch-up frame then translates to an FSN at or above replayStart and trims a not-yet-delivered data frame from the store-and-forward log. The existing catch-up tests reconstruct the dictionary from wire bytes and never assert ACK/trim accounting, so they were blind to this line; the enterprise SqlFailoverQwpClientLosslessTest ingests no symbols and never enters the catch-up path at all. CursorWebSocketSendLoopCatchUpAlignmentTest drives the catch-up against a stub client and asserts the catch-up frame's OK leaves the real engine's ackedFsn untouched, for both a single catch-up frame and a split (multi-frame) catch-up. Reverting the - catchUpFrames term fails both. Co-Authored-By: Claude Opus 4.8 (1M context) --- ...WebSocketSendLoopCatchUpAlignmentTest.java | 324 ++++++++++++++++++ 1 file changed, 324 insertions(+) create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java new file mode 100644 index 00000000..ed337f00 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java @@ -0,0 +1,324 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client.sf.cursor; + +import io.questdb.client.DefaultHttpClientConfiguration; +import io.questdb.client.cutlass.http.client.WebSocketClient; +import io.questdb.client.cutlass.qwp.client.WebSocketResponse; +import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine; +import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorWebSocketSendLoop; +import io.questdb.client.network.PlainSocketFactory; +import io.questdb.client.std.Files; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.Unsafe; +import io.questdb.client.test.tools.TestUtils; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.lang.reflect.Field; +import java.lang.reflect.Method; +import java.nio.charset.StandardCharsets; +import java.nio.file.Paths; + +import static org.junit.Assert.assertEquals; + +/** + * Guards the reconnect/failover symbol-dictionary catch-up ACK alignment in + * {@link CursorWebSocketSendLoop#setWireBaselineWithCatchUp}. + *

+ * On a fresh connection the loop re-registers the whole dictionary with a + * catch-up frame BEFORE replaying data frames. Each catch-up frame consumes a + * wire sequence, so the loop anchors {@code fsnAtZero = replayStart - catchUpFrames} + * to keep every catch-up frame mapped to an already-acked FSN. Dropping the + * {@code - catchUpFrames} term is silent data loss: a server ACK for a catch-up + * frame then translates through {@code engine.acknowledge(fsnAtZero + wireSeq)} + * to an FSN at or above {@code replayStart}, trimming a not-yet-delivered data + * frame from the store-and-forward log. + *

+ * The loop is constructed but never {@link CursorWebSocketSendLoop#start started}; + * the catch-up runs against a stub {@link WebSocketClient} that counts frames, and + * the OK is delivered straight into the inner {@code ResponseHandler} -- the same + * white-box idiom {@code CursorWebSocketSendLoopDurableAckTest} uses, because + * {@code setWireBaselineWithCatchUp} and the wire ports have no public entry point. + * {@link CursorSendEngine#ackedFsn()} is the authoritative trim watermark asserted + * against. + */ +public class CursorWebSocketSendLoopCatchUpAlignmentTest { + + private String tmpDir; + + @Before + public void setUp() { + tmpDir = Paths.get(System.getProperty("java.io.tmpdir"), + "qdb-cursor-catchup-" + System.nanoTime()).toString(); + assertEquals(0, Files.mkdir(tmpDir, Files.DIR_MODE_DEFAULT)); + } + + @After + public void tearDown() { + if (tmpDir == null) return; + long find = Files.findFirst(tmpDir); + if (find > 0) { + try { + int rc = 1; + while (rc > 0) { + String name = Files.utf8ToString(Files.findName(find)); + if (name != null && !".".equals(name) && !"..".equals(name)) { + Files.remove(tmpDir + "/" + name); + } + rc = Files.findNext(find); + } + } finally { + Files.findClose(find); + } + } + Files.remove(tmpDir); + } + + @Test + public void testCatchUpFrameAckDoesNotAdvanceTrimWatermark() throws Exception { + // Single catch-up frame (server advertises no cap). Two frames were + // acked before the reconnect (ackedFsn=1), FSN 2 is unacked. The catch-up + // frame's OK must NOT advance the watermark past 1 -- it carries no data, + // only the dictionary the fresh server needs before replay. + TestUtils.assertMemoryLeak(() -> { + CatchUpCapturingClient client = new CatchUpCapturingClient(0); // 0 => no cap => one frame + try (CursorSendEngine engine = newEngine()) { + appendFrames(engine, 3); // FSN 0,1,2 published + engine.acknowledge(1); // ackedFsn=1 => replayStart=2, FSN 2 still unacked + CursorWebSocketSendLoop loop = newLoop(engine, client); + try { + seedMirror(loop, "s0", "s1"); // sentDictCount=2 => catch-up fires + long replayStart = engine.ackedFsn() + 1L; // = 2 + + invokeSetWireBaselineWithCatchUp(loop, replayStart); + + assertEquals("whole dictionary fits one frame under no cap", + 1, client.framesSent); + + // Behavioural (the harm): the catch-up frame (wire seq 0) is + // OK'd by the fresh server. It carries no data, so it must + // resolve to an already-acked FSN and leave the trim watermark + // untouched -- advancing it would trim the undelivered FSN 2. + deliverOk(loop, 0); + assertEquals("catch-up frame ACK must not advance the trim watermark " + + "(would trim an undelivered data frame -> silent data loss)", + 1L, engine.ackedFsn()); + // Mechanism: the catch-up frames are anchored below replayStart. + assertEquals("fsnAtZero must be anchored catchUpFrames below replayStart", + replayStart - client.framesSent, readLong(loop, "fsnAtZero")); + } finally { + loop.close(); // frees the seeded mirror + the stub client's buffers + } + } + }); + } + + @Test + public void testSplitCatchUpFramesAcksDoNotAdvanceTrimWatermark() throws Exception { + // A small advertised cap splits the dictionary across several catch-up + // frames, so the fsnAtZero offset must subtract the full frame count. Ack + // the LAST catch-up wire sequence: it still maps below replayStart. With + // the offset dropped it would translate to replayStart+1 and over-trim. + TestUtils.assertMemoryLeak(() -> { + CatchUpCapturingClient client = new CatchUpCapturingClient(40); // budget 12 => one 11-byte symbol per frame + try (CursorSendEngine engine = newEngine()) { + appendFrames(engine, 5); // FSN 0..4 published + engine.acknowledge(2); // ackedFsn=2 => replayStart=3, FSN 3,4 unacked + CursorWebSocketSendLoop loop = newLoop(engine, client); + try { + seedMirror(loop, "symbol0000", "symbol0001"); // 11 bytes each -> two frames + long replayStart = engine.ackedFsn() + 1L; // = 3 + + invokeSetWireBaselineWithCatchUp(loop, replayStart); + + assertEquals("cap must split the two symbols across two frames", + 2, client.framesSent); + + // ACK the highest catch-up wire sequence (the last catch-up + // frame). It too must map below replayStart -- with the offset + // dropped it translates to replayStart+1 and over-trims. + deliverOk(loop, client.framesSent - 1); + assertEquals("no catch-up frame ACK may advance the trim watermark", + 2L, engine.ackedFsn()); + assertEquals("fsnAtZero must subtract the full split frame count", + replayStart - client.framesSent, readLong(loop, "fsnAtZero")); + } finally { + loop.close(); + } + } + }); + } + + private static void appendFrames(CursorSendEngine engine, int count) { + long buf = Unsafe.malloc(16, MemoryTag.NATIVE_DEFAULT); + try { + byte[] payload = "frame-bytes-padd".getBytes(StandardCharsets.US_ASCII); + for (int i = 0; i < payload.length; i++) { + Unsafe.getUnsafe().putByte(buf + i, payload[i]); + } + for (int i = 0; i < count; i++) { + engine.appendBlocking(buf, 16); + } + } finally { + Unsafe.free(buf, 16, MemoryTag.NATIVE_DEFAULT); + } + } + + // Delivers a 0-table STATUS_OK for {@code wireSeq} into the loop's response + // handler, mimicking the server acking a catch-up frame (which carries no tables). + private static void deliverOk(CursorWebSocketSendLoop loop, long wireSeq) throws Exception { + int size = 11; // status(1) + sequence(8) + tableCount(2) + long ptr = Unsafe.malloc(size, MemoryTag.NATIVE_DEFAULT); + try { + Unsafe.getUnsafe().putByte(ptr, WebSocketResponse.STATUS_OK); + Unsafe.getUnsafe().putLong(ptr + 1, wireSeq); + Unsafe.getUnsafe().putShort(ptr + 9, (short) 0); + Field f = CursorWebSocketSendLoop.class.getDeclaredField("responseHandler"); + f.setAccessible(true); + Object handler = f.get(loop); + Method m = handler.getClass().getDeclaredMethod("onBinaryMessage", long.class, int.class); + m.setAccessible(true); + m.invoke(handler, ptr, size); + } finally { + Unsafe.free(ptr, size, MemoryTag.NATIVE_DEFAULT); + } + } + + private static void invokeSetWireBaselineWithCatchUp(CursorWebSocketSendLoop loop, long replayStart) throws Exception { + Method m = CursorWebSocketSendLoop.class.getDeclaredMethod("setWireBaselineWithCatchUp", long.class); + m.setAccessible(true); + m.invoke(loop, replayStart); + } + + private CursorWebSocketSendLoop newLoop(CursorSendEngine engine, WebSocketClient client) { + return new CursorWebSocketSendLoop( + client, engine, 0L, CursorWebSocketSendLoop.DEFAULT_PARK_NANOS, + () -> { + throw new UnsupportedOperationException("test loop is never started"); + }, + 5_000L, 100L, 5_000L, false); + } + + private CursorSendEngine newEngine() { + return new CursorSendEngine(tmpDir, 16384); + } + + private static long readLong(CursorWebSocketSendLoop loop, String name) throws Exception { + Field f = CursorWebSocketSendLoop.class.getDeclaredField(name); + f.setAccessible(true); + return f.getLong(loop); + } + + // Populates the loop's native sent-dictionary mirror with {@code symbols} in + // the on-wire [len varint][utf8] layout, so setWireBaselineWithCatchUp sees a + // non-empty dictionary to re-register. loop.close() frees it. + private static void seedMirror(CursorWebSocketSendLoop loop, String... symbols) throws Exception { + int total = 0; + for (String s : symbols) { + int len = s.getBytes(StandardCharsets.UTF_8).length; + total += varintSize(len) + len; + } + long addr = Unsafe.malloc(total, MemoryTag.NATIVE_DEFAULT); + long p = addr; + for (String s : symbols) { + byte[] bytes = s.getBytes(StandardCharsets.UTF_8); + p = writeVarint(p, bytes.length); + for (byte b : bytes) { + Unsafe.getUnsafe().putByte(p++, b); + } + } + setField(loop, "sentDictBytesAddr", addr); + setIntField(loop, "sentDictBytesCapacity", total); + setIntField(loop, "sentDictBytesLen", total); + setIntField(loop, "sentDictCount", symbols.length); + } + + private static void setField(Object target, String name, long value) throws Exception { + Field f = CursorWebSocketSendLoop.class.getDeclaredField(name); + f.setAccessible(true); + f.setLong(target, value); + } + + private static void setIntField(Object target, String name, int value) throws Exception { + Field f = CursorWebSocketSendLoop.class.getDeclaredField(name); + f.setAccessible(true); + f.setInt(target, value); + } + + private static int varintSize(long value) { + int n = 1; + while (value > 0x7F) { + value >>>= 7; + n++; + } + return n; + } + + private static long writeVarint(long addr, long value) { + while (value > 0x7F) { + Unsafe.getUnsafe().putByte(addr++, (byte) ((value & 0x7F) | 0x80)); + value >>>= 7; + } + Unsafe.getUnsafe().putByte(addr++, (byte) value); + return addr; + } + + // Stub transport: completes no real I/O. getServerMaxBatchSize drives the + // catch-up split; sendBinary counts the frames the catch-up emitted. + private static final class CatchUpCapturingClient extends WebSocketClient { + private final int cap; + private int framesSent; + + CatchUpCapturingClient(int cap) { + super(DefaultHttpClientConfiguration.INSTANCE, PlainSocketFactory.INSTANCE); + this.cap = cap; + } + + @Override + public int getServerMaxBatchSize() { + return cap; + } + + @Override + public int getServerQwpVersion() { + return 1; + } + + @Override + public void sendBinary(long dataPtr, int length) { + framesSent++; + } + + @Override + protected void ioWait(int timeout, int op) { + } + + @Override + protected void setupIoWait() { + } + } +} From ff79b72f9c798b79e8a558314b34932277209d8c Mon Sep 17 00:00:00 2001 From: glasstiger Date: Fri, 10 Jul 2026 00:47:08 +0100 Subject: [PATCH 18/96] Cover the retriable catch-up send containment sendCatchUpChunk throws CatchUpSendException on a transient wire failure instead of calling fail(). From inside the catch-up fail() re-enters connectLoop -- desyncing the fsnAtZero/nextWireSeq wire mapping (a later ACK then trims un-acked store-and-forward frames), or overflowing the stack on a flapping connection -- turning a transient outage into a hard failure. Only the oversized-entry (non-retriable) terminal was covered; the retriable path had no test. testTransientCatchUpSendFailureIsRetriableNotTerminal drives the catch-up against a stub whose sendBinary throws, and asserts the failure surfaces as a retriable CatchUpSendException and leaves the producer-facing error latch clear. Reverting the throw to fail() fails it. Co-Authored-By: Claude Opus 4.8 (1M context) --- ...WebSocketSendLoopCatchUpAlignmentTest.java | 51 ++++++++++++++++++- 1 file changed, 50 insertions(+), 1 deletion(-) diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java index ed337f00..9b2195d9 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java @@ -39,11 +39,13 @@ import org.junit.Test; import java.lang.reflect.Field; +import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.nio.charset.StandardCharsets; import java.nio.file.Paths; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; /** * Guards the reconnect/failover symbol-dictionary catch-up ACK alignment in @@ -173,6 +175,42 @@ public void testSplitCatchUpFramesAcksDoNotAdvanceTrimWatermark() throws Excepti }); } + @Test + public void testTransientCatchUpSendFailureIsRetriableNotTerminal() throws Exception { + // A transient wire failure WHILE shipping the catch-up (the fresh + // connection drops mid-handshake) must surface as a retriable + // CatchUpSendException for the reconnect loop to handle -- it must NOT + // call fail(). From inside the catch-up fail() re-enters connectLoop + // (corrupting the fsnAtZero/nextWireSeq mapping, or overflowing the stack + // on a flapping connection) or, with no reconnect attempt reachable, + // latches a terminal -- turning a transient outage into a hard failure and + // breaking store-and-forward. Only the oversized-entry (non-retriable) + // terminal was covered; this pins the retriable path. + TestUtils.assertMemoryLeak(() -> { + CatchUpCapturingClient client = new CatchUpCapturingClient(0, true); // sendBinary throws + try (CursorSendEngine engine = newEngine()) { + appendFrames(engine, 2); + engine.acknowledge(0); // ackedFsn=0 => a real unacked frame exists behind the catch-up + CursorWebSocketSendLoop loop = newLoop(engine, client); + try { + seedMirror(loop, "s0", "s1"); // non-empty dict => catch-up fires and hits the failing send + try { + invokeSetWireBaselineWithCatchUp(loop, engine.ackedFsn() + 1L); + fail("a transient catch-up send failure must raise a retriable " + + "CatchUpSendException, not be swallowed into fail()/a terminal"); + } catch (InvocationTargetException e) { + assertEquals("transient catch-up send failure must surface as CatchUpSendException", + "CatchUpSendException", e.getCause().getClass().getSimpleName()); + } + // Retriable, not terminal: the producer-facing error latch stays clear. + loop.checkError(); + } finally { + loop.close(); + } + } + }); + } + private static void appendFrames(CursorSendEngine engine, int count) { long buf = Unsafe.malloc(16, MemoryTag.NATIVE_DEFAULT); try { @@ -288,14 +326,22 @@ private static long writeVarint(long addr, long value) { } // Stub transport: completes no real I/O. getServerMaxBatchSize drives the - // catch-up split; sendBinary counts the frames the catch-up emitted. + // catch-up split; sendBinary counts the frames the catch-up emitted, or -- + // when throwOnSend is set -- raises a transient wire error to model the fresh + // connection dropping mid-catch-up. private static final class CatchUpCapturingClient extends WebSocketClient { private final int cap; + private final boolean throwOnSend; private int framesSent; CatchUpCapturingClient(int cap) { + this(cap, false); + } + + CatchUpCapturingClient(int cap, boolean throwOnSend) { super(DefaultHttpClientConfiguration.INSTANCE, PlainSocketFactory.INSTANCE); this.cap = cap; + this.throwOnSend = throwOnSend; } @Override @@ -310,6 +356,9 @@ public int getServerQwpVersion() { @Override public void sendBinary(long dataPtr, int length) { + if (throwOnSend) { + throw new RuntimeException("transient wire failure during catch-up"); + } framesSent++; } From 8fd9ad19a2860b47097b8fe861df2f41c5027474 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Fri, 10 Jul 2026 00:55:59 +0100 Subject: [PATCH 19/96] Tidy catch-up comments and harden two edges Four minor cleanups on the delta symbol-dictionary catch-up, all behaviour-preserving on every reachable path: - The sentDict* field comment said the catch-up mirror is memory-mode only; it is also seeded and used in disk mode on a recovered / orphan-drained slot. Corrected. - positionCursorAt's javadoc said it runs after nextWireSeq was reset to 0, but the catch-up path leaves nextWireSeq past the frames it emitted. Corrected to describe setWireBaselineWithCatchUp anchoring the wire baseline; the method only moves the byte cursor. - The recovery-seed constructor set sentDictCount = pd.size() outside the loadedEntriesLen > 0 block. A recovered slot always has entries when size > 0, so the result is unchanged, but coupling the count to the mirror bytes stops sentDictCount ever claiming symbols the mirror does not hold. - sendDictCatchUp used Integer.MAX_VALUE as the no-cap per-frame budget, so sendCatchUpChunk's int frameLen could overflow on a multi-GB dictionary. Bound it by MAX_SENT_DICT_BYTES, the same ceiling ensureSentDictCapacity enforces. Unreachable at real cardinality (~200M+ symbols); defensive. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../sf/cursor/CursorWebSocketSendLoop.java | 26 ++++++++++++++----- 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java index 3e14de02..7a9086d8 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java @@ -186,7 +186,9 @@ public final class CursorWebSocketSendLoop implements QuietCloseable { private final long reconnectMaxDurationMillis; private final WebSocketResponse response = new WebSocketResponse(); private final ResponseHandler responseHandler = new ResponseHandler(); - // Delta symbol dictionary catch-up state (memory-mode only; see swapClient). + // Delta symbol dictionary catch-up state (see swapClient). Active in memory + // mode, and in disk mode on a recovered / orphan-drained slot (the constructor + // seeds sentDict* from the persisted dictionary there). // deltaDictEnabled gates all of it. The loop mirrors, in sentDict*, every // symbol it has ever sent -- the concatenated [len varint][utf8] bytes in // global-id order (sentDictBytes*) plus the count (sentDictCount) -- so that @@ -532,8 +534,12 @@ public CursorWebSocketSendLoop(WebSocketClient client, CursorSendEngine engine, ensureSentDictCapacity(len); Unsafe.getUnsafe().copyMemory(pd.loadedEntriesAddr(), sentDictBytesAddr, len); sentDictBytesLen = len; + // Set the count only alongside the bytes so sentDictCount can + // never claim symbols the mirror does not hold. A recovered slot + // always has loadedEntriesLen > 0 when size > 0, so this is the + // same result -- it just makes the coupling explicit. + sentDictCount = pd.size(); } - sentDictCount = pd.size(); } } this.fsnAtZero = fsnAtZero; @@ -1784,8 +1790,11 @@ private void ioLoop() { /** * Walk the engine's segments to find the one containing {@code targetFsn}, * and set {@code sendOffset} to the byte offset of that frame within it. - * This is called at startup and after every reconnect, after fsnAtZero has - * already been reset to {@code targetFsn} and nextWireSeq to 0. + * This is called at startup and after every reconnect, once + * {@link #setWireBaselineWithCatchUp} has anchored the wire baseline + * ({@code fsnAtZero} / {@code nextWireSeq}) -- which may leave {@code nextWireSeq} + * past the catch-up frames it emitted. This method only positions the byte + * cursor at {@code targetFsn}; it does not touch the wire mapping. *

* If {@code targetFsn} is already published, the method positions the byte * cursor exactly at that frame. If {@code targetFsn} is not published yet, @@ -2133,8 +2142,13 @@ private int sendDictCatchUp() { int cap = client.getServerMaxBatchSize(); // Symbol-bytes budget per frame, leaving room for the 12-byte header and // the two delta-section varints. cap <= 0 means the server advertised no - // limit -> send the whole dictionary in one frame. - int budget = cap > 0 ? Math.max(1, cap - QwpConstants.HEADER_SIZE - 16) : Integer.MAX_VALUE; + // limit -- send the whole dictionary in one frame, but still bound the + // budget by MAX_SENT_DICT_BYTES so sendCatchUpChunk's int frameLen + // (HEADER_SIZE + varints + symbolsLen) cannot overflow on a pathological + // multi-GB dictionary (unreachable at real cardinality; defensive). + int budget = cap > 0 + ? Math.max(1, cap - QwpConstants.HEADER_SIZE - 16) + : MAX_SENT_DICT_BYTES - QwpConstants.HEADER_SIZE - 16; int framesSent = 0; int chunkStartId = 0; long chunkStartAddr = sentDictBytesAddr; From 8fcd8359089e9b177fc24706a72644a15f72def6 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Fri, 10 Jul 2026 02:46:13 +0100 Subject: [PATCH 20/96] Fix delta symbol-dict recovery and reconnect bugs Close three ways the delta symbol-dictionary feature could lose or corrupt data on the reconnect and store-and-forward recovery paths. Run the torn-dictionary guard unconditionally. trySendOne gated the guard on deltaDictEnabled, which CursorSendEngine reports false when a recovered disk slot cannot open its persisted dictionary (fd exhaustion, a read-only remount, ENOSPC). The recorded frames are still delta frames, so replaying them against a fresh empty-dictionary server null-padded the missing ids and silently corrupted the table. The guard now decodes the delta start for every frame and fails terminally on a gap regardless of the flag; only the sent-dictionary mirror stays gated. Stop treating a catch-up frame as the head data frame. sendCatchUpChunk advances nextWireSeq, but onClose's poison-strike gate and handleServerRejection's pre-send gate read nextWireSeq > 0 as "a data frame was sent". A transient non-orderly close or NACK after the catch-up but before the first replay frame then charged a poison strike on a frame that never left, and after a few flaps escalated a transient outage to a PROTOCOL_VIOLATION terminal that quarantines an orphan drainer. A new dataFrameSentThisConnection flag, set only after a real ring frame sends, now gates both decisions, so the drainer keeps retrying as Invariant B requires. Bound the commit message's dictionary delta to the sent watermark. sendCommitMessage skips the write-ahead persist yet encoded a delta up to currentBatchMaxSymbolId, so a symbol left in the batch by a cancelled row (cancelRow rolls back neither currentBatchMaxSymbolId nor the global registration) rode out on the commit frame without being persisted. A recovered slot then under-seeded the producer against the surviving frame and misattributed the reused id. The commit now caps the delta at sentMaxSymbolId in delta mode, giving an empty delta. Each fix carries a regression test proven to fail when the fix is reverted: a directory-shadowed .symbol-dict (guard), a close after only the catch-up (poison gate), and a cancelled-row symbol on a transactional commit (delta bound). Co-Authored-By: Claude Opus 4.8 (1M context) --- .../qwp/client/QwpWebSocketSender.java | 19 ++- .../sf/cursor/CursorWebSocketSendLoop.java | 100 ++++++++----- .../qwp/client/DeltaDictRecoveryTest.java | 133 ++++++++++++++++++ ...ursorWebSocketSendLoopPoisonFrameTest.java | 51 +++++++ 4 files changed, 264 insertions(+), 39 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index 77e85392..4a499300 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -3623,10 +3623,23 @@ private void sendCommitMessage() { LOG.debug("Sending commit message for deferred batch"); } encoder.setDeferCommit(false); - // A commit carries no rows and no new symbols; in delta mode its empty - // delta simply starts at the server's current dictionary size. + // A commit carries no rows, and it must also carry NO new symbols. Unlike + // the flush paths, sendCommitMessage does NOT write-ahead-persist the + // dictionary, so shipping a symbol here would put an id on the wire that a + // recovered slot cannot rebuild from the persisted .symbol-dict, diverging + // the producer dictionary from the surviving frames and silently + // misattributing reused ids after a crash. currentBatchMaxSymbolId can sit + // ABOVE sentMaxSymbolId (e.g. a cancelled row: cancelRow does not roll back + // currentBatchMaxSymbolId or unregister the symbol), so bound the delta at + // what has already been sent -- and therefore already persisted. In delta + // mode pass sentMaxSymbolId, yielding an empty delta + // [sentMaxSymbolId+1 .. sentMaxSymbolId]; in full-dict mode keep + // currentBatchMaxSymbolId so the frame stays self-sufficient. Any symbol a + // cancelled row leaked is picked up (and persisted) by the next real flush, + // whose persistNewSymbolsBeforePublish resumes from pd.size(). + int commitBatchMaxId = deltaDictEnabled ? sentMaxSymbolId : currentBatchMaxSymbolId; encoder.beginMessage(0, globalSymbolDictionary, - symbolDeltaBaseline(), currentBatchMaxSymbolId); + symbolDeltaBaseline(), commitBatchMaxId); int messageSize = encoder.finishMessage(); QwpBufferWriter buffer = encoder.getBuffer(); ensureActiveBufferReady(); diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java index 7a9086d8..2772cdf8 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java @@ -203,6 +203,18 @@ public final class CursorWebSocketSendLoop implements QuietCloseable { // for the connection's lifetime (a reconnect may need the whole dictionary at // any moment), so it cannot be dropped; it is an intentional cost of the feature. private final boolean deltaDictEnabled; + // True once a real ring frame (data or commit) has been sent on the CURRENT + // connection, as opposed to only the dictionary catch-up. The catch-up + // consumes wire sequences (nextWireSeq), so nextWireSeq > 0 no longer implies + // "the head frame was sent": onClose's poison-strike gate and + // handleServerRejection's pre-send gate key off THIS instead. Without it, a + // transient outage AFTER the catch-up but BEFORE the first data frame (a + // flapping LB/middlebox that accepts the upgrade + catch-up then closes) would + // be mistaken for a deterministic head-frame rejection and escalate to a + // PROTOCOL_VIOLATION terminal -- breaking the store-and-forward "retry a + // transient outage forever" contract. Reset per connection in + // setWireBaselineWithCatchUp; set in trySendOne after a successful send. + private boolean dataFrameSentThisConnection; private long sentDictBytesAddr; private int sentDictBytesCapacity; private int sentDictBytesLen; @@ -1980,6 +1992,11 @@ private void swapClient(WebSocketClient newClient) { * first real connection via swapClient. */ private void setWireBaselineWithCatchUp(long replayStart) { + // Fresh connection: no data frame has been sent on it yet. Reset before the + // catch-up (which sends only dictionary frames) so onClose / + // handleServerRejection can tell "only the catch-up went out" from "the + // head data frame went out". + dataFrameSentThisConnection = false; if (client != null && deltaDictEnabled && sentDictCount > 0) { this.nextWireSeq = 0L; // The catch-up may span several frames when the dictionary exceeds the @@ -2338,29 +2355,31 @@ private boolean trySendOne() { return false; // payload not fully published yet } long frameAddr = base + sendOffset + MmapSegment.FRAME_HEADER_SIZE; - // -1 unless this is a delta frame; the guard decodes it once here and - // accumulateSentDict reuses it post-send, so the delta header is parsed - // once per frame rather than twice. - int deltaStart = -1; - if (deltaDictEnabled) { - // Torn-dictionary guard. In normal operation a delta frame's start id - // never exceeds the dictionary coverage established so far (replayed - // frames overlap the catch-up dict; fresh frames extend it - // contiguously). A gap here means the persisted dictionary was torn -- - // almost always by a host/power crash, which leaves segment frames on - // disk but loses recently-written dictionary entries (SF, like the rest - // of the store, is process-crash durable but not host-crash durable). - // Sending the frame would corrupt the table (the server would null-pad - // the missing ids), so fail terminally instead; the unreplayable data - // must be resent. - deltaStart = frameDeltaStart(frameAddr, payloadLen); - if (deltaStart > sentDictCount) { - recordFatal(new LineSenderException( - "recovered store-and-forward symbol dictionary is incomplete (likely a host crash): " - + "frame delta start " + deltaStart + " exceeds recovered dictionary size " - + sentDictCount + "; cannot replay without corrupting data -- resend required")); - return false; - } + // Torn-dictionary guard. Decode the delta start unconditionally (-1 for a + // non-delta frame); the guard MUST run even when deltaDictEnabled is false. + // A disk slot recovered with its persisted dictionary unavailable + // (PersistedSymbolDict.open() returned null -- fd exhaustion, a read-only + // remount, ENOSPC) reports deltaDictEnabled=false, yet its recorded frames + // are still DELTA frames (deltaStart > 0). Replaying those against a fresh + // empty-dictionary server would null-pad the missing ids and SILENTLY + // corrupt the table -- precisely what this guard exists to prevent -- so it + // cannot be gated on the very flag that goes false in that failure mode. In + // normal operation a delta frame's start id never exceeds the dictionary + // coverage established so far (replayed frames overlap the catch-up dict; + // fresh frames extend it contiguously), so a gap here means the recovered + // dictionary is incomplete (a host/power crash that lost recently-written + // entries, SF being process-crash but not host-crash durable). Fail + // terminally; the unreplayable data must be resent. Full-dict / fallback + // frames carry deltaStart=0 with sentDictCount=0, so 0 > 0 never + // false-positives; only the sent-dictionary mirror below stays gated on + // deltaDictEnabled. + int deltaStart = frameDeltaStart(frameAddr, payloadLen); + if (deltaStart > sentDictCount) { + recordFatal(new LineSenderException( + "recovered store-and-forward symbol dictionary is incomplete (likely a host crash): " + + "frame delta start " + deltaStart + " exceeds recovered dictionary size " + + sentDictCount + "; cannot replay without corrupting data -- resend required")); + return false; } try { client.sendBinary(frameAddr, payloadLen); @@ -2368,7 +2387,12 @@ private boolean trySendOne() { fail(t); return false; } - if (deltaStart >= 0) { + // A real ring frame (data or commit) has now gone out on this connection, + // as opposed to only the dictionary catch-up. onClose / handleServerRejection + // key their poison-strike vs pre-send decision off this, not off nextWireSeq + // (which the catch-up advances). + dataFrameSentThisConnection = true; + if (deltaDictEnabled && deltaStart >= 0) { // Mirror the symbols this frame introduced so a later reconnect can // rebuild the whole dictionary. Idempotent on replay: a frame whose // delta we already hold advances nothing. @@ -2619,7 +2643,7 @@ public void onClose(int code, String reason) { || code == WebSocketCloseCode.GOING_AWAY; LineSenderException cause = new LineSenderException( "WebSocket closed by server: code=" + code + " reason=" + reason); - if (!orderly && nextWireSeq > 0) { + if (!orderly && dataFrameSentThisConnection) { if (recordHeadRejectionStrike(Math.max(engine.ackedFsn(), highestOkFsn) + 1L)) { haltOnPoisonedFrame("ws-close[" + code + ' ' + WebSocketCloseCode.describe(code) + "]: " + reason, @@ -2726,17 +2750,21 @@ private void handleServerRejection(long wireSeq) { // value is only used to attribute an FSN to the error report -- // a rejection never advances the watermark. long highestSent = nextWireSeq - 1L; - if (highestSent < 0L) { - // Pre-send rejection: server emitted an error frame before - // we sent anything on this connection (typical after a - // fresh swapClient — auth failure, server-initiated halt, - // etc.). The server-named wireSeq does not correspond to - // any frame we sent, so clamping it to 0 and acknowledging - // fsnAtZero would silently advance ackedFsn past a real - // unsent batch (fsnAtZero == ackedFsn + 1 right after a - // swap). Skip the watermark advance entirely; still surface - // the error so the user's handler sees it and HALT errors - // remain producer-observable. + if (!dataFrameSentThisConnection) { + // Pre-send rejection: the server emitted an error frame before we + // sent any DATA frame on this connection (typical after a fresh + // swapClient -- auth failure, server-initiated halt, or a rejection + // of the dictionary catch-up itself). nextWireSeq may be > 0 here + // because the catch-up consumed wire sequences, so this keys off + // dataFrameSentThisConnection, not highestSent >= 0 -- otherwise a + // transient NACK of a catch-up frame would take the post-send + // poison-strike path and could escalate a transient outage to a + // terminal. The server-named wireSeq does not correspond to any + // data frame we sent, so clamping it to 0 and acknowledging + // fsnAtZero would silently advance ackedFsn past a real unsent + // batch. Skip the watermark advance entirely; still surface the + // error so the user's handler sees it and HALT errors remain + // producer-observable. handlePreSendRejection(wireSeq, status, category, policy); return; } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java index 143686b4..81fd8676 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java @@ -227,6 +227,139 @@ public void testTornDictionaryFailsCleanlyInsteadOfCorrupting() throws Exception }); } + @Test + public void testUnopenablePersistedDictStillGuardsAgainstReplayingDeltaFrames() throws Exception { + // C1 regression: when a recovered disk slot's persisted dictionary cannot be + // OPENED (fd exhaustion, a read-only remount, ENOSPC -- simulated here by a + // .symbol-dict that is a DIRECTORY, so both openRW and openCleanRW fail), + // CursorSendEngine.isDeltaDictEnabled() returns false. The recorded frames + // are still DELTA frames, and replaying them against a fresh + // empty-dictionary server would null-pad the missing ids and SILENTLY + // corrupt the table. The torn-dictionary guard must fire regardless of + // deltaDictEnabled -- pre-fix it was gated on that very flag, so the + // corrupting frame sailed through unguarded. Unlike + // testTornDictionaryFailsCleanlyInsteadOfCorrupting (dict present but empty, + // deltaDictEnabled=true), here the dict is UNOPENABLE (deltaDictEnabled=false). + assertMemoryLeak(() -> { + // Phase 1: each row introduces a new symbol => frame i carries deltaStart=i. + try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) { + int port = silent.getPort(); + silent.start(); + Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS)); + String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + + ";close_flush_timeout_millis=0;"; + try (Sender s1 = Sender.fromConfig(cfg)) { + for (int i = 0; i < 6; i++) { + s1.table("m").symbol("s", "sym-" + i).longColumn("v", i).atNow(); + s1.flush(); + } + } + } + + // Make the persisted dictionary UNOPENABLE: replace the .symbol-dict file + // with a directory of the same name so PersistedSymbolDict.open() returns + // null (both openRW and openCleanRW fail) and the engine reports + // deltaDictEnabled=false. Stamp the watermark at FSN 2 so replay starts + // at FSN 3 -- a frame whose delta starts at id 3, with ids 0..2 living + // only in the now-unreadable dictionary. + java.nio.file.Path slot = Paths.get(sfDir, "default"); + java.nio.file.Path dict = slot.resolve(".symbol-dict"); + java.nio.file.Files.delete(dict); + java.nio.file.Files.createDirectory(dict); + writeAckWatermark(slot.resolve(".ack-watermark"), 2); + + // Phase 2: recover against a fresh counting server. The guard must fire + // (frame deltaStart 3 > recovered dictionary size 0) and fail terminally + // rather than send a gapped frame that would corrupt the table. + CountingHandler handler = new CountingHandler(); + try (TestWebSocketServer good = new TestWebSocketServer(handler)) { + int port = good.getPort(); + good.start(); + Assert.assertTrue(good.awaitStart(5, TimeUnit.SECONDS)); + + String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";"; + LineSenderException terminal = null; + Sender s2 = Sender.fromConfig(cfg); + try { + long deadline = System.currentTimeMillis() + 10_000; + while (System.currentTimeMillis() < deadline && terminal == null) { + try { + s2.flush(); + Thread.sleep(20); + } catch (LineSenderException e) { + terminal = e; + } + } + } finally { + try { + s2.close(); + } catch (LineSenderException e) { + if (terminal == null) { + terminal = e; + } + } + } + Assert.assertEquals("no delta frame may be replayed when the persisted dictionary is unopenable", + 0, handler.frames.get()); + Assert.assertNotNull("an unopenable dictionary must surface a terminal error", terminal); + Assert.assertTrue(terminal.getMessage(), + terminal.getMessage().contains("symbol dictionary is incomplete")); + } + }); + } + + @Test + public void testCommitMessageDoesNotShipUnpersistedLeakedSymbol() throws Exception { + // C3 regression: sendCommitMessage does NOT write-ahead-persist the + // dictionary, so its frame must carry NO new symbol. A symbol left in the + // batch by a cancelled row -- cancelRow rolls back neither + // currentBatchMaxSymbolId nor the global-dictionary registration -- must not + // ride out on the commit frame: doing so puts an id on the wire that a + // recovered slot cannot rebuild from .symbol-dict, diverging the producer + // dictionary from the surviving frames and silently misattributing reused + // ids after a crash. The commit's delta must be bounded by sentMaxSymbolId + // (empty here), not currentBatchMaxSymbolId. Memory mode suffices to observe + // the wire behaviour; close() drains every frame to the server first. + assertMemoryLeak(() -> { + DictReconstructingHandler handler = new DictReconstructingHandler(); + try (TestWebSocketServer server = new TestWebSocketServer(handler)) { + int port = server.getPort(); + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + + // Transactional + autoFlushRows=1: every completed row auto-flushes + // as a DEFERRED batch (setting hasDeferredMessages); an explicit + // flush() then emits the commit message. + Sender sender = Sender.builder("ws::addr=localhost:" + port + ";") + .transactional(true) + .autoFlushRows(1) + .build(); + try { + // Row 1 registers "a"@0 and auto-flushes it deferred. + sender.table("m").symbol("s", "a").longColumn("v", 1L).atNow(); + // Register "b"@1 on a row that is then cancelled: "b" stays in + // the global dictionary and currentBatchMaxSymbolId advances to + // 1, but nothing persists or sends it. + sender.table("m").symbol("s", "b"); + sender.cancelRow(); + // Commit the deferred batch. The commit frame must carry an + // EMPTY delta -- NOT "b"@1. + sender.flush(); + } finally { + sender.close(); // drains every frame (incl. the commit) to the server + } + + // The server's reconstructed dictionary must hold ONLY "a". Pre-fix + // the commit shipped "b"@1, so the server saw a second symbol. + List dict = handler.dictSnapshot(); + Assert.assertEquals("commit frame must not ship the cancelled row's leaked symbol " + + "(recovery would then diverge from the persisted dictionary): " + dict, + 1, dict.size()); + Assert.assertEquals("a", dict.get(0)); + } + }); + } + @Test public void testFailedPublishDoesNotDuplicatePersistedSymbols() throws Exception { // Regression: persistNewSymbolsBeforePublish is a write-ahead -- it runs diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopPoisonFrameTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopPoisonFrameTest.java index 34ae5518..6a3d8079 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopPoisonFrameTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopPoisonFrameTest.java @@ -244,6 +244,41 @@ public void testNonOrderlyClosePoisonKeysOnOkLevelHeadOfLine() throws Exception }); } + @Test + public void testNonOrderlyCloseAfterOnlyCatchUpDoesNotStrike() throws Exception { + // C2 regression: the dictionary catch-up advances nextWireSeq WITHOUT + // sending a data frame. A non-orderly close in that window -- a flapping + // LB/middlebox that completes the upgrade, accepts the catch-up, then drops + // before the first replay frame -- must be strike-EXEMPT. Keying the + // poison-strike gate off nextWireSeq > 0 (rather than + // dataFrameSentThisConnection) charges a strike on a frame that was never + // sent; MAX_REJECTIONS such closes then escalate a TRANSIENT outage to a + // PROTOCOL_VIOLATION terminal, hard-failing the producer and quarantining an + // orphan drainer -- exactly what store-and-forward's retry-forever contract + // forbids. Mirror image of testNonOrderlyClosePoisonKeysOnOkLevelHeadOfLine, + // which sends a real data frame (setSentCount) and DOES escalate. + TestUtils.assertMemoryLeak(() -> { + List clients = new ArrayList<>(); + try (CursorSendEngine engine = newEngine()) { + appendFrames(engine, 2); + CursorWebSocketSendLoop loop = newDurableLoop(engine, clients); + for (int i = 0; i < MAX_REJECTIONS + 2; i++) { + // Model the catch-up: nextWireSeq advanced, but NO data frame + // sent. swapClient resets both on every recycle, so re-apply + // before each close. Pre-fix, each of these lands a strike on the + // never-sent head frame and the loop terminals by now. + setCatchUpWireSeqOnly(loop, 2); + deliverNonOrderlyClose(loop); + } + // No strike was ever charged, so nothing escalated: the loop stays + // retriable and the producer-facing error latch is clear. + loop.checkError(); + } finally { + closeAll(clients); + } + }); + } + @Test public void testNackRecycleIsPacedAgainstHealthyServer() throws Exception { // A reachable, healthy server that NACKs the head frame (RETRIABLE) @@ -969,6 +1004,22 @@ private static void setSentCount(CursorWebSocketSendLoop loop, long count) throw Field f = CursorWebSocketSendLoop.class.getDeclaredField("nextWireSeq"); f.setAccessible(true); f.setLong(loop, count); + // A non-zero sent count models DATA frames sent on this connection. The + // poison-strike / pre-send gates key off dataFrameSentThisConnection, not + // nextWireSeq (the dictionary catch-up advances nextWireSeq without sending + // a data frame), so keep the two in sync for these white-box scenarios. + Field d = CursorWebSocketSendLoop.class.getDeclaredField("dataFrameSentThisConnection"); + d.setAccessible(true); + d.setBoolean(loop, count > 0); + } + + // Sets ONLY nextWireSeq -- deliberately NOT dataFrameSentThisConnection -- to + // model the dictionary catch-up having advanced the wire sequence with no data + // frame sent yet. Contrast setSentCount, which sets both. + private static void setCatchUpWireSeqOnly(CursorWebSocketSendLoop loop, long count) throws Exception { + Field f = CursorWebSocketSendLoop.class.getDeclaredField("nextWireSeq"); + f.setAccessible(true); + f.setLong(loop, count); } private static long[] txns(long... v) { From fc8b4ba64dd0e7a52957891bcf44217e361f7774 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Fri, 10 Jul 2026 03:06:18 +0100 Subject: [PATCH 21/96] Adopt recovered dictionary into the send mirror On recovery the send loop copied the persisted dictionary's loaded-entries buffer into a fresh mirror allocation and left PersistedSymbolDict holding a second copy for the engine's lifetime -- roughly twice the dictionary size in native memory on a high-cardinality recovered slot, retained long after the one-time seed. The loop now adopts that buffer as its mirror backing via takeLoadedEntries(), which transfers ownership so the dictionary no longer retains or frees it. The producer's readLoadedSymbols() is the only other consumer and runs first (setCursorEngine seeds the producer before the loop is built; the drainer has no producer consumer), guarded by an assert. Add a recover-then-continue-ingest test. A file-mode sender writes symbols and crashes; a fresh sender recovers the slot and ingests a NEW symbol. It asserts the producer continues the dictionary from the recovered size instead of colliding at id 0, exercising seedGlobalDictionaryFromPersisted, which no prior test drove past recovery. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../sf/cursor/CursorWebSocketSendLoop.java | 12 +++- .../client/sf/cursor/PersistedSymbolDict.java | 37 ++++++++++-- .../qwp/client/DeltaDictRecoveryTest.java | 56 +++++++++++++++++++ 3 files changed, 97 insertions(+), 8 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java index 2772cdf8..f5f21207 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java @@ -543,8 +543,16 @@ public CursorWebSocketSendLoop(WebSocketClient client, CursorSendEngine engine, if (pd != null && pd.size() > 0) { int len = pd.loadedEntriesLen(); if (len > 0) { - ensureSentDictCapacity(len); - Unsafe.getUnsafe().copyMemory(pd.loadedEntriesAddr(), sentDictBytesAddr, len); + // Adopt the persisted dictionary's loaded-entries buffer as the + // mirror's initial backing instead of copying it and leaving the + // dictionary to retain a second copy for the engine's lifetime. + // The producer's readLoadedSymbols() -- the only other consumer -- + // runs earlier in setCursorEngine; the drainer path has no + // producer consumer. takeLoadedEntries() transfers ownership, so + // this loop's mirror lifecycle frees it (not pd.close()). The + // buffer's allocated size equals loadedEntriesLen. + sentDictBytesAddr = pd.takeLoadedEntries(); + sentDictBytesCapacity = len; sentDictBytesLen = len; // Set the count only alongside the bytes so sentDictCount can // never claim symbols the mirror does not hold. A recovered slot diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java index 15179e89..dbb3593c 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java @@ -94,14 +94,19 @@ public final class PersistedSymbolDict implements QuietCloseable { private static final int MAX_ENTRY_LEN = 1 << 20; private static final Logger LOG = LoggerFactory.getLogger(PersistedSymbolDict.class); private final int fd; - // In-memory copy of the entry region [len][utf8]... exactly as on disk, - // populated only when open() recovered existing entries (recovery / - // orphan-drain). Zero/empty for a freshly created file. Consumed once to - // seed the send loop's catch-up mirror and the producer's id map. - private final long loadedEntriesAddr; - private final int loadedEntriesLen; private long appendOffset; private boolean closed; + // In-memory copy of the entry region [len][utf8]... exactly as on disk, + // populated only when open() recovered existing entries (recovery / + // orphan-drain). Zero/empty for a freshly created file. Consumed once to seed + // the producer's id map (readLoadedSymbols) and then ADOPTED by the send loop's + // catch-up mirror (takeLoadedEntries), which takes ownership so this file no + // longer retains a second copy of the dictionary for the engine's lifetime. + private long loadedEntriesAddr; + private int loadedEntriesLen; + // True once takeLoadedEntries() handed loadedEntriesAddr to the send-loop + // mirror; guards a stale readLoadedSymbols() call (which must run first). + private boolean loadedEntriesTaken; private long scratchAddr; private int scratchCap; private int size; @@ -256,6 +261,7 @@ public int loadedEntriesLen() { * recovered. */ public ObjList readLoadedSymbols() { + assert !loadedEntriesTaken : "readLoadedSymbols() must run before takeLoadedEntries()"; ObjList out = new ObjList<>(Math.max(size, 1)); long p = loadedEntriesAddr; long limit = p + loadedEntriesLen; @@ -286,6 +292,25 @@ public int size() { return size; } + /** + * Transfers ownership of the loaded-entries buffer to the caller and returns + * its base address (0 when nothing was recovered). The send loop adopts it as + * the initial backing of its catch-up mirror rather than copying it, so this + * file stops retaining a second copy of the dictionary for the engine's + * lifetime. After this call {@link #close()} no longer frees the buffer -- the + * caller owns it. The producer's {@link #readLoadedSymbols()} is the only other + * consumer and MUST run first: {@code setCursorEngine} seeds the producer + * before the send loop is constructed, and the drainer path has no producer + * consumer. The buffer was allocated with {@link MemoryTag#NATIVE_DEFAULT}. + */ + public long takeLoadedEntries() { + long addr = loadedEntriesAddr; + loadedEntriesAddr = 0L; + loadedEntriesLen = 0; + loadedEntriesTaken = true; + return addr; + } + private static PersistedSymbolDict openExisting(String filePath, long fileLen) { int fd = Files.openRW(filePath); if (fd < 0) { diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java index 81fd8676..8174ed24 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java @@ -227,6 +227,62 @@ public void testTornDictionaryFailsCleanlyInsteadOfCorrupting() throws Exception }); } + @Test + public void testRecoveredSenderContinuesIngestingNewSymbols() throws Exception { + // M2 regression: seedGlobalDictionaryFromPersisted resumes the producer's + // dictionary and delta baseline from the persisted .symbol-dict, so a + // recovered sender that continues ingesting assigns the NEXT id, not a + // colliding low one. Without it the new symbol reuses a recovered id and the + // fresh server sees a redefinition -> silent misattribution. No prior test + // ingests on the recovered sender. Replay is from FSN 0 (no acks), so the + // recovered frames legitimately overlap the seeded dictionary -- this also + // pins that the redefinition guard does not false-positive on normal + // recovery. + assertMemoryLeak(() -> { + // Phase 1: ingest DISTINCT_SYMBOLS symbols, silent server, close-fast -> + // unacked frames + a full persisted dictionary. + try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) { + int port = silent.getPort(); + silent.start(); + Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS)); + String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + + ";sf_max_bytes=4096;close_flush_timeout_millis=0;"; + try (Sender s1 = Sender.fromConfig(cfg)) { + for (int i = 0; i < DISTINCT_SYMBOLS; i++) { + s1.table("m").symbol("s", "sym-" + i).longColumn("v", i).atNow(); + s1.flush(); + } + } + } + + // Phase 2: recover against a fresh server, then ingest a genuinely NEW + // symbol. The producer must continue at id DISTINCT_SYMBOLS. + DictReconstructingHandler handler = new DictReconstructingHandler(); + try (TestWebSocketServer good = new TestWebSocketServer(handler)) { + int port = good.getPort(); + good.start(); + Assert.assertTrue(good.awaitStart(5, TimeUnit.SECONDS)); + String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";"; + try (Sender s2 = Sender.fromConfig(cfg)) { + s2.table("m").symbol("s", "brand-new").longColumn("v", 99L).atNow(); + s2.flush(); + long deadline = System.currentTimeMillis() + 10_000; + while (System.currentTimeMillis() < deadline + && handler.maxDictSize() < DISTINCT_SYMBOLS + 1) { + Thread.sleep(20); + } + } + List dict = handler.dictSnapshot(); + Assert.assertEquals("recovered sender must continue the dictionary, not collide: " + dict, + DISTINCT_SYMBOLS + 1, dict.size()); + for (int i = 0; i < DISTINCT_SYMBOLS; i++) { + Assert.assertEquals("sym-" + i, dict.get(i)); + } + Assert.assertEquals("brand-new", dict.get(DISTINCT_SYMBOLS)); + } + }); + } + @Test public void testUnopenablePersistedDictStillGuardsAgainstReplayingDeltaFrames() throws Exception { // C1 regression: when a recovered disk slot's persisted dictionary cannot be From 8dd7723789615b679b5047ad74778c16567a264e Mon Sep 17 00:00:00 2001 From: glasstiger Date: Fri, 10 Jul 2026 09:54:44 +0100 Subject: [PATCH 22/96] Harden delta symbol-dict recovery and NACK gating Two fixes surfaced by review of the delta symbol-dictionary change. Recover oversized persisted symbols (C1). PersistedSymbolDict.openExisting rejected any entry larger than a fixed 1 MB ceiling as "oversized" and truncated the dictionary at that id, but the write path (appendSymbol/appendSymbols) caps nothing. A symbol value over 1 MB therefore persisted fine, yet a normal process-crash recovery dropped it and every higher id; the send loop's replay guard then fired a spurious "host crash / resend required" terminal -- a store-and-forward process-crash-durability violation on a reachable input (nothing caps symbol value length). Drop the fixed per-entry ceiling and bound entries only by the file length, which is the actual corruption guard and was already present. Keep the length in a long so a corrupt multi-gigabyte varint cannot wrap an int back under the check. The write and read paths now agree, so a legitimately persisted large symbol recovers. Guard the catch-up NACK pre-send gate (C2). handleServerRejection keys its pre-send branch off dataFrameSentThisConnection, because the symbol catch-up advances nextWireSeq without sending a data frame. That branch had no regression test: every existing NACK test sets both flags together, so reverting the gate to the old nextWireSeq-based predicate left the suite green. Add the server-NACK twin of testNonOrderlyCloseAfterOnlyCatchUpDoesNotStrike -- a WRITE_ERROR NACK of a catch-up frame must take the pre-send path (surface and recycle, no strike), not escalate a transient outage to a producer-fatal PROTOCOL_VIOLATION terminal. Both regression tests fail with their production line reverted and pass with it. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../client/sf/cursor/PersistedSymbolDict.java | 19 +++++--- ...ursorWebSocketSendLoopPoisonFrameTest.java | 36 +++++++++++++++ .../sf/cursor/PersistedSymbolDictTest.java | 45 +++++++++++++++++++ 3 files changed, 93 insertions(+), 7 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java index dbb3593c..ca769181 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java @@ -89,9 +89,6 @@ public final class PersistedSymbolDict implements QuietCloseable { static final int FILE_MAGIC = 0x31445953; // 'SYD1' little-endian static final int HEADER_SIZE = 8; static final byte VERSION = 1; - // Guards against a hostile/corrupt varint length driving a huge allocation - // or a runaway parse. Symbols are short; this is a generous ceiling. - private static final int MAX_ENTRY_LEN = 1 << 20; private static final Logger LOG = LoggerFactory.getLogger(PersistedSymbolDict.class); private final int fd; private long appendOffset; @@ -337,12 +334,20 @@ private static PersistedSymbolDict openExisting(String filePath, long fileLen) { if (vr == null) { break; // torn length varint } - int entryLen = (int) vr[0]; + long entryLen = vr[0]; int next = (int) vr[1]; - if (entryLen < 0 || entryLen > MAX_ENTRY_LEN || (long) next + entryLen > len) { - break; // torn/oversized entry -- self-healing tail + // The file length is the sole sanity bound: a length that runs past + // the file is a torn/incomplete trailing entry and stops the parse + // (self-healing tail). There is deliberately NO fixed per-entry + // ceiling -- the write path (appendSymbol/appendSymbols) applies none + // either, so a legitimately persisted large symbol must recover here, + // not be truncated and then trip the send loop's "resend required" + // guard on a normal process-crash recovery. entryLen stays a long so a + // corrupt multi-gigabyte length cannot wrap an int back under the check. + if ((long) next + entryLen > len) { + break; // torn/incomplete trailing entry -- self-healing tail } - pos = next + entryLen; + pos = next + (int) entryLen; count++; } int entriesLen = pos - HEADER_SIZE; diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopPoisonFrameTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopPoisonFrameTest.java index 6a3d8079..ab200e80 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopPoisonFrameTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopPoisonFrameTest.java @@ -279,6 +279,42 @@ public void testNonOrderlyCloseAfterOnlyCatchUpDoesNotStrike() throws Exception }); } + @Test + public void testNonOrderlyRejectionAfterOnlyCatchUpDoesNotStrike() throws Exception { + // C2 regression: the server-NACK twin of + // testNonOrderlyCloseAfterOnlyCatchUpDoesNotStrike. handleServerRejection's + // pre-send gate keys off dataFrameSentThisConnection, NOT nextWireSeq -- the + // dictionary catch-up advances nextWireSeq WITHOUT sending a data frame. A + // server NACK of a catch-up frame (nextWireSeq>0, no data frame sent) must + // take the pre-send path (surface + recycle, no strike), not the post-send + // poison-strike path. Pre-fix (gate on highestSent >= 0, i.e. nextWireSeq>0) + // each catch-up NACK strikes the never-sent head frame; MAX_REJECTIONS such + // strikes escalate a TRANSIENT outage to a producer-fatal PROTOCOL_VIOLATION + // terminal -- exactly what store-and-forward's retry-forever contract forbids. + // WRITE_ERROR (RETRIABLE) is the discriminating category: it DOES accrue + // strikes on the post-send path (see testNackRecycleIsPacedAgainstHealthyServer), + // so a regressed gate escalates here and checkError() throws. + TestUtils.assertMemoryLeak(() -> { + List clients = new ArrayList<>(); + try (CursorSendEngine engine = newEngine()) { + appendFrames(engine, 2); + CursorWebSocketSendLoop loop = newDurableLoop(engine, clients); + for (int i = 0; i < MAX_REJECTIONS + 2; i++) { + // Model the catch-up: nextWireSeq advanced, but NO data frame + // sent. The pre-send recycle (swapClient) resets nextWireSeq, so + // re-apply before each NACK (mirrors the onClose twin). + setCatchUpWireSeqOnly(loop, 2); + deliverRetriableNack(loop, 1, "disk full"); + } + // No strike was ever charged, so nothing escalated: the loop stays + // retriable and the producer-facing error latch is clear. + loop.checkError(); + } finally { + closeAll(clients); + } + }); + } + @Test public void testNackRecycleIsPacedAgainstHealthyServer() throws Exception { // A reachable, healthy server that NACKs the head frame (RETRIABLE) diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java index df6bab21..d580f886 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java @@ -27,6 +27,7 @@ import io.questdb.client.cutlass.qwp.client.GlobalSymbolDictionary; import io.questdb.client.cutlass.qwp.client.sf.cursor.PersistedSymbolDict; import io.questdb.client.std.ObjList; +import io.questdb.client.test.tools.TestUtils; import org.junit.Assert; import org.junit.Test; @@ -198,6 +199,50 @@ public void testEmptySymbolRoundTrips() throws Exception { }); } + @Test + public void testLargeSymbolRoundTripsAcrossReopen() throws Exception { + // C1 regression: the write path caps nothing, so a symbol larger than the + // old fixed 1 MB read ceiling must still recover intact. Before the fix, + // appendSymbol wrote the oversized entry but openExisting rejected it as + // "oversized", truncated the dictionary at that id (dropping it and every + // higher id), and a normal process-crash recovery then hard-failed with a + // spurious "host crash / resend required" terminal -- defeating store-and- + // forward's process-crash durability for large symbols. The file length is + // now the only bound, so the write and read paths agree. + assertMemoryLeak(() -> { + Path dir = Files.createTempDirectory("qwp-symdict"); + try { + // Just over the old 1 << 20 (1 MB) ceiling. + String big = TestUtils.repeat("x", (1 << 20) + 17); + PersistedSymbolDict d = PersistedSymbolDict.open(dir.toString()); + try { + d.appendSymbol("before"); + d.appendSymbol(big); + d.appendSymbol("after"); + Assert.assertEquals(3, d.size()); + } finally { + d.close(); + } + + // Recovery must load ALL three; pre-fix the reopen truncated at the + // big entry and came back with size 1 (only "before" survived). + PersistedSymbolDict re = PersistedSymbolDict.open(dir.toString()); + try { + Assert.assertEquals("large entry must survive recovery, not be truncated", + 3, re.size()); + ObjList s = re.readLoadedSymbols(); + Assert.assertEquals("before", s.getQuick(0)); + Assert.assertEquals(big, s.getQuick(1)); + Assert.assertEquals("after", s.getQuick(2)); + } finally { + re.close(); + } + } finally { + rmDir(dir); + } + }); + } + @Test public void testRemoveOrphanDeletesFile() throws Exception { assertMemoryLeak(() -> { From 8853d8fd5fe922ce6b27b475b84e315dac9271e1 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Fri, 10 Jul 2026 10:01:30 +0100 Subject: [PATCH 23/96] Persist symbol delta from frame, skip re-encode M1: persistNewSymbolsBeforePublish re-encoded each new symbol into a scratch buffer even though beginMessage had just written the identical [len][utf8] bytes into the frame's symbol-dict delta section. The encoder now records that entry region's offset and length; the producer writes those bytes straight to the slot .symbol-dict via a new PersistedSymbolDict.appendRawEntries, so a high-cardinality batch no longer re-encodes what the wire frame already carries. The common path (durable size == the frame's delta start id) takes the direct copy; a retry after a failed publish, where the durable size has run ahead of the wire baseline, falls back to re-encoding just the remaining suffix. The persisted bytes are byte-identical either way. M2: document, on the sendDictCatchUp oversized-entry terminal, that in a heterogeneous or rolling-cap cluster a symbol accepted under a larger cap can hit this hard stop on failover to a smaller-cap node and will not self-recover if a later node advertises a larger cap. Behavior is unchanged -- the terminal keeps the homogeneous common case livelock-free; this only records the tradeoff and the two ways to relax it (an ingest-side symbol-size bound, or a settle budget across reconnects before latching). Adds testAppendRawEntriesMatchesAppendSymbols; the file-mode persist, recovery, and failed-publish suites cover both the direct-copy and re-encode branches end to end. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../qwp/client/QwpWebSocketEncoder.java | 26 +++++++++ .../qwp/client/QwpWebSocketSender.java | 45 ++++++++++----- .../sf/cursor/CursorWebSocketSendLoop.java | 9 +++ .../client/sf/cursor/PersistedSymbolDict.java | 24 ++++++++ .../sf/cursor/PersistedSymbolDictTest.java | 56 +++++++++++++++++++ 5 files changed, 146 insertions(+), 14 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java index ced1a1b5..8013330a 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java @@ -39,6 +39,14 @@ public class QwpWebSocketEncoder implements QuietCloseable { private final QwpColumnWriter columnWriter = new QwpColumnWriter(); private NativeBufferWriter buffer; + // Byte offsets, within the buffer, of the symbol-dict delta ENTRY region + // ([len][utf8]... only, without the two section varints) that beginMessage + // last wrote. Let the producer persist those bytes straight to the slot's + // .symbol-dict instead of re-encoding the same symbols (see + // QwpWebSocketSender.persistNewSymbolsBeforePublish). Valid until the next + // beginMessage; stored as offsets so they survive a buffer realloc. + private int deltaEntriesEnd; + private int deltaEntriesStart; // QWP ingress always advertises Gorilla timestamp encoding. The column // writer still emits a per-column encoding byte and falls back to raw // values when delta-of-delta overflows int32. @@ -75,10 +83,12 @@ public void beginMessage( payloadStart = buffer.getPosition(); buffer.putVarint(deltaStart); buffer.putVarint(deltaCount); + deltaEntriesStart = buffer.getPosition(); for (int id = deltaStart; id < deltaStart + deltaCount; id++) { String symbol = globalDict.getSymbol(id); buffer.putString(symbol); } + deltaEntriesEnd = buffer.getPosition(); columnWriter.setBuffer(buffer); } @@ -122,6 +132,22 @@ public QwpBufferWriter getBuffer() { return buffer; } + /** + * Byte length of the symbol-dict delta ENTRY region ({@code [len][utf8]...}, + * excluding the two section varints) that {@link #beginMessage} last wrote. + */ + public int getDeltaEntriesLen() { + return deltaEntriesEnd - deltaEntriesStart; + } + + /** + * Byte offset, within {@link #getBuffer()}, of the symbol-dict delta ENTRY + * region {@link #beginMessage} last wrote. + */ + public int getDeltaEntriesStart() { + return deltaEntriesStart; + } + public void setDeferCommit(boolean defer) { if (defer) { flags |= FLAG_DEFER_COMMIT; diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index 4a499300..942af99b 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -3681,24 +3681,41 @@ private void persistNewSymbolsBeforePublish() { if (pd == null) { return; } - // Persist [pd.size() .. currentBatchMaxSymbolId] as ONE write. + // Persist [pd.size() .. currentBatchMaxSymbolId] as ONE write, BEFORE the + // frame is published. // // Resume from the dictionary's own durable size, NOT sentMaxSymbolId+1: - // appendSymbols advances pd.size() only after a full write, whereas + // the persist advances pd.size() only after a full write, whereas // sentMaxSymbolId only advances after the WHOLE frame is published (via // advanceSentMaxSymbolId, after activeBuffer.write). If a prior persist - // threw (a short write -- disk full/quota), the frame was not published - // and sentMaxSymbolId stayed put, while the symbols before the failing - // one are already on disk. Keying the resume point off sentMaxSymbolId+1 - // would then re-append that persisted prefix on the retry, duplicating - // entries and corrupting the dense id->symbol mapping recovery relies on - // (position i must be symbol id i). pd.size() resumes exactly past what is - // already durable (the write overwrites any torn trailing bytes at - // appendOffset), so the write-ahead is idempotent under retry. In the - // happy path pd.size() equals sentMaxSymbolId+1, so the range -- and the - // behaviour -- are unchanged; the single positioned write just replaces - // the previous one-syscall-per-new-symbol loop. - pd.appendSymbols(globalSymbolDictionary, pd.size(), currentBatchMaxSymbolId); + // threw (short write -- disk full/quota) or the publish threw, the frame + // was not published and sentMaxSymbolId stayed put, while the symbols + // before the failure are already on disk. Keying the resume point off + // sentMaxSymbolId+1 would re-append that persisted prefix on the retry, + // duplicating entries and corrupting the dense id->symbol mapping recovery + // relies on (position i must be symbol id i). pd.size() resumes exactly + // past what is already durable, so the write-ahead is idempotent. + int from = pd.size(); + if (currentBatchMaxSymbolId < from) { + return; // nothing new to persist (warm batch, or an idempotent retry) + } + // Fast path: the frame the encoder just built already holds these symbols + // in its delta section as [len][utf8]... -- byte-identical to what + // PersistedSymbolDict stores. In the common case pd.size() equals the + // frame's delta start id (sentMaxSymbolId+1), so persist those bytes + // straight from the frame instead of re-encoding the symbols. After a + // failed publish the durable size has run ahead of the wire baseline, so + // the frame's delta covers MORE than remains to persist; then re-encode + // just the [from .. currentBatchMaxSymbolId] suffix. + if (from == sentMaxSymbolId + 1) { + QwpBufferWriter buffer = encoder.getBuffer(); + pd.appendRawEntries( + buffer.getBufferPtr() + encoder.getDeltaEntriesStart(), + encoder.getDeltaEntriesLen(), + currentBatchMaxSymbolId - from + 1); + } else { + pd.appendSymbols(globalSymbolDictionary, from, currentBatchMaxSymbolId); + } } private void resetSymbolDictStateForNewConnection() { diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java index f5f21207..6b9eee60 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java @@ -2192,6 +2192,15 @@ private int sendDictCatchUp() { // Latch a terminal (the data must be resent after the cap is // raised) rather than calling fail() -- which, from inside the // catch-up, would re-enter connectLoop (see CatchUpSendException). + // + // Tradeoff (heterogeneous / rolling-cap clusters): a symbol + // accepted under a larger/absent cap can hit this on failover to a + // smaller-cap node, and the hard terminal does NOT self-recover if + // a later node advertises a larger cap -- the producer must be + // resumed after the data is resent (or the cap raised). Bounding + // symbol size at ingest, or a settle budget across reconnects + // before latching, would relax this, but both are larger changes; + // the terminal keeps the homogeneous common case livelock-free. LineSenderException err = new LineSenderException( "symbol dictionary entry too large for the server batch cap during catch-up [" + "entryBytes=" + entryBytes + ", budget=" + budget + ']'); diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java index ca769181..55c04ba7 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java @@ -148,6 +148,30 @@ public static void removeOrphan(String slotDir) { Files.remove(slotDir + "/" + FILE_NAME); } + /** + * Appends {@code count} already-encoded entries -- {@code [len varint][utf8]...}, + * the exact layout {@link #appendSymbols} produces -- verbatim from {@code addr} + * in a single write. The producer uses this when it already holds those bytes + * (the symbol-dict delta section the frame encoder just wrote), so it does not + * re-encode the same symbols. Writes {@code len} bytes and advances {@code size} + * by {@code count}. Same durability/idempotency contract as {@link #appendSymbols}: + * no fsync, and a short write throws WITHOUT advancing {@code size}/{@code + * appendOffset}, so a retry keyed off {@link #size()} re-persists the same range + * at the same offset. No-op when the range is empty or the dictionary is closed. + */ + public void appendRawEntries(long addr, int len, int count) { + if (closed || count <= 0 || len <= 0) { + return; + } + long written = Files.write(fd, addr, len, appendOffset); + if (written != len) { + throw new IllegalStateException("short write to " + FILE_NAME + + " [expected=" + len + ", actual=" + written + ']'); + } + appendOffset += len; + size += count; + } + /** * Appends one symbol, extending the on-disk dictionary. The caller appends a * frame's new symbols BEFORE publishing that frame, so the write ordering diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java index d580f886..67fd7438 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java @@ -91,6 +91,62 @@ public void testAppendPersistsAcrossReopen() throws Exception { }); } + @Test + public void testAppendRawEntriesMatchesAppendSymbols() throws Exception { + // M1: the producer persists the frame's already-encoded delta bytes via + // appendRawEntries instead of re-encoding the symbols. Those bytes are the + // same [len][utf8]... layout appendSymbols writes, so both must produce an + // identical, recoverable dictionary. Encode a range with appendSymbols, + // reopen to grab its on-disk entry bytes, replay them through + // appendRawEntries into a fresh dict, and assert the recovered symbols + // match -- including an empty entry mid-range. + assertMemoryLeak(() -> { + Path src = Files.createTempDirectory("qwp-symdict-src"); + Path dst = Files.createTempDirectory("qwp-symdict-dst"); + try { + GlobalSymbolDictionary dict = new GlobalSymbolDictionary(); + dict.getOrAddSymbol("AAPL"); // id 0 + dict.getOrAddSymbol(""); // id 1 -- empty entry mid-range + dict.getOrAddSymbol("MSFT"); // id 2 + + PersistedSymbolDict encoded = PersistedSymbolDict.open(src.toString()); + encoded.appendSymbols(dict, 0, 2); + encoded.close(); + + // Reopen to obtain the on-disk entry region [len][utf8]... verbatim, + // then replay it byte-for-byte into a fresh dict via appendRawEntries. + PersistedSymbolDict reopened = PersistedSymbolDict.open(src.toString()); + try { + PersistedSymbolDict raw = PersistedSymbolDict.open(dst.toString()); + try { + raw.appendRawEntries(reopened.loadedEntriesAddr(), + reopened.loadedEntriesLen(), reopened.size()); + Assert.assertEquals(3, raw.size()); + } finally { + raw.close(); + } + } finally { + reopened.close(); + } + + // The raw-appended dict must recover the same dense symbols. + PersistedSymbolDict recovered = PersistedSymbolDict.open(dst.toString()); + try { + Assert.assertEquals(3, recovered.size()); + ObjList symbols = recovered.readLoadedSymbols(); + Assert.assertEquals("AAPL", symbols.getQuick(0)); + Assert.assertEquals("", symbols.getQuick(1)); + Assert.assertEquals("MSFT", symbols.getQuick(2)); + } finally { + recovered.close(); + } + } finally { + rmDir(src); + rmDir(dst); + } + }); + } + @Test public void testAppendSymbolsBatchWritesDenseRange() throws Exception { // appendSymbols persists a whole id range in one write (the hot-path From 10a125c05d169dcb87dc38eeeab73d28621add70 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Fri, 10 Jul 2026 11:10:00 +0100 Subject: [PATCH 24/96] Guard catch-up frame overflow, add biting tests Two defensive spots in the delta symbol-dict path had no biting test, and one guard was incomplete (M3). Complete the catch-up frame-size guard (M3b). The int-overflow hardening covered ensureSentDictCapacity (the mirror growth) but not the single catch-up frame: with no server cap, frameLen = HEADER + varints + symbolsLen is an int and would wrap negative as symbolsLen approaches the mirror ceiling, feeding a bad Unsafe.malloc. sendDictCatchUp's budget already keeps each chunk under that bound, so this is unreachable at real cardinality -- but the guard must be local so a future caller cannot overflow it silently. Compute the size in long and fail loud (CatchUpSendException) before the malloc, matching the mirror-side guard. Cover accumulateSentDict's partial-overlap tail (M3a). A delta that straddles the mirror tip (deltaStart < sentDictCount < deltaEnd) must copy only the new tail, not drop the whole frame. The monotonic producer never emits a straddling delta in steady state, so reverting to the pre-fix drop-whole-frame guard passed every test; it is reachable on a torn-dict replay (mirror seeded smaller than a frame's coverage), where dropping the tail would leave the reconnect catch-up incomplete and shift server ids. Add a white-box test that drives the straddle directly. Both new tests fail with their production line reverted (the mirror stays at 1 id; the frame guard falls through to a negative malloc that throws IllegalArgumentException, not the clean terminal) and pass with it. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../sf/cursor/CursorWebSocketSendLoop.java | 18 ++- ...WebSocketSendLoopCatchUpAlignmentTest.java | 132 ++++++++++++++++++ 2 files changed, 148 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java index 6b9eee60..6d8ed64b 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java @@ -2234,10 +2234,24 @@ private int sendDictCatchUp() { * the caller turns it into a single, non-re-entrant reconnect. */ private void sendCatchUpChunk(int deltaStart, int deltaCount, long symbolsAddr, int symbolsLen) { - int payloadLen = NativeBufferWriter.varintSize(deltaStart) + // Compute the frame size in long and fail loud if it would overflow the int + // size math into a negative Unsafe.malloc. sendDictCatchUp already caps each + // chunk's symbol bytes under the budget, so this is unreachable at real + // cardinality -- but the mirror-side ensureSentDictCapacity guards the same + // math, and a future caller must not be able to overflow this one silently. + long payloadLenL = (long) NativeBufferWriter.varintSize(deltaStart) + NativeBufferWriter.varintSize(deltaCount) + symbolsLen; - int frameLen = QwpConstants.HEADER_SIZE + payloadLen; + long frameLenL = QwpConstants.HEADER_SIZE + payloadLenL; + if (frameLenL > MAX_SENT_DICT_BYTES) { + LineSenderException err = new LineSenderException( + "symbol dictionary catch-up frame exceeds the maximum size [" + + "frameLen=" + frameLenL + ", max=" + MAX_SENT_DICT_BYTES + ']'); + recordFatal(err); + throw new CatchUpSendException(err); + } + int payloadLen = (int) payloadLenL; + int frameLen = (int) frameLenL; long frame = Unsafe.malloc(frameLen, MemoryTag.NATIVE_DEFAULT); try { Unsafe.getUnsafe().putByte(frame, (byte) 'Q'); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java index 9b2195d9..79640f25 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java @@ -43,8 +43,12 @@ import java.lang.reflect.Method; import java.nio.charset.StandardCharsets; import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; /** @@ -211,6 +215,76 @@ public void testTransientCatchUpSendFailureIsRetriableNotTerminal() throws Excep }); } + @Test + public void testAccumulateSentDictPartialOverlapExtendsMirror() throws Exception { + // M3: accumulateSentDict must handle a delta that STRADDLES the mirror tip + // (deltaStart < sentDictCount < deltaStart+deltaCount) by copying only the + // new tail, not dropping the whole frame. The monotonic producer never emits + // a straddling delta in steady state (so the pre-fix drop-whole-frame guard + // passed every test), but a torn-dict replay can seed the mirror smaller than + // a frame's coverage. Seed the mirror with 1 symbol, feed a [0..2] delta, and + // assert the mirror extends to all 3 -- pre-fix it stayed at 1, leaving the + // reconnect catch-up incomplete and shifting server-side ids. + TestUtils.assertMemoryLeak(() -> { + CatchUpCapturingClient client = new CatchUpCapturingClient(0); + try (CursorSendEngine engine = newEngine()) { + CursorWebSocketSendLoop loop = newLoop(engine, client); + try { + seedMirror(loop, "aa"); // sentDictCount = 1, mirror holds "aa" + int[] frameLen = new int[1]; + long frame = buildDeltaFrame(0, new String[]{"aa", "bb", "cc"}, frameLen); + try { + Method m = CursorWebSocketSendLoop.class.getDeclaredMethod( + "accumulateSentDict", long.class, int.class, int.class); + m.setAccessible(true); + m.invoke(loop, frame, frameLen[0], 0); + } finally { + Unsafe.free(frame, frameLen[0], MemoryTag.NATIVE_DEFAULT); + } + assertEquals("straddling delta must extend the mirror to all 3 ids", + 3, readInt(loop, "sentDictCount")); + assertEquals("mirror must hold the two new tail symbols after the " + + "already-held prefix, gap-free", + Arrays.asList("aa", "bb", "cc"), readMirrorSymbols(loop)); + } finally { + loop.close(); + } + } + }); + } + + @Test + public void testCatchUpChunkFrameSizeOverflowFailsLoud() throws Exception { + // M3: sendDictCatchUp caps each chunk under the budget, so the single-frame + // catch-up path cannot overflow its int frameLen at any real cardinality. The + // guard must still be LOCAL -- a future caller must not be able to feed a + // wrapped-negative frameLen to Unsafe.malloc. An oversized symbolsLen must + // fail loud (CatchUpSendException) BEFORE the malloc; the guard fires before + // symbolsAddr is read, so a dummy address is fine. + TestUtils.assertMemoryLeak(() -> { + CatchUpCapturingClient client = new CatchUpCapturingClient(0); + try (CursorSendEngine engine = newEngine()) { + CursorWebSocketSendLoop loop = newLoop(engine, client); + try { + Method m = CursorWebSocketSendLoop.class.getDeclaredMethod( + "sendCatchUpChunk", int.class, int.class, long.class, int.class); + m.setAccessible(true); + // symbolsLen past the mirror ceiling: HEADER + varints + symbolsLen + // overflows an int, so the guard must reject it before malloc. + m.invoke(loop, 0, 1, 0L, Integer.MAX_VALUE - 4); + fail("an overflowing catch-up frame size must fail loud, not malloc negative"); + } catch (InvocationTargetException e) { + assertEquals("overflow must surface as CatchUpSendException", + "CatchUpSendException", e.getCause().getClass().getSimpleName()); + assertTrue("message must name the frame-size guard: " + e.getCause().getMessage(), + e.getCause().getMessage().contains("catch-up frame exceeds the maximum size")); + } finally { + loop.close(); + } + } + }); + } + private static void appendFrames(CursorSendEngine engine, int count) { long buf = Unsafe.malloc(16, MemoryTag.NATIVE_DEFAULT); try { @@ -226,6 +300,64 @@ private static void appendFrames(CursorSendEngine engine, int count) { } } + // Builds a QWP delta frame [12-byte header][deltaStart varint][deltaCount + // varint][ [len varint][utf8] ... ] for the given symbols. accumulateSentDict + // skips the header, so its content is irrelevant; the caller frees the frame. + private static long buildDeltaFrame(int deltaStart, String[] symbols, int[] outLen) { + int deltaCount = symbols.length; + int size = 12 + varintSize(deltaStart) + varintSize(deltaCount); + for (String s : symbols) { + size += varintSize(s.getBytes(StandardCharsets.UTF_8).length) + + s.getBytes(StandardCharsets.UTF_8).length; + } + long addr = Unsafe.malloc(size, MemoryTag.NATIVE_DEFAULT); + long p = writeVarint(addr + 12, deltaStart); + p = writeVarint(p, deltaCount); + for (String s : symbols) { + byte[] b = s.getBytes(StandardCharsets.UTF_8); + p = writeVarint(p, b.length); + for (byte x : b) { + Unsafe.getUnsafe().putByte(p++, x); + } + } + outLen[0] = size; + return addr; + } + + private static int readInt(CursorWebSocketSendLoop loop, String name) throws Exception { + Field f = CursorWebSocketSendLoop.class.getDeclaredField(name); + f.setAccessible(true); + return f.getInt(loop); + } + + // Parses the loop's native sent-dictionary mirror ([len varint][utf8]...) back + // into the symbol strings a reconnect catch-up would re-register. + private static List readMirrorSymbols(CursorWebSocketSendLoop loop) throws Exception { + long addr = readLong(loop, "sentDictBytesAddr"); + int len = readInt(loop, "sentDictBytesLen"); + List out = new ArrayList<>(); + long p = addr; + long limit = addr + len; + while (p < limit) { + long l = 0; + int shift = 0; + while (p < limit) { + byte b = Unsafe.getUnsafe().getByte(p++); + l |= (long) (b & 0x7F) << shift; + if ((b & 0x80) == 0) { + break; + } + shift += 7; + } + byte[] bytes = new byte[(int) l]; + for (int i = 0; i < l; i++) { + bytes[i] = Unsafe.getUnsafe().getByte(p++); + } + out.add(new String(bytes, StandardCharsets.UTF_8)); + } + return out; + } + // Delivers a 0-table STATUS_OK for {@code wireSeq} into the loop's response // handler, mimicking the server acking a catch-up frame (which carries no tables). private static void deliverOk(CursorWebSocketSendLoop loop, long wireSeq) throws Exception { From fa50e81f0288077d2e0040314fb77a0ce5a780b7 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Fri, 10 Jul 2026 12:26:08 +0100 Subject: [PATCH 25/96] Fix orphan-drain losing symbol-dict mirror seed The send loop seeded its reconnect catch-up mirror by calling PersistedSymbolDict.takeLoadedEntries(), a one-shot ownership transfer that zeroed the dictionary's loaded-entries buffer. The foreground sender builds a single loop, so that was safe. But BackgroundDrainer builds a fresh send loop per wire session against the SAME, persistent engine when a durable-ack capability gap forces a mid-drain recycle. The second loop then found an empty loaded-entries buffer, seeded an empty mirror (sentDictCount = 0), sent no reconnect catch-up, and the first replayed delta frame (deltaStart > 0) tripped the torn-dict guard -- falsely quarantining a healthy slot with a bogus "resend required" terminal and defeating the capability-gap settle-retry for delta slots. The send loop now COPIES the loaded entries into its own mirror and leaves the dictionary owning its buffer for the engine's lifetime, so every recycled loop re-seeds. PersistedSymbolDict.close() frees the dictionary's copy; each loop frees its own copy on exit. Peak footprint is unchanged (the drainer closes loop N before building loop N+1). Removes the now-unused takeLoadedEntries() and loadedEntriesTaken, and adds a regression test that builds two loops against one recovered engine and asserts the second re-seeds (pre-fix its mirror was empty). Co-Authored-By: Claude Opus 4.8 (1M context) --- .../sf/cursor/CursorWebSocketSendLoop.java | 24 ++++--- .../client/sf/cursor/PersistedSymbolDict.java | 33 ++------- ...CursorWebSocketSendLoopMirrorLeakTest.java | 70 +++++++++++++++++++ 3 files changed, 91 insertions(+), 36 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java index 6d8ed64b..d910a6ca 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java @@ -543,15 +543,21 @@ public CursorWebSocketSendLoop(WebSocketClient client, CursorSendEngine engine, if (pd != null && pd.size() > 0) { int len = pd.loadedEntriesLen(); if (len > 0) { - // Adopt the persisted dictionary's loaded-entries buffer as the - // mirror's initial backing instead of copying it and leaving the - // dictionary to retain a second copy for the engine's lifetime. - // The producer's readLoadedSymbols() -- the only other consumer -- - // runs earlier in setCursorEngine; the drainer path has no - // producer consumer. takeLoadedEntries() transfers ownership, so - // this loop's mirror lifecycle frees it (not pd.close()). The - // buffer's allocated size equals loadedEntriesLen. - sentDictBytesAddr = pd.takeLoadedEntries(); + // COPY the persisted dictionary's loaded-entries buffer into this + // loop's own mirror rather than taking ownership of it. The engine + // (and its PersistedSymbolDict) OUTLIVES this loop on the orphan + // drainer path: BackgroundDrainer builds a fresh send loop per wire + // session against the same engine on a durable-ack capability-gap + // recycle. A one-shot ownership transfer would leave every loop + // after the first with an EMPTY mirror -- it would then send no + // reconnect catch-up, and the first replayed delta frame + // (deltaStart > 0) would trip the torn-dict guard, falsely + // quarantining a healthy slot. Copying keeps the dictionary's + // loaded entries intact for the engine's lifetime so every + // recycled loop re-seeds; pd.close() (at engine close) frees the + // dictionary's copy, this loop frees its own copy on exit. + sentDictBytesAddr = Unsafe.malloc(len, MemoryTag.NATIVE_DEFAULT); + Unsafe.getUnsafe().copyMemory(pd.loadedEntriesAddr(), sentDictBytesAddr, len); sentDictBytesCapacity = len; sentDictBytesLen = len; // Set the count only alongside the bytes so sentDictCount can diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java index 55c04ba7..bf5fc39f 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java @@ -95,15 +95,14 @@ public final class PersistedSymbolDict implements QuietCloseable { private boolean closed; // In-memory copy of the entry region [len][utf8]... exactly as on disk, // populated only when open() recovered existing entries (recovery / - // orphan-drain). Zero/empty for a freshly created file. Consumed once to seed - // the producer's id map (readLoadedSymbols) and then ADOPTED by the send loop's - // catch-up mirror (takeLoadedEntries), which takes ownership so this file no - // longer retains a second copy of the dictionary for the engine's lifetime. + // orphan-drain). Zero/empty for a freshly created file. READ (not consumed) to + // seed the producer's id map (readLoadedSymbols) and to seed the send loop's + // catch-up mirror, which COPIES it. This file retains ownership for the engine's + // lifetime -- the orphan drainer builds a fresh send loop per wire session + // against the same engine, and each must re-seed its mirror -- and frees this + // buffer in close(). private long loadedEntriesAddr; private int loadedEntriesLen; - // True once takeLoadedEntries() handed loadedEntriesAddr to the send-loop - // mirror; guards a stale readLoadedSymbols() call (which must run first). - private boolean loadedEntriesTaken; private long scratchAddr; private int scratchCap; private int size; @@ -282,7 +281,6 @@ public int loadedEntriesLen() { * recovered. */ public ObjList readLoadedSymbols() { - assert !loadedEntriesTaken : "readLoadedSymbols() must run before takeLoadedEntries()"; ObjList out = new ObjList<>(Math.max(size, 1)); long p = loadedEntriesAddr; long limit = p + loadedEntriesLen; @@ -313,25 +311,6 @@ public int size() { return size; } - /** - * Transfers ownership of the loaded-entries buffer to the caller and returns - * its base address (0 when nothing was recovered). The send loop adopts it as - * the initial backing of its catch-up mirror rather than copying it, so this - * file stops retaining a second copy of the dictionary for the engine's - * lifetime. After this call {@link #close()} no longer frees the buffer -- the - * caller owns it. The producer's {@link #readLoadedSymbols()} is the only other - * consumer and MUST run first: {@code setCursorEngine} seeds the producer - * before the send loop is constructed, and the drainer path has no producer - * consumer. The buffer was allocated with {@link MemoryTag#NATIVE_DEFAULT}. - */ - public long takeLoadedEntries() { - long addr = loadedEntriesAddr; - loadedEntriesAddr = 0L; - loadedEntriesLen = 0; - loadedEntriesTaken = true; - return addr; - } - private static PersistedSymbolDict openExisting(String filePath, long fileLen) { int fd = Files.openRW(filePath); if (fd < 0) { diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopMirrorLeakTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopMirrorLeakTest.java index 19626ba4..7869882d 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopMirrorLeakTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopMirrorLeakTest.java @@ -33,6 +33,7 @@ import org.junit.Test; import java.io.IOException; +import java.lang.reflect.Field; import java.nio.file.Files; import java.nio.file.Path; import java.util.Comparator; @@ -96,6 +97,69 @@ public void testSeededMirrorFreedWhenLoopClosedWithoutStart() throws Exception { } } + @Test + public void testRecycledLoopReSeedsMirrorFromPersistedDict() throws Exception { + // C1 regression: the orphan drainer (BackgroundDrainer) builds a NEW + // CursorWebSocketSendLoop per wire session against the SAME, persistent + // engine when a durable-ack capability gap forces a mid-drain recycle. The + // recovery mirror seed must survive that recycle. If the first loop CONSUMED + // the persisted dictionary's loaded entries (a one-shot ownership transfer), + // the second loop seeds an EMPTY mirror (sentDictCount = 0), sends no + // reconnect catch-up, and the first replayed delta frame (deltaStart > 0) + // trips the torn-dict guard -- falsely quarantining a healthy slot with a + // bogus "resend required" terminal. Copying the entries (leaving the + // dictionary intact for the engine's lifetime) lets every recycled loop + // re-seed. Pre-fix, loop2's sentDictCount is 0 and this assertion fails. + Path sfDir = Files.createTempDirectory("qwp-mirror-reseed"); + try { + populateRecoverableSlot(sfDir); + Path slot = sfDir.resolve("default"); + assertMemoryLeak(() -> { + try (CursorSendEngine engine = new CursorSendEngine(slot.toString(), 4096)) { + PersistedSymbolDict pd = engine.getPersistedSymbolDict(); + Assert.assertNotNull(pd); + int dictSize = pd.size(); + Assert.assertTrue("recovery must load a non-empty dictionary", dictSize > 0); + + // Session 1 seeds its mirror from the persisted dictionary. + CursorWebSocketSendLoop loop1 = newRecoveryLoop(engine); + try { + Assert.assertEquals("session-1 mirror must seed from the persisted dict", + dictSize, readInt(loop1, "sentDictCount")); + } finally { + loop1.close(); + } + + // Session 2 against the SAME engine (the drainer recycle): the + // seed must NOT have been consumed -- the mirror must re-seed to + // the full dictionary so the reconnect catch-up is complete. + CursorWebSocketSendLoop loop2 = newRecoveryLoop(engine); + try { + Assert.assertEquals("recycled session-2 mirror must re-seed from the " + + "persisted dict (pre-fix it was 0)", + dictSize, readInt(loop2, "sentDictCount")); + } finally { + loop2.close(); + } + } + }); + } finally { + rmDir(sfDir); + } + } + + // Constructs a recovery send loop but does NOT start it: the ctor seeds the + // catch-up mirror synchronously, which is all these tests observe. The + // reconnect factory is never invoked. + private static CursorWebSocketSendLoop newRecoveryLoop(CursorSendEngine engine) { + return new CursorWebSocketSendLoop( + null, engine, 0, 1_000_000L, + () -> { + throw new IOException("no reconnect in this test"); + }, + 0, 0, 1); + } + private static void populateRecoverableSlot(Path sfDir) throws Exception { try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) { int port = silent.getPort(); @@ -117,6 +181,12 @@ private static void populateRecoverableSlot(Path sfDir) throws Exception { } } + private static int readInt(CursorWebSocketSendLoop loop, String name) throws Exception { + Field f = CursorWebSocketSendLoop.class.getDeclaredField(name); + f.setAccessible(true); + return f.getInt(loop); + } + private static void rmDir(Path dir) throws IOException { if (dir == null || !Files.exists(dir)) { return; From 444199ea881da44aa9ee988df1b220ff5b568d20 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Fri, 10 Jul 2026 12:32:53 +0100 Subject: [PATCH 26/96] Fix symbol-dict durability doc; add recovery test The PersistedSymbolDict class doc claimed a host-crash-torn dictionary is "caught at replay by the send loop's guard ... rather than corrupting the target table." The guard is tail-only: it fires only when a frame's delta start id exceeds the recovered dictionary size. It does NOT catch an interior page lost out of order (reads back as zeroes that parse as empty-string entries) or a failed best-effort truncate that leaves a stale trailing entry -- either shifts the dense id->symbol mapping without changing the entry count the guard checks, so a replay silently misattributes symbols. Correct the class doc and the truncate comment to describe the actual, tail-only protection and name the residual host-crash exposure (within the already-documented "not host-crash durable" boundary); note that a per-entry or running CRC would close it. Add testRecoveryAfterFailedPublishReplaysGapFree: a failed publish persists a frame's symbol without recording the frame, leaving the persisted dictionary a strict superset of the recorded frames. The test recovers the slot on a fresh sender against a dictionary-reconstructing server and asserts the catch-up re-registers the whole superset (the unrecorded symbol included) and the replay is gap-free -- the end-to-end fail -> recover -> replay chain the existing no-duplicate test omitted. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../client/sf/cursor/PersistedSymbolDict.java | 23 ++++- .../qwp/client/DeltaDictRecoveryTest.java | 94 +++++++++++++++++++ 2 files changed, 112 insertions(+), 5 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java index bf5fc39f..986039c2 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java @@ -67,9 +67,18 @@ * dictionary is a superset of every recoverable frame's references. It is NOT * sufficient for a host/power crash, where unflushed pages can be lost out * of order and the dictionary may end up torn relative to the frames it serves -- - * exactly as the segment frames themselves may be lost on a host crash. A torn - * dictionary is caught at replay by the send loop's guard, which fails loudly - * (the unreplayable data must be resent) rather than corrupting the target table. + * exactly as the segment frames themselves may be lost on a host crash. The send + * loop's replay guard catches the COMMON host-crash outcome -- a truncated tail, + * where a frame's delta start id exceeds the recovered dictionary size -- and + * fails loudly (the unreplayable data must be resent) rather than sending a + * gapped frame. It does NOT catch every host-crash tear: an interior page lost + * out of order reads back as zeroes that parse as empty-string entries, and a + * failed best-effort truncate (see {@link #open}) can leave a stale trailing + * entry -- either SHIFTS the dense id->symbol mapping without changing the entry + * count the guard checks, so a replay silently misattributes symbols. That + * residual exposure sits within the "not host-crash durable" boundary above; a + * per-entry or running CRC would be needed to close it (the 3 reserved header + * bytes leave room for a future integrity field). *

* A torn trailing entry from a crash mid-append is self-healing: {@link #open} * stops parsing at the first incomplete entry and the next append overwrites it. @@ -367,8 +376,12 @@ private static PersistedSymbolDict openExisting(String filePath, long fileLen) { // shift every subsequent dense id. Without this, appendOffset already // lands just past the last complete entry, so the next append // overwrites FROM there, but only truncation guarantees nothing - // survives BEYOND it. Best-effort: a failed truncate just forgoes the - // hardening and falls back to the overwrite-from-appendOffset behaviour. + // survives BEYOND it. Best-effort: a failed truncate (rare -- an I/O + // error) leaves a latent silent-corruption path, not merely a lost + // optimization: if a later, shorter append then leaves stale bytes past + // its end, a subsequent recovery mis-parses them as a ghost symbol and + // shifts the dense ids, and the count-based replay guard does not catch + // it (see the class-level durability note). long validLen = HEADER_SIZE + entriesLen; if (validLen < len) { Files.truncate(fd, validLen); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java index 8174ed24..222eeebe 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java @@ -480,6 +480,100 @@ public void testFailedPublishDoesNotDuplicatePersistedSymbols() throws Exception }); } + @Test + public void testRecoveryAfterFailedPublishReplaysGapFree() throws Exception { + // M3 end-to-end: chains a failed publish -> fresh-process recovery -> replay. + // A failed publish persists the frame's symbol (write-ahead) but does NOT + // record the frame, so the persisted dictionary becomes a strict SUPERSET of + // the recorded frames' references. A recovering sender must still replay + // gap-free: it re-registers the whole (superset) dictionary via the catch-up + // -- including the symbol whose frame never reached disk -- so the fresh + // server reconstructs a complete, gap-free dictionary. The sibling + // testFailedPublishDoesNotDuplicatePersistedSymbols proves the dict has no + // duplicate after the failed publish; this proves the resulting slot then + // recovers and replays end-to-end against a real server. + assertMemoryLeak(() -> { + // Phase 1: a silent server (no acks) + a small SF segment. Four small + // rows register sym-0..sym-3 and their frames are recorded; a fifth, + // oversized row registers (persists) sym-4 but its frame is too large for + // the segment, so appendBlocking throws and the frame is NOT recorded. + try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) { + int port = silent.getPort(); + silent.start(); + Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS)); + String cfg = "ws::addr=localhost:" + port + + ";sf_dir=" + sfDir + + ";sf_max_bytes=4096" + + ";close_flush_timeout_millis=0;"; + Sender s1 = Sender.fromConfig(cfg); + try { + for (int i = 0; i < 4; i++) { + s1.table("m").symbol("s", "sym-" + i).longColumn("v", i).atNow(); + s1.flush(); + } + // Oversized frame: sym-4 is persisted (write-ahead) before the + // publish fails, so the dictionary runs one ahead of the frames. + s1.table("m").symbol("s", "sym-4") + .stringColumn("p", TestUtils.repeat("x", 8000)) + .longColumn("v", 4).atNow(); + try { + s1.flush(); + Assert.fail("oversized frame must fail to publish"); + } catch (LineSenderException expected) { + // PAYLOAD_TOO_LARGE -- frame not recorded, sym-4 stays persisted + } + } finally { + try { + // Re-flushes the still-buffered oversized row and fails again + // (expected); resources are still released, and the idempotent + // write-ahead does not re-append sym-4. + s1.close(); + } catch (LineSenderException ignored) { + } + } + } + + // The persisted dictionary must hold the superset: sym-0..sym-4 (5 ids), + // one more than the four recorded frames reference, with no duplicate. + PersistedSymbolDict pd = PersistedSymbolDict.open(Paths.get(sfDir, "default").toString()); + Assert.assertNotNull(pd); + try { + Assert.assertEquals("failed publish must leave the dict a superset (sym-0..sym-4)", + 5, pd.size()); + } finally { + pd.close(); + } + + // Phase 2: recover against a fresh server that reconstructs its + // dictionary from the wire. The recovering sender must re-register all 5 + // symbols via a catch-up (sym-4 exists ONLY in the dictionary -- no frame + // carries it) and replay the 4 recorded frames, leaving a complete, + // gap-free server dictionary. + DictReconstructingHandler handler = new DictReconstructingHandler(); + try (TestWebSocketServer good = new TestWebSocketServer(handler)) { + int port = good.getPort(); + good.start(); + Assert.assertTrue(good.awaitStart(5, TimeUnit.SECONDS)); + String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";"; + try (Sender ignored = Sender.fromConfig(cfg)) { + long deadline = System.currentTimeMillis() + 5_000; + while (System.currentTimeMillis() < deadline && handler.maxDictSize() < 5) { + Thread.sleep(20); + } + } + Assert.assertTrue("recovery must send a full-dictionary catch-up frame", + handler.sawCatchUpFrame); + List dict = handler.dictSnapshot(); + Assert.assertEquals("recovered dictionary must include the failed-publish symbol", + 5, dict.size()); + for (int i = 0; i < 5; i++) { + Assert.assertEquals("dictionary id " + i + " must be gap-free", + "sym-" + i, dict.get(i)); + } + } + }); + } + private static void writeAckWatermark(java.nio.file.Path path, long fsn) throws IOException { byte[] buf = new byte[16]; ByteBuffer bb = ByteBuffer.wrap(buf).order(ByteOrder.LITTLE_ENDIAN); From 7a9565397c18aaa5411434fd8a27f0f49317b416 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Fri, 10 Jul 2026 13:48:20 +0100 Subject: [PATCH 27/96] Discard a surviving symbol dict on fresh start A fresh store-and-forward slot inherited a stale .symbol-dict when the best-effort removeOrphan failed to delete it (e.g. a Windows share lock) or a crash landed in the close window: open() parses and TRUSTS any survivor. Because the fresh-start producer is not seeded from the dictionary (that is gated on wasRecoveredFromDisk, while the send-loop mirror and the persist resume point key off pd.size()), the producer's ids then diverged from the dictionary the send loop replays, silently misattributing symbol values after a reconnect. The dictionary is load-bearing, unlike the max()-clamped ack watermark whose removeOrphan+open pattern it had copied. Enforce "fresh start -> empty dictionary" at the source: openClean() truncates any survivor via openCleanRW instead of trusting it, and if the clean open itself fails the sender falls back to full self-sufficient frames, which is also safe. The recovery path keeps open(); removeOrphan stays for the fully-drained close. Both regression tests fail without the fix (dict size 2, not 0): - PersistedSymbolDictTest.testOpenCleanDiscardsSurvivingDictionary - EmptyOrphanSlotChurnTest.testFreshStartDiscardsSurvivingStaleDictionary Co-Authored-By: Claude Opus 4.8 (1M context) --- .../client/sf/cursor/CursorSendEngine.java | 17 +++++-- .../client/sf/cursor/PersistedSymbolDict.java | 29 +++++++++-- .../sf/cursor/EmptyOrphanSlotChurnTest.java | 44 +++++++++++++++++ .../sf/cursor/PersistedSymbolDictTest.java | 49 +++++++++++++++++++ 4 files changed, 131 insertions(+), 8 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java index 3bc0a185..4b692288 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java @@ -321,10 +321,19 @@ private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager man if (!memoryMode) { AckWatermark.removeOrphan(sfDir); watermarkInProgress = AckWatermark.open(sfDir); - // Same stale-side-file hygiene for the symbol dictionary: a - // fresh slot starts with an empty dictionary. - PersistedSymbolDict.removeOrphan(sfDir); - persistedDictInProgress = PersistedSymbolDict.open(sfDir); + // A fresh slot MUST start with an EMPTY symbol dictionary. + // Unlike the ack watermark above -- a discardable optimization a + // max() clamp protects -- the dictionary is load-bearing: a + // delta frame referencing an id missing from it is unrecoverable, + // and a STALE dictionary inherited here (the segments are gone, so + // the producer is NOT seeded from it) shifts the dense id->symbol + // mapping and silently misattributes symbols on the next + // reconnect. openClean() truncates any survivor to empty rather + // than trusting a best-effort delete that may have failed (e.g. a + // Windows share lock); if the clean open itself fails, + // persistedSymbolDict stays null and the sender falls back to full + // self-sufficient frames, which is also safe. + persistedDictInProgress = PersistedSymbolDict.openClean(sfDir); } MmapSegment initial; String initialPath = null; diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java index 986039c2..3fc76fd4 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java @@ -147,10 +147,31 @@ public static PersistedSymbolDict open(String slotDir) { } /** - * Best-effort removal of a stale dictionary file. Used at fresh-start (a - * stale dict with no segments behind it is meaningless) and at fully-drained - * close (the slot is empty, nothing references the dictionary any more), - * mirroring {@link AckWatermark#removeOrphan}. + * Opens the dictionary in {@code slotDir} as a FRESH, EMPTY file, discarding + * any surviving content. This is the fresh-start counterpart to {@link #open}: + * a slot with no recovered segments must start with an empty dictionary, so a + * dictionary left by a prior lifecycle -- a fully-drained slot whose + * best-effort delete failed, or a crash in the close window -- must NOT be + * inherited. Unlike {@link #open}, which parses and TRUSTS an existing file for + * recovery/orphan-drain replay, this truncates it: the fresh-start producer is + * not seeded from the dictionary, so trusting a survivor would leave the + * producer's ids diverged from the dictionary the send loop replays and + * silently misattribute symbols on the next reconnect. Truncating (rather than + * relying on an unlink succeeding first) closes the gap even when the delete is + * refused -- e.g. a Windows share lock. Returns {@code null} on I/O failure, so + * the caller falls back to full self-sufficient frames exactly as {@link #open} + * does. + */ + public static PersistedSymbolDict openClean(String slotDir) { + return openFresh(slotDir + "/" + FILE_NAME); + } + + /** + * Best-effort removal of a stale dictionary file. Used at fully-drained close + * (the slot is empty, nothing references the dictionary any more), mirroring + * {@link AckWatermark#removeOrphan}. The fresh-start path deliberately does NOT + * use this -- it opens a clean dictionary via {@link #openClean} instead, so a + * failed delete cannot leave a stale dictionary a new session would trust. */ public static void removeOrphan(String slotDir) { Files.remove(slotDir + "/" + FILE_NAME); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/EmptyOrphanSlotChurnTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/EmptyOrphanSlotChurnTest.java index 5c7607a6..2efee2e4 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/EmptyOrphanSlotChurnTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/EmptyOrphanSlotChurnTest.java @@ -25,6 +25,7 @@ package io.questdb.client.test.cutlass.qwp.client.sf.cursor; import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine; +import io.questdb.client.cutlass.qwp.client.sf.cursor.PersistedSymbolDict; import io.questdb.client.std.Files; import io.questdb.client.test.tools.TestUtils; import org.junit.After; @@ -35,6 +36,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; /** * Regression test for M6 — drainer adopting an empty orphan slot would @@ -87,6 +89,48 @@ public void tearDown() { Files.remove(sfDir); } + @Test + public void testFreshStartDiscardsSurvivingStaleDictionary() throws Exception { + // Regression: a prior fully-drained lifecycle can leave a stale + // .symbol-dict behind (a best-effort delete that failed, or a crash in the + // close window) with NO segments. A fresh start must DISCARD it -- the + // dictionary is load-bearing and the fresh-start producer is not seeded + // from it, so trusting a survivor would diverge the producer ids from the + // dictionary the send loop replays and misattribute symbols on reconnect. + TestUtils.assertMemoryLeak(() -> { + // Pre-seed a stale dictionary in the slot, with no segments behind it. + PersistedSymbolDict stale = PersistedSymbolDict.open(sfDir); + assertNotNull(stale); + try { + stale.appendSymbol("staleX"); + stale.appendSymbol("staleY"); + assertEquals(2, stale.size()); + } finally { + stale.close(); + } + + // A fresh start (no recovered segments) must open a CLEAN, empty + // dictionary -- not inherit the survivor. + try (CursorSendEngine engine = new CursorSendEngine(sfDir, 4L * 1024 * 1024)) { + assertFalse("fresh start must not report a disk recovery", + engine.wasRecoveredFromDisk()); + PersistedSymbolDict pd = engine.getPersistedSymbolDict(); + assertNotNull(pd); + assertEquals("fresh start must discard the surviving stale dictionary", + 0, pd.size()); + } + + // The survivor's bytes are physically gone, not just hidden. + PersistedSymbolDict reopened = PersistedSymbolDict.open(sfDir); + assertNotNull(reopened); + try { + assertEquals(0, reopened.size()); + } finally { + reopened.close(); + } + }); + } + @Test public void testNeverPublishedCloseLeavesNoSfaFiles() throws Exception { TestUtils.assertMemoryLeak(() -> { diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java index 67fd7438..cc45c23b 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java @@ -299,6 +299,55 @@ public void testLargeSymbolRoundTripsAcrossReopen() throws Exception { }); } + @Test + public void testOpenCleanDiscardsSurvivingDictionary() throws Exception { + // A fresh start must NOT inherit a dictionary left by a prior lifecycle: + // openClean() truncates any survivor to empty, where open() would recover + // (and TRUST) it. Trusting a survivor whose segments are gone -- the + // fresh-start producer is not seeded from it -- shifts the dense id->symbol + // mapping and misattributes symbols on the next reconnect. + assertMemoryLeak(() -> { + Path dir = Files.createTempDirectory("qwp-symdict-clean"); + try { + PersistedSymbolDict stale = PersistedSymbolDict.open(dir.toString()); + Assert.assertNotNull(stale); + try { + stale.appendSymbol("staleX"); + stale.appendSymbol("staleY"); + Assert.assertEquals(2, stale.size()); + } finally { + stale.close(); + } + + // Fresh start: openClean yields an EMPTY dictionary regardless of + // the survivor, and appends from id 0 again. + PersistedSymbolDict fresh = PersistedSymbolDict.openClean(dir.toString()); + Assert.assertNotNull(fresh); + try { + Assert.assertEquals(0, fresh.size()); + Assert.assertEquals(0, fresh.readLoadedSymbols().size()); + fresh.appendSymbol("freshA"); + Assert.assertEquals(1, fresh.size()); + } finally { + fresh.close(); + } + + // The survivor's bytes are physically gone, not just hidden: a + // subsequent recovery open() sees only the post-clean content. + PersistedSymbolDict reopened = PersistedSymbolDict.open(dir.toString()); + Assert.assertNotNull(reopened); + try { + Assert.assertEquals(1, reopened.size()); + Assert.assertEquals("freshA", reopened.readLoadedSymbols().getQuick(0)); + } finally { + reopened.close(); + } + } finally { + rmDir(dir); + } + }); + } + @Test public void testRemoveOrphanDeletesFile() throws Exception { assertMemoryLeak(() -> { From 6d14727421438637ed9e3be5391d34a19e94c567 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Fri, 10 Jul 2026 14:05:04 +0100 Subject: [PATCH 28/96] Harden delta symbol-dict send and persist paths Four follow-ups from the delta symbol-dictionary review. Catch-up NACK no longer launders the poison detector. After a reconnect the loop ships the head data frame before tryReceiveAcks reads the server's NACK of a dictionary catch-up frame, so handleServerRejection saw dataFrameSentThisConnection=true and attributed the NACK a wire seq below the replay head -- fsn = fsnAtZero+cappedSeq, negative when replayStart < catchUpFrames. recordHeadRejectionStrike then set poisonFsn to that value (often -1, the "no suspect" sentinel), wiping a genuine in-progress poison run and reporting a bogus fsn. Route an fsn <= ackedFsn rejection to the pre-send path (surface + recycle, no strike, no watermark advance), symmetric with the success path's engine.acknowledge() no-op below ackedFsn. A real replayed data frame is at fsn > ackedFsn, so it is never caught. Regression test: testPostSendCatchUpNackDoesNotStrikeOrLaunderPoisonState. Persist short-write now surfaces as LineSenderException. A disk-full during the write-ahead persist threw a raw IllegalStateException that escaped flush() unwrapped, unlike every other flush-path failure (e.g. the cursor append in sealAndSwapBuffer). Wrap it so a caller catching LineSenderException around flush() sees a persist disk-full too; a JVM Error still propagates. PersistedSymbolDict close() and the append methods are synchronized. close() is callable from any thread (a shutdown hook); without mutual exclusion a close racing an in-flight append could free the scratch buffer or close the fd mid-write and let the write land on a descriptor the OS has reused for another file (silent cross-file corruption). The appendSymbols re-encode branch now has a regression test. After a failed publish leaves the durable dictionary size ahead of the wire baseline, a later flush introducing a new symbol re-encodes only the [pd.size() .. currentBatchMaxSymbolId] suffix; keying that off pd.size() (not sentMaxSymbolId+1) keeps it idempotent. Regression test: testFailedPublishThenNewSymbolPersistsSuffixWithoutDuplicating. Both new tests fail with their production hunk reverted. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../qwp/client/QwpWebSocketSender.java | 32 +++++++--- .../sf/cursor/CursorWebSocketSendLoop.java | 19 ++++++ .../client/sf/cursor/PersistedSymbolDict.java | 19 ++++-- .../qwp/client/DeltaDictRecoveryTest.java | 64 +++++++++++++++++++ ...ursorWebSocketSendLoopPoisonFrameTest.java | 62 ++++++++++++++++++ 5 files changed, 181 insertions(+), 15 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index 942af99b..bb1256dd 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -3707,14 +3707,30 @@ private void persistNewSymbolsBeforePublish() { // failed publish the durable size has run ahead of the wire baseline, so // the frame's delta covers MORE than remains to persist; then re-encode // just the [from .. currentBatchMaxSymbolId] suffix. - if (from == sentMaxSymbolId + 1) { - QwpBufferWriter buffer = encoder.getBuffer(); - pd.appendRawEntries( - buffer.getBufferPtr() + encoder.getDeltaEntriesStart(), - encoder.getDeltaEntriesLen(), - currentBatchMaxSymbolId - from + 1); - } else { - pd.appendSymbols(globalSymbolDictionary, from, currentBatchMaxSymbolId); + try { + if (from == sentMaxSymbolId + 1) { + QwpBufferWriter buffer = encoder.getBuffer(); + pd.appendRawEntries( + buffer.getBufferPtr() + encoder.getDeltaEntriesStart(), + encoder.getDeltaEntriesLen(), + currentBatchMaxSymbolId - from + 1); + } else { + pd.appendSymbols(globalSymbolDictionary, from, currentBatchMaxSymbolId); + } + } catch (Throwable t) { + // A short write (disk full / quota) to the persisted dictionary throws + // a low-level IllegalStateException. Surface it as a LineSenderException + // -- like every other flush-path failure, e.g. the cursor append in + // sealAndSwapBuffer -- so a caller catching LineSenderException around + // flush() also catches a disk-full during the write-ahead persist. The + // persist ran before publish and pd.size() did not advance on the short + // write, so the still-buffered rows re-persist the same range + // idempotently on retry. A JVM Error is never a persist failure; let it + // propagate. + if (t instanceof Error) { + throw (Error) t; + } + throw new LineSenderException("failed to persist symbol dictionary before publish", t); } } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java index d910a6ca..478d33c5 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java @@ -2811,6 +2811,25 @@ private void handleServerRejection(long wireSeq) { wireSeq, highestSent); } long fsn = fsnAtZero + cappedSeq; + if (fsn <= engine.ackedFsn()) { + // The clamped wire seq maps at or below the replay head, so this + // NACK is for a dictionary catch-up frame -- which occupies the + // already-acked wire sequences below replayStart -- not a data frame + // this connection sent. (dataFrameSentThisConnection can be true here + // because trySendOne ships the head data frame before tryReceiveAcks + // reads the catch-up's NACK in the same loop iteration.) Attributing + // it a data FSN would key recordHeadRejectionStrike() off a + // below-baseline FSN -- negative when replayStart < catchUpFrames -- + // colliding with the poisonFsn == -1 "no suspect" sentinel and + // laundering a genuine poison run, and would report a bogus FSN. + // Treat it exactly like a pre-send rejection: surface + recycle, no + // poison strike, no watermark advance. Symmetric with the success + // path, where engine.acknowledge() no-ops at or below ackedFsn. A + // real replayed data frame is at fsn > ackedFsn, so it is never + // caught here. + handlePreSendRejection(wireSeq, status, category, policy); + return; + } // Best-effort table attribution: the parser populates // response.tableNames on error frames the same way it does on // STATUS_OK. If exactly one table was named, surface it; if diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java index 3fc76fd4..d2651923 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java @@ -83,9 +83,14 @@ * A torn trailing entry from a crash mid-append is self-healing: {@link #open} * stops parsing at the first incomplete entry and the next append overwrites it. *

- * Lifecycle: single-writer (the producer / user thread). Read once at - * {@link #open} to seed in-memory state on recovery or orphan-drain. Owner - * (the engine) closes it. Not thread-safe for concurrent writers. + * Lifecycle: single-writer (the producer / user thread) for appends. Read + * once at {@link #open} to seed in-memory state on recovery or orphan-drain. The + * owner (the engine) closes it, and {@code close()} is callable from any thread + * (a shutdown hook, test cleanup). {@code close()} and the append methods are + * therefore {@code synchronized}: without that, a close racing an in-flight append + * could free the scratch buffer or close the fd mid-write and let the write land + * on a descriptor the OS has reused for another file (silent cross-file + * corruption). Not thread-safe for concurrent writers. */ public final class PersistedSymbolDict implements QuietCloseable { @@ -188,7 +193,7 @@ public static void removeOrphan(String slotDir) { * appendOffset}, so a retry keyed off {@link #size()} re-persists the same range * at the same offset. No-op when the range is empty or the dictionary is closed. */ - public void appendRawEntries(long addr, int len, int count) { + public synchronized void appendRawEntries(long addr, int len, int count) { if (closed || count <= 0 || len <= 0) { return; } @@ -208,7 +213,7 @@ public void appendRawEntries(long addr, int len, int count) { * (see the class-level durability note). Assigns the next dense id implicitly * (the entry's position). */ - public void appendSymbol(CharSequence symbol) { + public synchronized void appendSymbol(CharSequence symbol) { if (closed) { return; } @@ -244,7 +249,7 @@ public void appendSymbol(CharSequence symbol) { * range and overwrites at the same offset. No-op when the range is empty or * the dictionary is closed. */ - public void appendSymbols(GlobalSymbolDictionary dict, int from, int to) { + public synchronized void appendSymbols(GlobalSymbolDictionary dict, int from, int to) { if (closed || to < from) { return; } @@ -270,7 +275,7 @@ public void appendSymbols(GlobalSymbolDictionary dict, int from, int to) { } @Override - public void close() { + public synchronized void close() { if (closed) { return; } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java index 222eeebe..58f9cfa9 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java @@ -480,6 +480,70 @@ public void testFailedPublishDoesNotDuplicatePersistedSymbols() throws Exception }); } + @Test + public void testFailedPublishThenNewSymbolPersistsSuffixWithoutDuplicating() throws Exception { + // Regression for the re-encode (else) branch of persistNewSymbolsBeforePublish. + // After a failed publish leaves the durable dictionary size ahead of the wire + // baseline (pd.size() > sentMaxSymbolId+1), a later flush that introduces a NEW + // symbol cannot reuse the frame's already-encoded fast-path bytes -- it + // re-encodes just the [pd.size() .. currentBatchMaxSymbolId] suffix via + // appendSymbols. Keying that off pd.size() (not sentMaxSymbolId+1) keeps it + // idempotent: the already-persisted prefix is NOT re-appended. + assertMemoryLeak(() -> { + try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) { + int port = silent.getPort(); + silent.start(); + Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS)); + // Small segment: any frame carrying the padded row fails to publish with + // PAYLOAD_TOO_LARGE deterministically (no backpressure timing). + String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";sf_max_bytes=1024;"; + String pad = TestUtils.repeat("x", 2000); // frame >> 1024-byte segment + Sender sender = Sender.fromConfig(cfg); + try { + // Flush 1: s0 is persisted (write-ahead) before the oversized frame + // fails to publish. pd.size()=1, sentMaxSymbolId stays -1. + sender.table("m").symbol("s", "s0").stringColumn("p", pad).longColumn("v", 1L).atNow(); + try { + sender.flush(); + Assert.fail("oversized frame must fail to publish"); + } catch (LineSenderException expected) { + } + + // Add a NEW symbol s1 (id 1). The failed s0 row is still buffered, so + // the batch is {s0, s1} and the durable size (1) has run ahead of the + // wire baseline (-1) -- the state that selects the appendSymbols branch. + sender.table("m").symbol("s", "s1").stringColumn("p", pad).longColumn("v", 2L).atNow(); + try { + sender.flush(); + Assert.fail("oversized frame must fail to publish"); + } catch (LineSenderException expected) { + } + + // The else branch persisted ONLY s1 (the suffix). The dictionary holds + // s0, s1 exactly once each. Pre-fix (appendSymbols from + // sentMaxSymbolId+1) re-appended s0, giving size 3. + PersistedSymbolDict pd = PersistedSymbolDict.open(Paths.get(sfDir, "default").toString()); + Assert.assertNotNull(pd); + try { + Assert.assertEquals("re-encode suffix must not duplicate the persisted prefix", + 2, pd.size()); + Assert.assertEquals("s0", pd.readLoadedSymbols().getQuick(0)); + Assert.assertEquals("s1", pd.readLoadedSymbols().getQuick(1)); + } finally { + pd.close(); + } + } finally { + try { + sender.close(); + } catch (LineSenderException ignored) { + // close() re-flushes the still-buffered oversized rows and fails + // again (PAYLOAD_TOO_LARGE); expected, resources still released. + } + } + } + }); + } + @Test public void testRecoveryAfterFailedPublishReplaysGapFree() throws Exception { // M3 end-to-end: chains a failed publish -> fresh-process recovery -> replay. diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopPoisonFrameTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopPoisonFrameTest.java index ab200e80..d413d82e 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopPoisonFrameTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopPoisonFrameTest.java @@ -656,6 +656,46 @@ public void testPoisonDwellHoldsEscalationUntilWallClockWindowElapses() throws E }); } + @Test + public void testPostSendCatchUpNackDoesNotStrikeOrLaunderPoisonState() throws Exception { + // Regression: after a reconnect the loop ships the head DATA frame + // (dataFrameSentThisConnection=true) BEFORE tryReceiveAcks reads the server's + // NACK of a dictionary CATCH-UP frame in the same loop iteration. That NACK + // names a wire seq below the replay head, so its fsn = fsnAtZero+cappedSeq is + // at or below ackedFsn -- negative when replayStart < catchUpFrames. It must + // NOT charge a poison strike: recordHeadRejectionStrike(fsn) would set + // poisonFsn to that value (the common shape yields -1, the "no suspect" + // sentinel), laundering any genuine in-progress poison run and reporting a + // bogus fsn. The fix routes an fsn <= ackedFsn rejection to the pre-send path + // (surface + recycle, no strike), symmetric with the success path's + // engine.acknowledge() no-op below ackedFsn. + TestUtils.assertMemoryLeak(() -> { + List clients = new ArrayList<>(); + try (CursorSendEngine engine = newEngine()) { + appendFrames(engine, 2); + CursorWebSocketSendLoop loop = newDurableLoop(engine, clients); + // Model: 2 catch-up frames (wire seq 0,1) + 1 data frame (wire seq 2) + // sent, nothing acked (ackedFsn = -1), so fsnAtZero = replayStart(0) - + // catchUpFrames(2) = -2. A NACK of catch-up wire seq 0 maps to fsn -2, + // strictly below the -1 sentinel so the assertion discriminates. + setPostCatchUpDataFrameBaseline(loop, -2L, 3L); + assertEquals("precondition: no poison suspect yet", + -1L, getLongField(loop, "poisonFsn")); + + deliverRetriableNack(loop, 0, "disk full"); + + // The catch-up NACK was surfaced + recycled but charged NO strike, so + // the sentinel is intact. Pre-fix it was set to -2, laundering the + // poison detector's state. + assertEquals("catch-up NACK must not set/launder the poison sentinel", + -1L, getLongField(loop, "poisonFsn")); + loop.checkError(); + } finally { + closeAll(clients); + } + }); + } + @Test public void testPostSendNotWritableNackNeverEscalatesToPoisonTerminal() throws Exception { // RETRIABLE_OTHER (NOT_WRITABLE) is a node-state verdict, not a frame @@ -1058,6 +1098,28 @@ private static void setCatchUpWireSeqOnly(CursorWebSocketSendLoop loop, long cou f.setLong(loop, count); } + // Models a fresh connection that has shipped the dictionary catch-up (advancing + // fsnAtZero below the replay head) AND the head data frame: sets fsnAtZero, + // nextWireSeq, and dataFrameSentThisConnection=true together. + private static void setPostCatchUpDataFrameBaseline(CursorWebSocketSendLoop loop, + long fsnAtZero, long nextWireSeq) throws Exception { + Field z = CursorWebSocketSendLoop.class.getDeclaredField("fsnAtZero"); + z.setAccessible(true); + z.setLong(loop, fsnAtZero); + Field w = CursorWebSocketSendLoop.class.getDeclaredField("nextWireSeq"); + w.setAccessible(true); + w.setLong(loop, nextWireSeq); + Field d = CursorWebSocketSendLoop.class.getDeclaredField("dataFrameSentThisConnection"); + d.setAccessible(true); + d.setBoolean(loop, true); + } + + private static long getLongField(CursorWebSocketSendLoop loop, String name) throws Exception { + Field f = CursorWebSocketSendLoop.class.getDeclaredField(name); + f.setAccessible(true); + return f.getLong(loop); + } + private static long[] txns(long... v) { return v; } From fdd7141f246839f9df002d2a8e5b0c919bce02ba Mon Sep 17 00:00:00 2001 From: glasstiger Date: Fri, 10 Jul 2026 14:37:42 +0100 Subject: [PATCH 29/96] Tidy delta symbol-dict dead code and edges Minor follow-ups from the review. Remove the unused CursorSendEngine.isMemoryMode() -- it had no callers; isDeltaDictEnabled() tests sfDir == null directly. Sort the NativeBufferWriter import into its alphabetical place among the qwp.client imports in CursorWebSocketSendLoop. Harden PersistedSymbolDict.openExisting: hoist entriesAddr / entriesLen so the catch frees the loaded-entries buffer too. It is unreachable today (nothing between that malloc and the return throws, and on success the buffer is transferred to the returned dict), but the error path no longer leaks if a future edit adds a throwing step. Return a defensive copy from DeltaDictCatchUpTest's dictFor() instead of the live inner list: the caller iterates it unlocked while the server thread may still be appending, matching the sibling dictSnapshot(). Add testTornDictionaryOneIdGapFailsCleanly -- the tightest torn-dictionary boundary (deltaStart == recoveredSize + 1), the common one-entry-short host-crash tail. The existing torn test uses a 3-id gap, so it does not pin the guard's false-negative edge; a "deltaStart > size + 1" mutation passes it but ships the gapped frame here (the new test fails on it). Co-Authored-By: Claude Opus 4.8 (1M context) --- .../client/sf/cursor/CursorSendEngine.java | 11 --- .../sf/cursor/CursorWebSocketSendLoop.java | 2 +- .../client/sf/cursor/PersistedSymbolDict.java | 12 ++- .../qwp/client/DeltaDictCatchUpTest.java | 8 +- .../qwp/client/DeltaDictRecoveryTest.java | 75 +++++++++++++++++++ 5 files changed, 92 insertions(+), 16 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java index 4b692288..c962a56d 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java @@ -629,17 +629,6 @@ public long getTotalBackpressureStalls() { return backpressureStallCount.get(); } - /** - * True when this engine has no store-and-forward directory: the ring lives - * only in malloc'd memory, so it cannot be recovered after a crash and no - * orphan drainer ever replays it. Only in-process reconnect/failover replays - * its frames, which is what makes send-time symbol-dict catch-up (rather than - * fully self-sufficient frames) safe. - */ - public boolean isMemoryMode() { - return sfDir == null; - } - /** * Whether the sender may delta-encode symbol dictionaries on this engine. * Always true in memory mode (the send loop keeps an in-process catch-up diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java index 478d33c5..acb56cb7 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java @@ -31,12 +31,12 @@ import io.questdb.client.cutlass.http.client.WebSocketFrameHandler; import io.questdb.client.cutlass.http.client.WebSocketUpgradeException; import io.questdb.client.cutlass.line.LineSenderException; +import io.questdb.client.cutlass.qwp.client.NativeBufferWriter; import io.questdb.client.cutlass.qwp.client.QwpAuthFailedException; import io.questdb.client.cutlass.qwp.client.QwpDurableAckMismatchException; import io.questdb.client.cutlass.qwp.client.QwpIngressRoleRejectedException; import io.questdb.client.cutlass.qwp.client.QwpRoleMismatchException; import io.questdb.client.cutlass.qwp.client.QwpVersionMismatchException; -import io.questdb.client.cutlass.qwp.client.NativeBufferWriter; import io.questdb.client.cutlass.qwp.client.WebSocketResponse; import io.questdb.client.cutlass.qwp.protocol.QwpConstants; import io.questdb.client.cutlass.qwp.websocket.WebSocketCloseCode; diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java index d2651923..889104ec 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java @@ -353,6 +353,8 @@ private static PersistedSymbolDict openExisting(String filePath, long fileLen) { return null; } long buf = 0L; + long entriesAddr = 0L; + int entriesLen = 0; try { int len = (int) fileLen; buf = Unsafe.malloc(len, MemoryTag.NATIVE_DEFAULT); @@ -388,8 +390,7 @@ private static PersistedSymbolDict openExisting(String filePath, long fileLen) { pos = next + (int) entryLen; count++; } - int entriesLen = pos - HEADER_SIZE; - long entriesAddr = 0L; + entriesLen = pos - HEADER_SIZE; if (entriesLen > 0) { entriesAddr = Unsafe.malloc(entriesLen, MemoryTag.NATIVE_DEFAULT); Unsafe.getUnsafe().copyMemory(buf + HEADER_SIZE, entriesAddr, entriesLen); @@ -417,6 +418,13 @@ private static PersistedSymbolDict openExisting(String filePath, long fileLen) { if (buf != 0L) { Unsafe.free(buf, (int) fileLen, MemoryTag.NATIVE_DEFAULT); } + // entriesAddr is transferred to the returned dict on the success path, + // and the catch only runs before that return, so freeing it here cannot + // double-free. Unreachable today (nothing between its malloc and the + // return throws), but keeps the error path leak-free against a future edit. + if (entriesAddr != 0L) { + Unsafe.free(entriesAddr, entriesLen, MemoryTag.NATIVE_DEFAULT); + } Files.close(fd); LOG.warn("symbol dict {} recovery failed ({}); recreating", filePath, String.valueOf(t)); return null; diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictCatchUpTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictCatchUpTest.java index 0d2ff9f7..d35a648e 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictCatchUpTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictCatchUpTest.java @@ -272,7 +272,9 @@ private static class CatchUpHandler implements TestWebSocketServer.WebSocketServ synchronized List dictFor(int connNumber) { return connNumber <= dictsByConn.size() - ? dictsByConn.get(connNumber - 1) + // Copy under the lock: the caller iterates it unlocked while the + // server thread may still be appending to the live inner list. + ? new ArrayList<>(dictsByConn.get(connNumber - 1)) : new ArrayList<>(); } @@ -406,7 +408,9 @@ private static class SplitCatchUpHandler implements TestWebSocketServer.WebSocke synchronized List dictFor(int connNumber) { return connNumber <= dictsByConn.size() - ? dictsByConn.get(connNumber - 1) + // Copy under the lock: the caller iterates it unlocked while the + // server thread may still be appending to the live inner list. + ? new ArrayList<>(dictsByConn.get(connNumber - 1)) : new ArrayList<>(); } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java index 58f9cfa9..1439ac20 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java @@ -227,6 +227,81 @@ public void testTornDictionaryFailsCleanlyInsteadOfCorrupting() throws Exception }); } + @Test + public void testTornDictionaryOneIdGapFailsCleanly() throws Exception { + // Boundary variant of testTornDictionaryFailsCleanlyInsteadOfCorrupting: the + // recovered dictionary is short by EXACTLY ONE id -- the tightest gap the + // guard must still reject (deltaStart == recoveredSize + 1). A one-entry-short + // tail is the common host-crash outcome, so this pins the guard's + // false-negative edge: a "deltaStart > size + 1" mutation would let this + // single-id gap through and null-pad the missing symbol on the server. + assertMemoryLeak(() -> { + // Phase 1: each row introduces a new symbol (frame i carries deltaStart=i). + try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) { + int port = silent.getPort(); + silent.start(); + Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS)); + String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + + ";close_flush_timeout_millis=0;"; + try (Sender s1 = Sender.fromConfig(cfg)) { + for (int i = 0; i < 6; i++) { + s1.table("m").symbol("s", "sym-" + i).longColumn("v", i).atNow(); + s1.flush(); + } + } + } + + // Lose the whole dictionary (truncate to the 8-byte header, size 0) but + // stamp the watermark at FSN 0, so recovery replays from FSN 1 -- a frame + // with deltaStart=1. The dictionary covers no ids, so id 0 is the single + // missing entry: deltaStart(1) == recoveredSize(0) + 1. Because size is 0 + // no catch-up is sent, so any counted frame would be the gapped data frame. + java.nio.file.Path slot = Paths.get(sfDir, "default"); + java.nio.file.Path dict = slot.resolve(".symbol-dict"); + byte[] header = Arrays.copyOf(java.nio.file.Files.readAllBytes(dict), 8); + java.nio.file.Files.write(dict, header); + writeAckWatermark(slot.resolve(".ack-watermark"), 0); + + // Phase 2: recover against a fresh counting server. The guard must fire on + // the very first replay frame (deltaStart 1 > recovered size 0) and fail + // terminally rather than send a frame that null-pads id 0. + CountingHandler handler = new CountingHandler(); + try (TestWebSocketServer good = new TestWebSocketServer(handler)) { + int port = good.getPort(); + good.start(); + Assert.assertTrue(good.awaitStart(5, TimeUnit.SECONDS)); + + String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";"; + LineSenderException terminal = null; + Sender s2 = Sender.fromConfig(cfg); + try { + long deadline = System.currentTimeMillis() + 10_000; + while (System.currentTimeMillis() < deadline && terminal == null) { + try { + s2.flush(); + Thread.sleep(20); + } catch (LineSenderException e) { + terminal = e; + } + } + } finally { + try { + s2.close(); + } catch (LineSenderException e) { + if (terminal == null) { + terminal = e; + } + } + } + Assert.assertEquals("a one-id gap must still block replay to a fresh server", + 0, handler.frames.get()); + Assert.assertNotNull("a one-id-short dictionary must surface a terminal error", terminal); + Assert.assertTrue(terminal.getMessage(), + terminal.getMessage().contains("delta start 1 exceeds recovered dictionary size 0")); + } + }); + } + @Test public void testRecoveredSenderContinuesIngestingNewSymbols() throws Exception { // M2 regression: seedGlobalDictionaryFromPersisted resumes the producer's From 2fdbbf969e10bf7cef3986633bba599d8510e504 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Fri, 10 Jul 2026 17:46:16 +0100 Subject: [PATCH 30/96] Fix catch-up terminal on a homogeneous batch cap The reconnect symbol-dictionary catch-up latched a terminal for any entry exceeding the packing budget (cap - HEADER_SIZE - 16). That reserve is larger than the minimal data-frame overhead, so a symbol the producer had already shipped under a given cap could exceed the budget and trip the terminal on reconnect -- permanently hard-failing a running producer on a homogeneous single-cap cluster, the exact failure the store-and-forward path exists to survive. sendDictCatchUp now tests the actual solo catch-up frame (header + the entry's delta varints + the entry bytes) against the cap. That frame is always smaller than the data frame that already carried the entry, so an accepted symbol always re-registers on the same cap; a genuinely oversized entry on a shrunk (heterogeneous) cap still terminates. The conservative packing budget stays, used only for multi-entry chunk splitting. Add a fixed-cap regression test: a 173-char symbol under a fixed cap of 200 is accepted, then re-registers gap-free across a reconnect. It fails before this change, where the reverted condition terminates a frame that fits the cap. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../sf/cursor/CursorWebSocketSendLoop.java | 38 +++++++++--- .../qwp/client/DeltaDictCatchUpTest.java | 61 +++++++++++++++++++ 2 files changed, 91 insertions(+), 8 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java index acb56cb7..bd9a54bc 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java @@ -2171,12 +2171,21 @@ private long readVarintAt(long p, long limit) { */ private int sendDictCatchUp() { int cap = client.getServerMaxBatchSize(); - // Symbol-bytes budget per frame, leaving room for the 12-byte header and - // the two delta-section varints. cap <= 0 means the server advertised no - // limit -- send the whole dictionary in one frame, but still bound the - // budget by MAX_SENT_DICT_BYTES so sendCatchUpChunk's int frameLen - // (HEADER_SIZE + varints + symbolsLen) cannot overflow on a pathological - // multi-GB dictionary (unreachable at real cardinality; defensive). + // The frame ceiling a catch-up chunk must not exceed: the server's + // advertised cap, or -- when the server advertises none (cap <= 0) -- + // MAX_SENT_DICT_BYTES so sendCatchUpChunk's int frameLen (HEADER_SIZE + + // varints + symbolsLen) cannot overflow on a pathological multi-GB + // dictionary (unreachable at real cardinality; defensive). Used by the + // single-entry terminal below, which measures the real solo frame. + int frameLimit = cap > 0 ? cap : MAX_SENT_DICT_BYTES; + // Symbol-bytes budget for PACKING several entries into one chunk, leaving + // room for the 12-byte header and the two delta-section varints. Kept + // deliberately conservative (reserving 16 for the varints): it only makes a + // multi-entry chunk split marginally earlier, never over the cap. It must + // NOT gate the single-entry terminal -- that reserve is larger than the + // minimal data-frame overhead, so an entry the producer already shipped + // under this cap could exceed the reserve yet still fit its own catch-up + // frame; the terminal tests the real solo frame against frameLimit instead. int budget = cap > 0 ? Math.max(1, cap - QwpConstants.HEADER_SIZE - 16) : MAX_SENT_DICT_BYTES - QwpConstants.HEADER_SIZE - 16; @@ -2192,7 +2201,20 @@ private int sendDictCatchUp() { long len = readVarintAt(p, limit); // entry length prefix; varintEnd -> just past prefix long entryEnd = varintEnd + len; // just past [len varint][utf8 bytes] long entryBytes = entryEnd - entryStart; - if (entryBytes > budget) { + // The exact table-less frame sendCatchUpChunk would build for THIS entry + // alone: header + deltaStart varint (the entry's own global id) + + // deltaCount varint (1) + the entry bytes. Terminal only when even that + // solo frame exceeds the cap -- i.e. the entry genuinely cannot be + // re-registered. Testing the real solo frame (not the conservative + // packing budget above) is what keeps a HOMOGENEOUS cluster + // livelock-free: an entry the producer already shipped in a data frame + // under this cap (header + delta varints + entry + schema + >=1 row) is + // strictly larger than its bare catch-up frame, so it always fits here. + long soloFrameLen = QwpConstants.HEADER_SIZE + + NativeBufferWriter.varintSize(chunkStartId + chunkSymbols) + + NativeBufferWriter.varintSize(1) + + entryBytes; + if (soloFrameLen > frameLimit) { // Non-retriable: the entry will not shrink and the same cluster // re-advertises the same cap, so reconnecting would livelock. // Latch a terminal (the data must be resent after the cap is @@ -2209,7 +2231,7 @@ private int sendDictCatchUp() { // the terminal keeps the homogeneous common case livelock-free. LineSenderException err = new LineSenderException( "symbol dictionary entry too large for the server batch cap during catch-up [" - + "entryBytes=" + entryBytes + ", budget=" + budget + ']'); + + "frameLen=" + soloFrameLen + ", cap=" + cap + ']'); recordFatal(err); throw new CatchUpSendException(err); } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictCatchUpTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictCatchUpTest.java index d35a648e..811c0c67 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictCatchUpTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictCatchUpTest.java @@ -99,6 +99,67 @@ public void testReconnectCatchUpRebuildsDictionary() throws Exception { }); } + @Test + public void testFixedCapNearBoundarySymbolCatchesUpWithoutTerminal() throws Exception { + // Regression (homogeneous single cap): a symbol whose length sits just below + // the advertised cap is ACCEPTED into a data frame (messageSize <= cap) and + // enters the sent-dictionary mirror. On reconnect the catch-up must + // re-register it under the SAME cap -- the bare catch-up frame (header + two + // varints + the entry) is smaller than the data frame that already shipped + // it (which also carried the table schema + a row), so it fits. + // + // Pre-fix, the single-entry terminal used the conservative PACKING budget + // (cap - HEADER_SIZE - 16), which is stricter than the producer's publish + // gate (messageSize <= cap) by more than the minimal data-frame overhead. So + // a symbol accepted onto the wire under cap C could exceed that budget and + // trip a spurious "during catch-up" terminal, permanently hard-failing a + // running producer on its first transient reconnect. Concretely at cap=200: + // table("t").symbol("s", <173 chars>).atNow() encodes to 198 bytes (<=200, + // accepted), its dict entry is 2+173=175 bytes (> old budget 172 -> old + // terminal), while the real solo catch-up frame is 12+1+1+175=189 (<=200 -> + // fits). Unlike testCatchUpEntryTooLargeForCapFailsTerminally (a genuinely + // oversized entry on a shrunk cap, which MUST still terminate), this entry + // is legally shippable and must NOT terminate. + final int cap = 200; + final String nearCapSymbol = TestUtils.repeat("x", 173); + assertMemoryLeak(() -> { + CatchUpHandler handler = new CatchUpHandler(); + try (TestWebSocketServer server = new TestWebSocketServer(handler)) { + server.setAdvertisedMaxBatchSize(cap); // same cap on every handshake (homogeneous) + int port = server.getPort(); + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + + try (Sender sender = Sender.fromConfig("ws::addr=localhost:" + port + ";")) { + // Symbol-only row so the near-cap symbol drives the frame size. + sender.table("t").symbol("s", nearCapSymbol).atNow(); + sender.flush(); + waitFor(() -> handler.dictFor(1).size() >= 1, 5_000); + waitFor(() -> handler.conn1Closed, 5_000); + + // The reconnect runs the catch-up under the SAME cap. Pre-fix + // this latched a terminal (surfacing on this flush); post-fix the + // catch-up ships the near-cap symbol and the flush goes through. + sender.table("t").symbol("s", "beta").atNow(); + sender.flush(); + waitFor(() -> handler.connectionsAccepted.get() >= 2 + && handler.dictFor(2).size() >= 2, 5_000); + } + + // Connection 2's dictionary, rebuilt purely from the frames it + // received, must hold the near-cap symbol (re-registered by the + // catch-up) and beta, gap-free -- proving the catch-up SHIPPED rather + // than terminated. + List conn2 = handler.dictFor(2); + Assert.assertTrue("2nd connection saw a catch-up frame with 0 tables", + handler.sawZeroTableFrameOnConn2); + Assert.assertEquals("2nd connection dictionary size", 2, conn2.size()); + Assert.assertEquals(nearCapSymbol, conn2.get(0)); + Assert.assertEquals("beta", conn2.get(1)); + } + }); + } + @Test public void testCatchUpEntryTooLargeForCapFailsTerminally() throws Exception { // A dictionary entry that exceeds the reconnect server's per-chunk budget From 8cfd7bdc86796c6a5c976d8da3d780a19e3d957d Mon Sep 17 00:00:00 2001 From: glasstiger Date: Fri, 10 Jul 2026 17:54:43 +0100 Subject: [PATCH 31/96] Fix recovery id desync on UTF-8-colliding symbols Two DISTINCT source symbols that collapse to the same UTF-8 bytes -- lone UTF-16 surrogates, which the encoder maps to '?' -- get distinct producer ids and persist as separate .symbol-dict entries. On recovery, seedGlobalDictionaryFromPersisted replayed them through getOrAddSymbol, which de-dups the decoded "?" strings, leaving the producer dictionary (and sentMaxSymbolId) one short of pd.size(). That desynced the delta baseline from the send-loop catch-up mirror (which uses pd.size()) and, after a reconnect, silently misattributed later well-formed symbols. Add GlobalSymbolDictionary.addRecoveredSymbol, which appends at the next id WITHOUT de-duplicating, and seed recovery through it so the producer id space always matches the persisted entry count. The reverse lookup keeps the highest id for a colliding string, which is harmless -- both ids encode to identical bytes. A GlobalSymbolDictionary unit test pins the no-dedup contract, and a new recovery test seeds a slot with two lone-surrogate symbols and asserts the producer dictionary and sentMaxSymbolId match the persisted entry count. It fails before this change (dictionary size 1 vs the file's 2). Co-Authored-By: Claude Opus 4.8 (1M context) --- .../qwp/client/GlobalSymbolDictionary.java | 31 ++++++++ .../qwp/client/QwpWebSocketSender.java | 12 +++- .../qwp/client/DeltaDictRecoveryTest.java | 71 +++++++++++++++++++ .../client/GlobalSymbolDictionaryTest.java | 30 ++++++++ 4 files changed, 143 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/GlobalSymbolDictionary.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/GlobalSymbolDictionary.java index 3b4c9a90..d1b0f13e 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/GlobalSymbolDictionary.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/GlobalSymbolDictionary.java @@ -50,6 +50,37 @@ public GlobalSymbolDictionary(int initialCapacity) { this.idToSymbol = new ObjList<>(initialCapacity); } + /** + * Appends {@code symbol} at the next sequential id, matching a recovered / + * persisted dictionary's dense id order, WITHOUT de-duplicating. + *

+ * Recovery ({@code QwpWebSocketSender.seedGlobalDictionaryFromPersisted}) + * replays the persisted entries in id order to rebuild this dictionary. It must + * NOT collapse two source strings that decode to the same characters, because + * the persisted {@code .symbol-dict}, the on-wire delta and the I/O-thread + * catch-up mirror all key on the entry POSITION (id), not on the string. The + * only strings that collide this way are malformed lone UTF-16 surrogates, + * which the UTF-8 encoder maps to {@code '?'}: {@link #getOrAddSymbol} would + * de-dup them and leave this dictionary SHORTER than the persisted entry count, + * desyncing the producer's delta baseline from the catch-up mirror (which uses + * {@code pd.size()}) and silently misattributing later symbols. Appending + * unconditionally keeps {@link #size()} equal to that count. The reverse lookup + * keeps the highest id for a colliding string, which is harmless: both ids + * encode to the same bytes, so resolving either is equivalent. + * + * @param symbol the recovered symbol string (must not be null) + * @return the id assigned (the previous {@link #size()}) + */ + public int addRecoveredSymbol(String symbol) { + if (symbol == null) { + throw new IllegalArgumentException("symbol cannot be null"); + } + int newId = idToSymbol.size(); + symbolToId.put(symbol, newId); + idToSymbol.add(symbol); + return newId; + } + /** * Clears all symbols from the dictionary. *

diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index bb1256dd..23d1240f 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -3750,6 +3750,16 @@ private void resetSymbolDictStateForNewConnection() { * the slot's persisted dictionary (ids assigned in the same ascending order, * so they match the recovered frames) and resumes the delta baseline at the * recovered tip, so newly ingested symbols continue above the recovered ids. + *

+ * Uses {@link GlobalSymbolDictionary#addRecoveredSymbol} (append, NOT de-dup): + * the persisted dictionary, the on-wire delta and the send-loop catch-up mirror + * all key on the entry POSITION (id), so the producer id space must match the + * persisted entry count exactly. {@code getOrAddSymbol} would collapse two + * source strings that decode to the same characters -- only malformed lone + * UTF-16 surrogates, which UTF-8-encode to {@code '?'} -- leaving this + * dictionary shorter than {@code pd.size()} and desyncing + * {@code sentMaxSymbolId} from the mirror's {@code sentDictCount = pd.size()}, + * which silently misattributes later symbols after a reconnect. */ private void seedGlobalDictionaryFromPersisted(PersistedSymbolDict pd) { if (pd == null || pd.size() == 0) { @@ -3757,7 +3767,7 @@ private void seedGlobalDictionaryFromPersisted(PersistedSymbolDict pd) { } ObjList symbols = pd.readLoadedSymbols(); for (int i = 0, n = symbols.size(); i < n; i++) { - globalSymbolDictionary.getOrAddSymbol(symbols.getQuick(i)); + globalSymbolDictionary.addRecoveredSymbol(symbols.getQuick(i)); } sentMaxSymbolId = globalSymbolDictionary.size() - 1; } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java index 1439ac20..e46d0148 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java @@ -36,6 +36,7 @@ import org.junit.Test; import java.io.IOException; +import java.lang.reflect.Field; import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.nio.charset.StandardCharsets; @@ -713,6 +714,76 @@ public void testRecoveryAfterFailedPublishReplaysGapFree() throws Exception { }); } + @Test + public void testRecoverySeedKeepsUtf8CollidingSymbolsInLockstep() throws Exception { + // M2 regression: two DISTINCT source symbols that collapse to the SAME UTF-8 + // bytes -- lone UTF-16 surrogates, which the encoder maps to '?' -- get + // distinct producer ids and persist as separate entries. On recovery the + // producer must rebuild its id space to match the persisted entry count + // exactly. Pre-fix, seedGlobalDictionaryFromPersisted used getOrAddSymbol, + // which de-duped the two decoded "?" strings, leaving the producer + // dictionary (and sentMaxSymbolId) one short of pd.size() -- desyncing from + // the send-loop catch-up mirror (which uses pd.size()) and silently + // misattributing later symbols after a reconnect. addRecoveredSymbol appends + // without de-duping, keeping producer and mirror in lockstep. + assertMemoryLeak(() -> { + // Phase 1: ingest two lone-surrogate symbols in file mode, close-fast. + try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) { + int port = silent.getPort(); + silent.start(); + Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS)); + String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + + ";close_flush_timeout_millis=0;"; + try (Sender s1 = Sender.fromConfig(cfg)) { + s1.table("m").symbol("s", "\uD800").longColumn("v", 1L).atNow(); // lone surrogate -> '?' + s1.flush(); + s1.table("m").symbol("s", "\uD801").longColumn("v", 2L).atNow(); // a DIFFERENT one -> '?' + s1.flush(); + } + } + + // The persisted dictionary holds TWO entries (both encode to '?'). + int persistedSize; + PersistedSymbolDict pd = PersistedSymbolDict.open(Paths.get(sfDir, "default").toString()); + Assert.assertNotNull(pd); + try { + persistedSize = pd.size(); + } finally { + pd.close(); + } + Assert.assertEquals("two colliding symbols must persist as two entries", 2, persistedSize); + + // Phase 2: recover. The seeded producer id space must match pd.size(). + DictReconstructingHandler handler = new DictReconstructingHandler(); + try (TestWebSocketServer good = new TestWebSocketServer(handler)) { + int port = good.getPort(); + good.start(); + Assert.assertTrue(good.awaitStart(5, TimeUnit.SECONDS)); + String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";"; + try (Sender s2 = Sender.fromConfig(cfg)) { + Assert.assertEquals("recovered producer dictionary must match the persisted " + + "entry count (not de-duped), else the delta baseline desyncs from the mirror", + persistedSize, globalDictSize(s2)); + Assert.assertEquals("delta baseline must resume at the persisted tip", + persistedSize - 1, intField(s2, "sentMaxSymbolId")); + } + } + }); + } + + private static int globalDictSize(Sender sender) throws Exception { + Field f = sender.getClass().getDeclaredField("globalSymbolDictionary"); + f.setAccessible(true); + Object dict = f.get(sender); + return (int) dict.getClass().getMethod("size").invoke(dict); + } + + private static int intField(Sender sender, String name) throws Exception { + Field f = sender.getClass().getDeclaredField(name); + f.setAccessible(true); + return f.getInt(sender); + } + private static void writeAckWatermark(java.nio.file.Path path, long fsn) throws IOException { byte[] buf = new byte[16]; ByteBuffer bb = ByteBuffer.wrap(buf).order(ByteOrder.LITTLE_ENDIAN); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/GlobalSymbolDictionaryTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/GlobalSymbolDictionaryTest.java index b8945d40..2f38ab02 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/GlobalSymbolDictionaryTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/GlobalSymbolDictionaryTest.java @@ -31,6 +31,36 @@ public class GlobalSymbolDictionaryTest { + @Test + public void testAddRecoveredSymbol_appendsWithoutDeduplicating() { + // Recovery replays persisted entries in id order. Distinct source strings + // that decode to the same characters -- lone UTF-16 surrogates both + // UTF-8-encode to '?', so they read back as the string "?" -- must keep + // DISTINCT ids, so the producer id space matches the persisted entry count. + // getOrAddSymbol de-dups them; addRecoveredSymbol must not. + GlobalSymbolDictionary dedup = new GlobalSymbolDictionary(); + dedup.getOrAddSymbol("?"); + dedup.getOrAddSymbol("?"); + assertEquals("getOrAddSymbol de-dups colliding strings", 1, dedup.size()); + + GlobalSymbolDictionary recovered = new GlobalSymbolDictionary(); + assertEquals(0, recovered.addRecoveredSymbol("?")); + assertEquals(1, recovered.addRecoveredSymbol("?")); + assertEquals(2, recovered.addRecoveredSymbol("nvda")); + assertEquals("addRecoveredSymbol keeps colliding entries distinct", 3, recovered.size()); + + // Dense id -> symbol mapping is preserved position-for-position. + assertEquals("?", recovered.getSymbol(0)); + assertEquals("?", recovered.getSymbol(1)); + assertEquals("nvda", recovered.getSymbol(2)); + + // A later ingest of a colliding string reuses the highest recovered id + // (harmless -- both encode to identical bytes), and a genuinely new symbol + // continues past the recovered tip. + assertEquals(1, recovered.getOrAddSymbol("?")); + assertEquals(3, recovered.getOrAddSymbol("brand-new")); + } + @Test public void testAddSymbol_assignsSequentialIds() { GlobalSymbolDictionary dict = new GlobalSymbolDictionary(); From 0b4ab3ff1c0ac5a044da8eb49c6a69fbb7528a72 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Fri, 10 Jul 2026 18:06:40 +0100 Subject: [PATCH 32/96] Harden symbol-dict resource teardown paths Three defensive fixes on currently-unreachable teardown paths, each guarding a future edit from a native-memory hazard: - PersistedSymbolDict.close() now nulls loadedEntriesAddr/Len after freeing them (like scratchAddr), so a post-close read of the non-closed-guarded getters cannot dereference freed memory. - CursorWebSocketSendLoop resets sentDictCount alongside the buffer in both mirror-free sites, keeping the mirror all-or-nothing. start() has no closed guard, so a hypothetical close()-then-start() would otherwise observe a non-zero count against a freed buffer and drive setWireBaselineWithCatchUp into a null-mirror catch-up. - PersistedSymbolDict.openFresh() now closes the fd on any throw (the header malloc moves inside the try, plus a catch), mirroring openExisting; previously a throw in the header-write body freed the scratch buffer but leaked the fd. Tests: testCloseNullsLoadedEntries asserts the getters read 0 after close; the mirror-leak test now also asserts sentDictCount is reset. Both fail before their fix. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../sf/cursor/CursorWebSocketSendLoop.java | 6 +++++ .../client/sf/cursor/PersistedSymbolDict.java | 20 ++++++++++++-- ...CursorWebSocketSendLoopMirrorLeakTest.java | 8 ++++++ .../sf/cursor/PersistedSymbolDictTest.java | 26 +++++++++++++++++++ 4 files changed, 58 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java index bd9a54bc..7593cf47 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java @@ -907,6 +907,11 @@ public synchronized void close() { sentDictBytesAddr = 0; sentDictBytesCapacity = 0; sentDictBytesLen = 0; + // Reset the count alongside the buffer so the mirror stays all-or- + // nothing: a hypothetical close()-then-start() (start() has no closed + // guard) must not observe a non-zero sentDictCount against a freed + // buffer and drive setWireBaselineWithCatchUp into a null-mirror catch-up. + sentDictCount = 0; } } @@ -1793,6 +1798,7 @@ private void ioLoop() { sentDictBytesAddr = 0; sentDictBytesCapacity = 0; sentDictBytesLen = 0; + sentDictCount = 0; // keep the mirror all-or-nothing (see close()) } shutdownLatch.countDown(); // Failed-stop hand-off (see delegateEngineClose): the owner could diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java index 889104ec..7f557708 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java @@ -282,6 +282,11 @@ public synchronized void close() { closed = true; if (loadedEntriesAddr != 0L) { Unsafe.free(loadedEntriesAddr, loadedEntriesLen, MemoryTag.NATIVE_DEFAULT); + // Null after freeing (like scratchAddr below) so a future accessor that + // reads loadedEntriesAddr()/loadedEntriesLen() post-close cannot + // dereference freed native memory; the getters are not closed-guarded. + loadedEntriesAddr = 0L; + loadedEntriesLen = 0; } if (scratchAddr != 0L) { Unsafe.free(scratchAddr, scratchCap, MemoryTag.NATIVE_DEFAULT); @@ -437,8 +442,9 @@ private static PersistedSymbolDict openFresh(String filePath) { LOG.warn("symbol dict {} could not be created (rc={}); proceeding without it", filePath, fd); return null; } - long hdr = Unsafe.malloc(HEADER_SIZE, MemoryTag.NATIVE_DEFAULT); + long hdr = 0L; try { + hdr = Unsafe.malloc(HEADER_SIZE, MemoryTag.NATIVE_DEFAULT); Unsafe.getUnsafe().putInt(hdr, FILE_MAGIC); Unsafe.getUnsafe().putByte(hdr + 4, VERSION); Unsafe.getUnsafe().putByte(hdr + 5, (byte) 0); @@ -451,8 +457,18 @@ private static PersistedSymbolDict openFresh(String filePath) { LOG.warn("symbol dict {} header write failed; proceeding without it", filePath); return null; } + } catch (Throwable t) { + // Unreachable today (Files.write is native and returns -1 rather than + // throwing; the Unsafe puts target a valid 8-byte buffer and an 8-byte + // malloc cannot realistically OOM), but close the fd against a future + // edit so it cannot leak -- mirroring openExisting's error handling. + Files.close(fd); + LOG.warn("symbol dict {} creation failed ({}); proceeding without it", filePath, String.valueOf(t)); + return null; } finally { - Unsafe.free(hdr, HEADER_SIZE, MemoryTag.NATIVE_DEFAULT); + if (hdr != 0L) { + Unsafe.free(hdr, HEADER_SIZE, MemoryTag.NATIVE_DEFAULT); + } } return new PersistedSymbolDict(fd, HEADER_SIZE, 0, 0L, 0); } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopMirrorLeakTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopMirrorLeakTest.java index 7869882d..057e1a5f 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopMirrorLeakTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopMirrorLeakTest.java @@ -89,7 +89,15 @@ public void testSeededMirrorFreedWhenLoopClosedWithoutStart() throws Exception { 0, 0, 1); // Close without start(): the ctor-seeded mirror is this // thread's to free, since the I/O loop never ran. + Assert.assertTrue("precondition: the ctor seeded a non-empty mirror", + readInt(loop, "sentDictCount") > 0); loop.close(); + // close() must reset sentDictCount alongside freeing the buffer, + // so the mirror stays all-or-nothing: a hypothetical post-close + // start() (no closed guard) cannot read a stale count against a + // freed buffer and drive a null-mirror catch-up. + Assert.assertEquals("close() must reset sentDictCount to 0", + 0, readInt(loop, "sentDictCount")); } }); } finally { diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java index cc45c23b..aa37031b 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java @@ -228,6 +228,32 @@ public void testBadMagicIsRecreatedEmpty() throws Exception { }); } + @Test + public void testCloseNullsLoadedEntries() throws Exception { + // close() must null loadedEntriesAddr/Len after freeing them (like + // scratchAddr), so an accidental post-close read of the getters cannot + // dereference freed native memory. Pre-fix the pointer survived close() + // non-zero. + assertMemoryLeak(() -> { + Path dir = Files.createTempDirectory("qwp-symdict"); + try { + PersistedSymbolDict d = PersistedSymbolDict.open(dir.toString()); + d.appendSymbol("AAPL"); + d.close(); + + // Reopen so recovery loads the entries into native memory. + PersistedSymbolDict re = PersistedSymbolDict.open(dir.toString()); + Assert.assertTrue("recovery must load entries into native memory", + re.loadedEntriesAddr() != 0L && re.loadedEntriesLen() > 0); + re.close(); + Assert.assertEquals("close() must null loadedEntriesAddr", 0L, re.loadedEntriesAddr()); + Assert.assertEquals("close() must null loadedEntriesLen", 0, re.loadedEntriesLen()); + } finally { + rmDir(dir); + } + }); + } + @Test public void testEmptySymbolRoundTrips() throws Exception { assertMemoryLeak(() -> { From 186ae10dec23ff02e982b87a49047b33764d1a24 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Fri, 10 Jul 2026 18:15:51 +0100 Subject: [PATCH 33/96] Defer catch-up commit; fix stale delta comments Three delta symbol-dict cleanups (a fourth reported item, a dead CursorSendEngine.isMemoryMode(), was already removed earlier on this branch, so nothing to do there): - sendCatchUpChunk now sets FLAG_DEFER_COMMIT on the catch-up frame. It carries dictionary entries but no rows, so it must never trigger a server-side commit. Today it is always the first frame on a fresh (empty-transaction) connection, so committing nothing is a no-op -- but that invariant is load-bearing and was unasserted. Deferring the empty commit removes the dependency, so a future mid-stream catch-up cannot prematurely commit an in-flight deferred transaction. The dictionary delta still registers, as any deferred data frame's does. - flushPendingRows' comment claimed delta mode is "memory-mode only"; file-mode store-and-forward ships monotonic deltas too once the persisted dictionary opened. Corrected. - The deltaDictEnabled field comment implied delta mode only applies to recovered / orphan-drained disk slots; it applies to fresh disk slots too. Clarified that recovery additionally seeds the mirror. testReconnectCatchUpRebuildsDictionary now asserts the catch-up frame carries FLAG_DEFER_COMMIT; it fails without the flag. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../qwp/client/QwpWebSocketSender.java | 3 ++- .../sf/cursor/CursorWebSocketSendLoop.java | 19 ++++++++++++++++--- .../qwp/client/DeltaDictCatchUpTest.java | 8 ++++++++ 3 files changed, 26 insertions(+), 4 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index 23d1240f..5915f0fb 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -3455,7 +3455,8 @@ private void flushPendingRows(boolean deferCommit) { ensureActiveBufferReady(); // In full-dict mode every frame is self-sufficient: it carries the whole // symbol dictionary from id 0 so orphan-drain / recovery replay to a fresh - // server never dangles a symbol id. In delta mode (memory-mode only) each + // server never dangles a symbol id. In delta mode (memory-mode, and + // file-mode store-and-forward once the persisted dictionary opened) each // frame carries only ids above sentMaxSymbolId; a reconnect re-registers // the dictionary via an I/O-thread catch-up frame before replay, so the // producer's monotonic baseline stays valid across the wire boundary. diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java index 7593cf47..87d1dab0 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java @@ -187,8 +187,11 @@ public final class CursorWebSocketSendLoop implements QuietCloseable { private final WebSocketResponse response = new WebSocketResponse(); private final ResponseHandler responseHandler = new ResponseHandler(); // Delta symbol dictionary catch-up state (see swapClient). Active in memory - // mode, and in disk mode on a recovered / orphan-drained slot (the constructor - // seeds sentDict* from the persisted dictionary there). + // mode, and in disk mode whenever the per-slot persisted dictionary opened -- + // fresh slots included, not just recovered / orphan-drained ones. On a + // recovered / orphan-drained slot the constructor additionally SEEDS sentDict* + // from that persisted dictionary; a fresh slot starts with an empty mirror and + // grows it as frames are sent. // deltaDictEnabled gates all of it. The loop mirrors, in sentDict*, every // symbol it has ever sent -- the concatenated [len varint][utf8] bytes in // global-id order (sentDictBytes*) plus the count (sentDictCount) -- so that @@ -2293,8 +2296,18 @@ private void sendCatchUpChunk(int deltaStart, int deltaCount, long symbolsAddr, Unsafe.getUnsafe().putByte(frame + 2, (byte) 'P'); Unsafe.getUnsafe().putByte(frame + 3, (byte) '1'); Unsafe.getUnsafe().putByte(frame + 4, (byte) client.getServerQwpVersion()); + // FLAG_DEFER_COMMIT: the catch-up carries dictionary entries but NO + // rows, so it must never trigger a server-side commit. Today it is + // always the first frame on a fresh (empty-transaction) connection, so + // committing nothing is a no-op -- but that invariant is load-bearing + // and unasserted. Deferring the (empty) commit removes the dependency: + // a future mid-stream catch-up cannot prematurely commit an in-flight + // deferred transaction. The dictionary delta still registers (as any + // deferred data frame's does); only the row commit is deferred, and the + // next real frame commits it. Unsafe.getUnsafe().putByte(frame + QwpConstants.HEADER_OFFSET_FLAGS, - (byte) (QwpConstants.FLAG_GORILLA | QwpConstants.FLAG_DELTA_SYMBOL_DICT)); + (byte) (QwpConstants.FLAG_GORILLA | QwpConstants.FLAG_DELTA_SYMBOL_DICT + | QwpConstants.FLAG_DEFER_COMMIT)); Unsafe.getUnsafe().putShort(frame + 6, (short) 0); // tableCount Unsafe.getUnsafe().putInt(frame + 8, payloadLen); long q = writeVarintAt(frame + QwpConstants.HEADER_SIZE, deltaStart); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictCatchUpTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictCatchUpTest.java index 811c0c67..e7f27db1 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictCatchUpTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictCatchUpTest.java @@ -92,6 +92,9 @@ public void testReconnectCatchUpRebuildsDictionary() throws Exception { List conn2 = handler.dictFor(2); Assert.assertTrue("2nd connection saw a catch-up frame with 0 tables", handler.sawZeroTableFrameOnConn2); + Assert.assertTrue("the catch-up frame carries no rows, so it must defer its " + + "(empty) commit -- FLAG_DEFER_COMMIT set", + handler.catchUpDeferredOnConn2); Assert.assertEquals("2nd connection dictionary size", 2, conn2.size()); Assert.assertEquals("alpha", conn2.get(0)); Assert.assertEquals("beta", conn2.get(1)); @@ -326,6 +329,9 @@ private static class CatchUpHandler implements TestWebSocketServer.WebSocketServ // (rather than a fixed sleep) before sending batch 2, so batch 2 cannot // race into connection 1's pre-close window and must land on the reconnect. volatile boolean conn1Closed; + // Set from the flags byte of the zero-table catch-up frame on connection 2: + // the catch-up carries no rows and must defer its (empty) commit. + volatile boolean catchUpDeferredOnConn2; volatile boolean sawZeroTableFrameOnConn2; private final List> dictsByConn = new CopyOnWriteArrayList<>(); private TestWebSocketServer.ClientHandler currentClient; @@ -352,6 +358,8 @@ public synchronized void onBinaryMessage(TestWebSocketServer.ClientHandler clien accumulate(data, dict); if (connNumber == 2 && tableCount(data) == 0) { sawZeroTableFrameOnConn2 = true; + // FLAG_DEFER_COMMIT is bit 0x01 of the flags byte (offset 5). + catchUpDeferredOnConn2 = (data[5] & 0x01) != 0; } try { client.sendBinary(buildAck(nextSeq.getAndIncrement())); From fef7203593c13a1c8308b91afe081d91ee0779dd Mon Sep 17 00:00:00 2001 From: glasstiger Date: Fri, 10 Jul 2026 19:34:11 +0100 Subject: [PATCH 34/96] Plug ensureConnected leak; consolidate varint helpers Review cleanups: - ensureConnected's start()-failure catch now closes cursorSendLoop before nulling client. Previously it closed only the client; because the connected flag is set past the catch, a caller that retries -- re-entering ensureConnected and reassigning cursorSendLoop -- orphaned a recovered slot's ctor-seeded native mirror. That is a reachable leak on the recovered-slot + start()/dispatcher-OOM + retry path. close() is idempotent and frees the mirror via its loopNeverRan path, and also closes the shared client, so the following client.close() is a no-op. - Consolidated the triplicated raw-address varint writer into NativeBufferWriter.writeVarint (writeVarintDirect now delegates to it); PersistedSymbolDict and the catch-up frame builder in CursorWebSocketSendLoop use it, and PersistedSymbolDict.varintSize now reuses NativeBufferWriter.varintSize. - Alphabetized CursorSendEngine (getPersistedSymbolDict before getTotalBackpressureStalls before isDeltaDictEnabled) and moved PersistedSymbolDict.decodeVarint to the front of its private-static cluster. - readVarintAt gained the shift > 35 defensive bound that decodeVarint already has (unreachable today, all inputs are freshly-encoded or validated varints, but consistent). - CursorWebSocketSendLoopCatchUpAlignmentTest now uses the existing TestUtils.createTmpDir/removeTmpDir instead of reinventing them. Not changed: the reported dead CursorSendEngine.isMemoryMode() and a PersistedSymbolDict MAX_ENTRY_LEN constant do not exist at HEAD (the former removed earlier on this branch, the latter never present), and the NativeBufferWriter import is already correctly ordered. The buildAck/waitFor/rmDir/readVarint test helpers are duplicated across ~19 pre-existing test files, so consolidating them belongs in a dedicated repo-wide test-hygiene change rather than this feature PR. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../qwp/client/NativeBufferWriter.java | 21 ++++-- .../qwp/client/QwpWebSocketSender.java | 11 +++ .../client/sf/cursor/CursorSendEngine.java | 18 ++--- .../sf/cursor/CursorWebSocketSendLoop.java | 21 +++--- .../client/sf/cursor/PersistedSymbolDict.java | 75 +++++++------------ ...WebSocketSendLoopCatchUpAlignmentTest.java | 24 +----- 6 files changed, 77 insertions(+), 93 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/NativeBufferWriter.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/NativeBufferWriter.java index aad95f3c..cb6ebb92 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/NativeBufferWriter.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/NativeBufferWriter.java @@ -76,6 +76,21 @@ public static int varintSize(long value) { return (64 - Long.numberOfLeadingZeros(value) + 6) / 7; } + /** + * Writes {@code value} as an unsigned LEB128 varint directly at native address + * {@code addr} and returns the address just past the last byte. The canonical + * raw-address varint writer shared by the SF cursor's persisted dictionary and + * catch-up frame builder. + */ + public static long writeVarint(long addr, long value) { + while (value > 0x7F) { + Unsafe.getUnsafe().putByte(addr++, (byte) ((value & 0x7F) | 0x80)); + value >>>= 7; + } + Unsafe.getUnsafe().putByte(addr++, (byte) value); + return addr; + } + @Override public void close() { if (bufferPtr != 0) { @@ -336,11 +351,7 @@ public void skip(int bytes) { } private static void writeVarintDirect(long addr, long value) { - while (value > 0x7F) { - Unsafe.getUnsafe().putByte(addr++, (byte) ((value & 0x7F) | 0x80)); - value >>>= 7; - } - Unsafe.getUnsafe().putByte(addr, (byte) value); + writeVarint(addr, value); } private void encodeUtf8(CharSequence value, int utf8Len) { diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index 5915f0fb..d00ab872 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -3362,6 +3362,17 @@ private void ensureConnected() { cursorSendLoop.setConnectionDispatcher(connectionDispatcher); cursorSendLoop.start(); } catch (Throwable t) { + // start() (or dispatcher construction) failed after cursorSendLoop was + // assigned. Close it so a caller that retries -- re-entering + // ensureConnected and reassigning cursorSendLoop above -- cannot orphan + // a recovered slot's ctor-seeded native mirror (freed only by close() + // or the I/O loop, neither of which has run). close() is idempotent and + // frees the mirror via its loopNeverRan path; it also closes the shared + // client, so the client.close() below is a safe idempotent no-op. + if (cursorSendLoop != null) { + cursorSendLoop.close(); + cursorSendLoop = null; + } if (client != null) { client.close(); client = null; diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java index c962a56d..6c34124d 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java @@ -619,6 +619,15 @@ public MmapSegment firstSealed() { return ring.firstSealed(); } + /** + * The engine's persisted symbol dictionary, or {@code null} in memory mode + * (and in disk mode if it failed to open). The producer appends new symbols + * to it; recovery / orphan-drain read its loaded entries to seed catch-up. + */ + public PersistedSymbolDict getPersistedSymbolDict() { + return persistedSymbolDict; + } + /** * Number of times {@link #appendBlocking} hit * {@link SegmentRing#BACKPRESSURE_NO_SPARE} on its first attempt and @@ -641,15 +650,6 @@ public boolean isDeltaDictEnabled() { return sfDir == null || persistedSymbolDict != null; } - /** - * The engine's persisted symbol dictionary, or {@code null} in memory mode - * (and in disk mode if it failed to open). The producer appends new symbols - * to it; recovery / orphan-drain read its loaded entries to seed catch-up. - */ - public PersistedSymbolDict getPersistedSymbolDict() { - return persistedSymbolDict; - } - /** * Pass-through to {@link SegmentRing#nextSealedAfter(MmapSegment)}. */ diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java index 87d1dab0..a855cc50 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java @@ -2161,6 +2161,14 @@ private long readVarintAt(long p, long limit) { break; } shift += 7; + if (shift > 35) { + // Defensive bound, matching PersistedSymbolDict.decodeVarint: a + // canonical entry-length / delta varint is <= 5 bytes. Every caller + // reads freshly-encoded, CRC- or openExisting-validated bytes, so + // this is unreachable, but it stops a corrupt continuation run from + // over-shifting into a garbage length. + break; + } } varintEnd = cur; return value; @@ -2310,8 +2318,8 @@ private void sendCatchUpChunk(int deltaStart, int deltaCount, long symbolsAddr, | QwpConstants.FLAG_DEFER_COMMIT)); Unsafe.getUnsafe().putShort(frame + 6, (short) 0); // tableCount Unsafe.getUnsafe().putInt(frame + 8, payloadLen); - long q = writeVarintAt(frame + QwpConstants.HEADER_SIZE, deltaStart); - q = writeVarintAt(q, deltaCount); + long q = NativeBufferWriter.writeVarint(frame + QwpConstants.HEADER_SIZE, deltaStart); + q = NativeBufferWriter.writeVarint(q, deltaCount); Unsafe.getUnsafe().copyMemory(symbolsAddr, q, symbolsLen); client.sendBinary(frame, frameLen); } catch (Throwable t) { @@ -2331,15 +2339,6 @@ private void sendCatchUpChunk(int deltaStart, int deltaCount, long symbolsAddr, totalFramesSent.incrementAndGet(); } - private static long writeVarintAt(long addr, long value) { - while (value > 0x7F) { - Unsafe.getUnsafe().putByte(addr++, (byte) ((value & 0x7F) | 0x80)); - value >>>= 7; - } - Unsafe.getUnsafe().putByte(addr++, (byte) value); - return addr; - } - private boolean tryReceiveAcks() { boolean any = false; try { diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java index 7f557708..f00acbd4 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java @@ -25,6 +25,7 @@ package io.questdb.client.cutlass.qwp.client.sf.cursor; import io.questdb.client.cutlass.qwp.client.GlobalSymbolDictionary; +import io.questdb.client.cutlass.qwp.client.NativeBufferWriter; import io.questdb.client.std.Files; import io.questdb.client.std.MemoryTag; import io.questdb.client.std.ObjList; @@ -218,10 +219,10 @@ public synchronized void appendSymbol(CharSequence symbol) { return; } int utf8Len = Utf8s.utf8Bytes(symbol); - int varLen = varintSize(utf8Len); + int varLen = NativeBufferWriter.varintSize(utf8Len); int recLen = varLen + utf8Len; ensureScratch(recLen); - long p = writeVarint(scratchAddr, utf8Len); + long p = NativeBufferWriter.writeVarint(scratchAddr, utf8Len); if (utf8Len > 0) { Utf8s.strCpyUtf8(symbol, p, utf8Len); } @@ -257,9 +258,9 @@ public synchronized void appendSymbols(GlobalSymbolDictionary dict, int from, in for (int id = from; id <= to; id++) { CharSequence symbol = dict.getSymbol(id); int utf8Len = Utf8s.utf8Bytes(symbol); - int recLen = varintSize(utf8Len) + utf8Len; + int recLen = NativeBufferWriter.varintSize(utf8Len) + utf8Len; ensureScratch(len + recLen); - long p = writeVarint(scratchAddr + len, utf8Len); + long p = NativeBufferWriter.writeVarint(scratchAddr + len, utf8Len); if (utf8Len > 0) { Utf8s.strCpyUtf8(symbol, p, utf8Len); } @@ -351,6 +352,30 @@ public int size() { return size; } + /** + * Decodes an unsigned LEB128 varint from {@code buf[pos..limit)}. Returns + * {@code [value, newPos]} or {@code null} if the varint is truncated + * (torn tail). + */ + private static long[] decodeVarint(long buf, int pos, int limit) { + long value = 0; + int shift = 0; + int cur = pos; + while (cur < limit) { + byte b = Unsafe.getUnsafe().getByte(buf + cur); + cur++; + value |= (long) (b & 0x7F) << shift; + if ((b & 0x80) == 0) { + return new long[]{value, cur}; + } + shift += 7; + if (shift > 35) { + return null; // implausible for an entry length; treat as torn + } + } + return null; + } + private static PersistedSymbolDict openExisting(String filePath, long fileLen) { int fd = Files.openRW(filePath); if (fd < 0) { @@ -473,48 +498,6 @@ private static PersistedSymbolDict openFresh(String filePath) { return new PersistedSymbolDict(fd, HEADER_SIZE, 0, 0L, 0); } - /** - * Decodes an unsigned LEB128 varint from {@code buf[pos..limit)}. Returns - * {@code [value, newPos]} or {@code null} if the varint is truncated - * (torn tail). - */ - private static long[] decodeVarint(long buf, int pos, int limit) { - long value = 0; - int shift = 0; - int cur = pos; - while (cur < limit) { - byte b = Unsafe.getUnsafe().getByte(buf + cur); - cur++; - value |= (long) (b & 0x7F) << shift; - if ((b & 0x80) == 0) { - return new long[]{value, cur}; - } - shift += 7; - if (shift > 35) { - return null; // implausible for an entry length; treat as torn - } - } - return null; - } - - private static int varintSize(long value) { - int n = 1; - while (value > 0x7F) { - value >>>= 7; - n++; - } - return n; - } - - private static long writeVarint(long addr, long value) { - while (value > 0x7F) { - Unsafe.getUnsafe().putByte(addr++, (byte) ((value & 0x7F) | 0x80)); - value >>>= 7; - } - Unsafe.getUnsafe().putByte(addr++, (byte) value); - return addr; - } - private void ensureScratch(int required) { if (scratchCap >= required) { return; diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java index 79640f25..3814d826 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java @@ -30,7 +30,6 @@ import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine; import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorWebSocketSendLoop; import io.questdb.client.network.PlainSocketFactory; -import io.questdb.client.std.Files; import io.questdb.client.std.MemoryTag; import io.questdb.client.std.Unsafe; import io.questdb.client.test.tools.TestUtils; @@ -42,7 +41,6 @@ import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.nio.charset.StandardCharsets; -import java.nio.file.Paths; import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -78,30 +76,12 @@ public class CursorWebSocketSendLoopCatchUpAlignmentTest { @Before public void setUp() { - tmpDir = Paths.get(System.getProperty("java.io.tmpdir"), - "qdb-cursor-catchup-" + System.nanoTime()).toString(); - assertEquals(0, Files.mkdir(tmpDir, Files.DIR_MODE_DEFAULT)); + tmpDir = TestUtils.createTmpDir("qdb-cursor-catchup-"); } @After public void tearDown() { - if (tmpDir == null) return; - long find = Files.findFirst(tmpDir); - if (find > 0) { - try { - int rc = 1; - while (rc > 0) { - String name = Files.utf8ToString(Files.findName(find)); - if (name != null && !".".equals(name) && !"..".equals(name)) { - Files.remove(tmpDir + "/" + name); - } - rc = Files.findNext(find); - } - } finally { - Files.findClose(find); - } - } - Files.remove(tmpDir); + TestUtils.removeTmpDir(tmpDir); } @Test From cbaf474c270d16a45f8041cfdc03fe40855f11a6 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Fri, 10 Jul 2026 21:18:12 +0100 Subject: [PATCH 35/96] Add a per-entry CRC to the SF symbol dictionary The .symbol-dict side-file was page-cache durable but carried no integrity check, so a host-crash interior tear -- a lost page reading back as zeroes, or a stale entry left past the end by a failed truncate -- could shift the dense id->symbol map without changing the entry count the send-loop replay guard checks, silently misattributing symbol-column values on recovery. The SF segment frames are CRC-32C protected; the dictionary they depend on was not. Each entry now carries a trailing CRC-32C over its [len][utf8] bytes, the same checksum MmapSegment uses. openExisting verifies every entry and stops at the first torn or mismatched one, so recovery trusts only the intact prefix and the guard then forces a resend of the rest -- turning a silent misattribution into a fail-clean "resend required". open() strips the CRCs into the wire-shaped loaded-entries buffer, so the catch-up mirror seed and appendRawEntries keep the file==wire contract and the send loop stays untouched. The tail truncate is now failure-checked: a file open() cannot trim is untrusted and recreated empty rather than left exposing stale bytes. Bumps the on-disk format to v2 with a version check; existing files are recreated. A tear that leaves a CRC-matching byte run is still undetected, a 1-in-2^32 collision per entry, no weaker than the frames. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../client/sf/cursor/PersistedSymbolDict.java | 246 ++++++++++++------ .../sf/cursor/PersistedSymbolDictTest.java | 56 ++++ 2 files changed, 222 insertions(+), 80 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java index f00acbd4..6bdc8162 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java @@ -26,6 +26,7 @@ import io.questdb.client.cutlass.qwp.client.GlobalSymbolDictionary; import io.questdb.client.cutlass.qwp.client.NativeBufferWriter; +import io.questdb.client.std.Crc32c; import io.questdb.client.std.Files; import io.questdb.client.std.MemoryTag; import io.questdb.client.std.ObjList; @@ -53,12 +54,15 @@ * Layout (little-endian): * 

  *   offset 0: u32 magic = 'SYD1'
- *   offset 4: u8  version = 1
+ *   offset 4: u8  version = 2
  *   offset 5: 3 bytes reserved (zero)
- *   offset 8: entries, each [len: varint][utf8 bytes], in ascending global-id order
+ *   offset 8: entries, each [len: varint][utf8 bytes][crc32c: u32], in ascending global-id order
  *
* Symbol id {@code i} is the {@code i}-th entry (ids are dense and assigned - * sequentially from 0), so no id needs to be stored. + * sequentially from 0), so no id needs to be stored. Each entry carries a + * CRC-32C over its {@code [len][utf8]} bytes (the same checksum the SF segment + * frames use), so a torn or stale entry is detected on recovery instead of + * being silently mis-parsed. *

* Durability / write-ahead ordering: the producer appends the symbols a * frame introduces BEFORE that frame is published to the ring, but does NOT @@ -68,18 +72,27 @@ * dictionary is a superset of every recoverable frame's references. It is NOT * sufficient for a host/power crash, where unflushed pages can be lost out * of order and the dictionary may end up torn relative to the frames it serves -- - * exactly as the segment frames themselves may be lost on a host crash. The send - * loop's replay guard catches the COMMON host-crash outcome -- a truncated tail, - * where a frame's delta start id exceeds the recovered dictionary size -- and - * fails loudly (the unreplayable data must be resent) rather than sending a - * gapped frame. It does NOT catch every host-crash tear: an interior page lost - * out of order reads back as zeroes that parse as empty-string entries, and a - * failed best-effort truncate (see {@link #open}) can leave a stale trailing - * entry -- either SHIFTS the dense id->symbol mapping without changing the entry - * count the guard checks, so a replay silently misattributes symbols. That - * residual exposure sits within the "not host-crash durable" boundary above; a - * per-entry or running CRC would be needed to close it (the 3 reserved header - * bytes leave room for a future integrity field). + * exactly as the segment frames themselves may be lost on a host crash. Two + * layers keep a host-crash tear from silently corrupting data: + *

    + *
  • The per-entry CRC-32C: {@link #open} verifies every entry and stops at + * the first one whose checksum fails, so an interior page lost out of + * order (reading back as zeroes) or a stale entry left past the end by a + * failed truncate is DETECTED and the trusted region ends before it -- + * recovery never mis-parses a corrupt entry as a real symbol nor shifts + * the dense id->symbol map. The truncate that drops a torn/stale tail is + * now failure-checked (see {@link #open}): a file that cannot be trimmed + * is untrusted and recreated empty rather than left exposing stale bytes.
    • + *
  • The send loop's replay guard: once recovery trusts only the intact + * prefix, a surviving frame whose delta start id exceeds that prefix + * fails loudly (the unreplayable data must be resent) rather than sending + * a gapped frame.
    • + *
+ * Together these turn every detectable host-crash tear into a fail-clean + * "resend required" instead of a silent symbol misattribution -- the same + * CRC-32C protection the segment frames carry. A tear that happened to leave a + * byte run whose CRC still matches is not distinguished, but that is a 1-in-2^32 + * collision per corrupted entry, no weaker than the frames' own checksum. *

* A torn trailing entry from a crash mid-append is self-healing: {@link #open} * stops parsing at the first incomplete entry and the next append overwrites it. @@ -101,9 +114,10 @@ public final class PersistedSymbolDict implements QuietCloseable { * OrphanScanner, trim) skip it automatically. */ public static final String FILE_NAME = ".symbol-dict"; + static final int CRC_SIZE = 4; // u32 CRC-32C trailing every entry static final int FILE_MAGIC = 0x31445953; // 'SYD1' little-endian static final int HEADER_SIZE = 8; - static final byte VERSION = 1; + static final byte VERSION = 2; // v2 appended the per-entry CRC-32C private static final Logger LOG = LoggerFactory.getLogger(PersistedSymbolDict.class); private final int fd; private long appendOffset; @@ -184,26 +198,56 @@ public static void removeOrphan(String slotDir) { } /** - * Appends {@code count} already-encoded entries -- {@code [len varint][utf8]...}, - * the exact layout {@link #appendSymbols} produces -- verbatim from {@code addr} - * in a single write. The producer uses this when it already holds those bytes - * (the symbol-dict delta section the frame encoder just wrote), so it does not - * re-encode the same symbols. Writes {@code len} bytes and advances {@code size} - * by {@code count}. Same durability/idempotency contract as {@link #appendSymbols}: - * no fsync, and a short write throws WITHOUT advancing {@code size}/{@code - * appendOffset}, so a retry keyed off {@link #size()} re-persists the same range - * at the same offset. No-op when the range is empty or the dictionary is closed. + * Appends {@code count} wire entries -- {@code [len varint][utf8]...}, the + * symbol-dict delta section the frame encoder just wrote -- to the on-disk + * dictionary, computing and appending a per-entry CRC-32C as it copies so the + * producer does not re-encode the symbols. The on-disk layout is + * {@code [len varint][utf8][crc32c]} per entry (see the class layout note); the + * {@code addr}/{@code len} bytes carry no CRC, so this walks the {@code count} + * entries to insert one. Advances {@code size} by {@code count}. Same + * durability/idempotency contract as {@link #appendSymbols}: no fsync, and a + * short write throws WITHOUT advancing {@code size}/{@code appendOffset}, so a + * retry keyed off {@link #size()} re-persists the same range at the same + * offset. No-op when the range is empty or the dictionary is closed. */ public synchronized void appendRawEntries(long addr, int len, int count) { if (closed || count <= 0 || len <= 0) { return; } - long written = Files.write(fd, addr, len, appendOffset); - if (written != len) { + int outLen = len + count * CRC_SIZE; + ensureScratch(outLen); + long src = addr; + long srcLimit = addr + len; + long dst = scratchAddr; + for (int i = 0; i < count; i++) { + long entryStart = src; + long symLen = 0; + int shift = 0; + while (src < srcLimit) { + byte b = Unsafe.getUnsafe().getByte(src++); + symLen |= (long) (b & 0x7F) << shift; + if ((b & 0x80) == 0) { + break; + } + shift += 7; + } + long entryEnd = src + symLen; // src is just past the len varint + if (entryEnd > srcLimit) { + throw new IllegalStateException("malformed raw symbol-dict entries to " + FILE_NAME + + " [entry=" + i + ", count=" + count + ']'); + } + int wireSpan = (int) (entryEnd - entryStart); // [len][utf8] + Unsafe.getUnsafe().copyMemory(entryStart, dst, wireSpan); + Unsafe.getUnsafe().putInt(dst + wireSpan, Crc32c.update(Crc32c.INIT, entryStart, wireSpan)); + dst += wireSpan + CRC_SIZE; + src = entryEnd; + } + long written = Files.write(fd, scratchAddr, outLen, appendOffset); + if (written != outLen) { throw new IllegalStateException("short write to " + FILE_NAME - + " [expected=" + len + ", actual=" + written + ']'); + + " [expected=" + outLen + ", actual=" + written + ']'); } - appendOffset += len; + appendOffset += outLen; size += count; } @@ -212,7 +256,9 @@ public synchronized void appendRawEntries(long addr, int len, int count) { * frame's new symbols BEFORE publishing that frame, so the write ordering * (dictionary entry before referencing frame) holds; no fsync is performed * (see the class-level durability note). Assigns the next dense id implicitly - * (the entry's position). + * (the entry's position). Writes {@code [len varint][utf8][crc32c]}, the CRC + * covering the {@code [len][utf8]} bytes so a torn/stale entry is detected on + * recovery. */ public synchronized void appendSymbol(CharSequence symbol) { if (closed) { @@ -220,12 +266,14 @@ public synchronized void appendSymbol(CharSequence symbol) { } int utf8Len = Utf8s.utf8Bytes(symbol); int varLen = NativeBufferWriter.varintSize(utf8Len); - int recLen = varLen + utf8Len; + int wireLen = varLen + utf8Len; // [len][utf8] + int recLen = wireLen + CRC_SIZE; // + trailing crc ensureScratch(recLen); long p = NativeBufferWriter.writeVarint(scratchAddr, utf8Len); if (utf8Len > 0) { Utf8s.strCpyUtf8(symbol, p, utf8Len); } + Unsafe.getUnsafe().putInt(scratchAddr + wireLen, Crc32c.update(Crc32c.INIT, scratchAddr, wireLen)); long written = Files.write(fd, scratchAddr, recLen, appendOffset); if (written != recLen) { throw new IllegalStateException("short write to " + FILE_NAME @@ -237,11 +285,12 @@ public synchronized void appendSymbol(CharSequence symbol) { /** * Appends the dense id range {@code [from .. to]} in a SINGLE write. Encodes - * the whole {@code [len varint][utf8]...} region into scratch first, then - * issues one positioned write -- versus one {@code pwrite(2)} per symbol via - * {@link #appendSymbol}. That per-symbol syscall count is the hot-path cost - * on a high-cardinality batch (one new symbol per row), which is exactly the - * store-and-forward workload delta encoding targets. Callers pass the + * the whole {@code [len varint][utf8][crc32c]...} region into scratch first, + * then issues one positioned write -- versus one {@code pwrite(2)} per symbol + * via {@link #appendSymbol}. That per-symbol syscall count is the hot-path + * cost on a high-cardinality batch (one new symbol per row), which is exactly + * the store-and-forward workload delta encoding targets. Each entry carries a + * trailing CRC-32C over its {@code [len][utf8]} bytes. Callers pass the * dictionary and the range so the ids resolve to their symbol strings. *

* Same durability and idempotency contract as {@link #appendSymbol}: no @@ -258,13 +307,15 @@ public synchronized void appendSymbols(GlobalSymbolDictionary dict, int from, in for (int id = from; id <= to; id++) { CharSequence symbol = dict.getSymbol(id); int utf8Len = Utf8s.utf8Bytes(symbol); - int recLen = NativeBufferWriter.varintSize(utf8Len) + utf8Len; - ensureScratch(len + recLen); - long p = NativeBufferWriter.writeVarint(scratchAddr + len, utf8Len); + int wireLen = NativeBufferWriter.varintSize(utf8Len) + utf8Len; // [len][utf8] + ensureScratch(len + wireLen + CRC_SIZE); + long entryStart = scratchAddr + len; + long p = NativeBufferWriter.writeVarint(entryStart, utf8Len); if (utf8Len > 0) { Utf8s.strCpyUtf8(symbol, p, utf8Len); } - len += recLen; + Unsafe.getUnsafe().putInt(entryStart + wireLen, Crc32c.update(Crc32c.INIT, entryStart, wireLen)); + len += wireLen + CRC_SIZE; } long written = Files.write(fd, scratchAddr, len, appendOffset); if (written != len) { @@ -389,61 +440,96 @@ private static PersistedSymbolDict openExisting(String filePath, long fileLen) { int len = (int) fileLen; buf = Unsafe.malloc(len, MemoryTag.NATIVE_DEFAULT); long read = Files.read(fd, buf, len, 0); - if (read != len || Unsafe.getUnsafe().getInt(buf) != FILE_MAGIC) { - LOG.warn("symbol dict {} unreadable or bad magic; recreating", filePath); + if (read != len + || Unsafe.getUnsafe().getInt(buf) != FILE_MAGIC + || Unsafe.getUnsafe().getByte(buf + 4) != VERSION) { + LOG.warn("symbol dict {} unreadable, bad magic or unknown version; recreating", filePath); Unsafe.free(buf, len, MemoryTag.NATIVE_DEFAULT); Files.close(fd); return null; } - // Parse complete entries starting after the header; stop at the - // first torn/incomplete trailing entry. - int pos = HEADER_SIZE; + // Parse complete, CRC-valid entries after the header; stop at the first + // torn/incomplete OR crc-mismatched entry. The CRC turns an interior + // tear (a lost page reading back as zeroes) or a stale entry left past + // the end by a failed truncate into a clean stop point, so recovery + // trusts only the intact prefix instead of silently mis-parsing a + // corrupt entry and shifting the dense id->symbol map. + int diskPos = HEADER_SIZE; // walks the on-disk [len][utf8][crc] entries int count = 0; - while (pos < len) { - long[] vr = decodeVarint(buf, pos, len); + int wireLen = 0; // running size of the crc-stripped copy + while (diskPos < len) { + long[] vr = decodeVarint(buf, diskPos, len); if (vr == null) { break; // torn length varint } - long entryLen = vr[0]; - int next = (int) vr[1]; - // The file length is the sole sanity bound: a length that runs past - // the file is a torn/incomplete trailing entry and stops the parse - // (self-healing tail). There is deliberately NO fixed per-entry - // ceiling -- the write path (appendSymbol/appendSymbols) applies none - // either, so a legitimately persisted large symbol must recover here, - // not be truncated and then trip the send loop's "resend required" - // guard on a normal process-crash recovery. entryLen stays a long so a - // corrupt multi-gigabyte length cannot wrap an int back under the check. - if ((long) next + entryLen > len) { - break; // torn/incomplete trailing entry -- self-healing tail + long symLen = vr[0]; + int afterVar = (int) vr[1]; + // [len varint][utf8] then a u32 CRC. symLen stays a long so a + // corrupt multi-gigabyte length cannot wrap an int back under the + // bound check. No fixed per-entry ceiling -- the write path applies + // none, so a legitimately large symbol must recover here. + long wireEnd = (long) afterVar + symLen; // end of [len][utf8] + if (wireEnd + CRC_SIZE > len) { + break; // torn/incomplete trailing entry (its CRC doesn't fit) + } + int wireEndI = (int) wireEnd; + int wireSpan = wireEndI - diskPos; + int crcStored = Unsafe.getUnsafe().getInt(buf + wireEndI); + int crcCalc = Crc32c.update(Crc32c.INIT, buf + diskPos, wireSpan); + if (crcCalc != crcStored) { + break; // corrupt/stale entry -- stop before it (fail-clean) } - pos = next + (int) entryLen; + diskPos = wireEndI + CRC_SIZE; + wireLen += wireSpan; count++; } - entriesLen = pos - HEADER_SIZE; - if (entriesLen > 0) { - entriesAddr = Unsafe.malloc(entriesLen, MemoryTag.NATIVE_DEFAULT); - Unsafe.getUnsafe().copyMemory(buf + HEADER_SIZE, entriesAddr, entriesLen); + int diskConsumed = diskPos - HEADER_SIZE; // valid entries incl. their CRCs + // Materialise the trusted entries as WIRE bytes ([len][utf8]..., no + // CRC) so loadedEntries*/readLoadedSymbols and the send-loop catch-up + // mirror stay wire-shaped -- the on-disk CRC is stripped here, once, at + // open. A second no-alloc walk over the already-validated region. + if (wireLen > 0) { + entriesAddr = Unsafe.malloc(wireLen, MemoryTag.NATIVE_DEFAULT); + long dst = entriesAddr; + int p = HEADER_SIZE; + for (int i = 0; i < count; i++) { + int vp = p; + long symLen = 0; + int shift = 0; + while (true) { + byte b = Unsafe.getUnsafe().getByte(buf + vp++); + symLen |= (long) (b & 0x7F) << shift; + if ((b & 0x80) == 0) { + break; + } + shift += 7; + } + int wireSpan = (vp - p) + (int) symLen; // [len][utf8], no CRC + Unsafe.getUnsafe().copyMemory(buf + p, dst, wireSpan); + dst += wireSpan; + p += wireSpan + CRC_SIZE; // skip the entry's CRC + } } + entriesLen = wireLen; Unsafe.free(buf, len, MemoryTag.NATIVE_DEFAULT); buf = 0L; - // Physically drop any torn/stale trailing bytes (a crash mid-append) - // so a LATER, shorter append cannot leave residue past its own end - // that a future recovery mis-parses as a ghost symbol -- which would - // shift every subsequent dense id. Without this, appendOffset already - // lands just past the last complete entry, so the next append - // overwrites FROM there, but only truncation guarantees nothing - // survives BEYOND it. Best-effort: a failed truncate (rare -- an I/O - // error) leaves a latent silent-corruption path, not merely a lost - // optimization: if a later, shorter append then leaves stale bytes past - // its end, a subsequent recovery mis-parses them as a ghost symbol and - // shifts the dense ids, and the count-based replay guard does not catch - // it (see the class-level durability note). - long validLen = HEADER_SIZE + entriesLen; - if (validLen < len) { - Files.truncate(fd, validLen); + // Drop any torn/stale trailing bytes so a LATER, shorter append cannot + // leave residue past its own end. Unlike before, the truncate result IS + // checked: a file we cannot trim could still expose stale post-end bytes + // whose (self-consistent) per-entry CRC the parse would accept at a + // shifted position, so a failed truncate makes the file untrusted -- + // open() then recreates it empty (fail-clean) rather than risk a silent + // misattribution. + long validLen = HEADER_SIZE + diskConsumed; + if (validLen < len && !Files.truncate(fd, validLen)) { + LOG.warn("symbol dict {} could not drop its torn/stale tail (truncate failed); recreating", filePath); + if (entriesAddr != 0L) { + Unsafe.free(entriesAddr, entriesLen, MemoryTag.NATIVE_DEFAULT); + } + Files.close(fd); + return null; } - return new PersistedSymbolDict(fd, HEADER_SIZE + entriesLen, count, entriesAddr, entriesLen); + return new PersistedSymbolDict(fd, validLen, count, entriesAddr, entriesLen); } catch (Throwable t) { if (buf != 0L) { Unsafe.free(buf, (int) fileLen, MemoryTag.NATIVE_DEFAULT); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java index aa37031b..cfcf4cf5 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java @@ -281,6 +281,62 @@ public void testEmptySymbolRoundTrips() throws Exception { }); } + @Test + public void testInteriorCorruptionIsCaughtNotSilentlyMisattributed() throws Exception { + // A host-crash interior tear (a lost page reading back as zeroes) or a + // stale entry left past the end by a failed truncate can change the bytes + // of a NON-trailing entry. Without the per-entry CRC the parse would + // accept those bytes, shifting the dense id->symbol map and silently + // misattributing symbol-column values on replay. With the CRC the corrupt + // entry fails verification and the parse stops there, so recovery trusts + // only the intact prefix (fail-clean: the send loop's torn-dict guard then + // forces a resend of the rest). + assertMemoryLeak(() -> { + Path dir = Files.createTempDirectory("qwp-symdict"); + try { + PersistedSymbolDict d = PersistedSymbolDict.open(dir.toString()); + try { + d.appendSymbol("s0"); + d.appendSymbol("s1"); + d.appendSymbol("s2"); + d.appendSymbol("s3"); + d.appendSymbol("s4"); + Assert.assertEquals(5, d.size()); + } finally { + d.close(); + } + + // Corrupt one byte inside the 3rd entry's UTF-8 (id 2). On-disk + // entry layout is [len varint][utf8][crc32c u32]; a 2-byte ASCII + // symbol is 1 + 2 + 4 = 7 bytes, after the 8-byte header: + // header[0,8) e0[8,15) e1[15,22) e2[22,29) ... + // Offset 23 is "s2"'s first UTF-8 byte; flipping it leaves e2's + // stored CRC stale. + Path f = dir.resolve(".symbol-dict"); + byte[] bytes = Files.readAllBytes(f); + bytes[23] ^= 0x7F; + Files.write(f, bytes); + + PersistedSymbolDict re = PersistedSymbolDict.open(dir.toString()); + Assert.assertNotNull(re); + try { + // Only the intact prefix [s0, s1] is trusted; the corrupt e2 + // and everything after it are dropped. No recovered symbol is + // the corrupted string -- the tear is DETECTED, never silently + // misattributed. + Assert.assertEquals("parse must stop at the corrupt interior entry", 2, re.size()); + ObjList s = re.readLoadedSymbols(); + Assert.assertEquals("s0", s.getQuick(0)); + Assert.assertEquals("s1", s.getQuick(1)); + } finally { + re.close(); + } + } finally { + rmDir(dir); + } + }); + } + @Test public void testLargeSymbolRoundTripsAcrossReopen() throws Exception { // C1 regression: the write path caps nothing, so a symbol larger than the From 6cc1307e4b1421e422be8f853c612d4059a4d7d0 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Fri, 10 Jul 2026 21:48:52 +0100 Subject: [PATCH 36/96] Document split-flush and dict-getter contracts Two documentation-only additions to the QWP store-and-forward client, making previously implicit contracts explicit. No behavior change. flushPendingRowsSplit: add a "Not atomic across frames" note. A sealAndSwapBuffer() failure on split frame k>1 leaves frames 1..k-1 published as deferred; the throw skips the trailing table-buffer reset, so the next flush re-emits the whole batch and the eventual commit commits the already-published prefix twice. This is pre-existing and within store-and-forward's at-least-once contract (absorbed by a DEDUP table or a durable-ack await); the note names the atomic-split fix (roll back or skip the published prefix on retry) as the larger follow-up. PersistedSymbolDict: mark loadedEntriesAddr(), loadedEntriesLen() and readLoadedSymbols() as construction-phase only. They read the native entry region that close() frees and nulls, with no closed-guard, so they are safe only before the slot's I/O thread and any producer append start; reading them from a running thread would risk a use-after-free. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../cutlass/qwp/client/QwpWebSocketSender.java | 16 ++++++++++++++++ .../client/sf/cursor/PersistedSymbolDict.java | 15 ++++++++++++++- 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index d00ab872..39db7d41 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -3526,6 +3526,22 @@ private void flushPendingRows(boolean deferCommit) { * own message. All messages except the last carry FLAG_DEFER_COMMIT * so the server appends rows without committing until the final * message arrives. + *

+ * Not atomic across frames. The frames publish one at a time, so a + * publish failure partway through -- {@link #sealAndSwapBuffer()} throwing on + * frame k>1, e.g. a backpressure deadline or the buffer-recycle timeout -- + * leaves frames 1..k-1 already on the ring as deferred (appended, not yet + * committed). The throw propagates past the {@code resetTableBuffersAfterFlush} + * at the end of the loop, so the source rows survive in their table buffers + * and the NEXT flush re-emits the whole batch; the eventual commit then + * commits the already-published prefix alongside the re-sent copies, + * delivering those rows at-least-once (duplicated), not exactly-once. This is + * within store-and-forward's at-least-once contract -- a DEDUP table or a + * durable-ack await absorbs the duplicate, and the symbol-dict state stays + * consistent on the retry (the re-sent frames carry empty deltas and the + * write-ahead persist is a {@code pd.size()} no-op). Making the split atomic + * (rolling back the published prefix, or skipping it on retry) would be a + * larger change. * * @param deferCommit when true, ALL messages (including the last) * carry FLAG_DEFER_COMMIT. When false, only the diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java index 6bdc8162..bb7884bd 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java @@ -354,13 +354,22 @@ public synchronized void close() { * Base address of the loaded entry region -- the concatenated * {@code [len][utf8]} bytes of every recovered symbol in id order, exactly * as a delta section carries them. Zero when nothing was recovered. + *

+ * Construction-phase only. This hands out a raw pointer into native + * memory that {@link #close()} frees and nulls, with no closed-guard and no + * synchronization. It is safe to read only BEFORE the slot's I/O thread and + * any producer append start -- i.e. while the send loop is being constructed + * or an orphan-drain is seeding its mirror, both of which happen-before those + * threads. A caller that reads it from a running thread races {@code close()} + * and can dereference freed memory (use-after-free). */ public long loadedEntriesAddr() { return loadedEntriesAddr; } /** - * Byte length of {@link #loadedEntriesAddr()}. + * Byte length of {@link #loadedEntriesAddr()}. Construction-phase only, for + * the same reason -- see {@link #loadedEntriesAddr()}. */ public int loadedEntriesLen() { return loadedEntriesLen; @@ -371,6 +380,10 @@ public int loadedEntriesLen() { * (entry {@code i} is symbol id {@code i}). Used once on recovery to * repopulate the producer's global dictionary. Empty when nothing was * recovered. + *

+ * Construction-phase only -- like {@link #loadedEntriesAddr()}, this + * walks the native entry region {@link #close()} frees, with no closed-guard, + * so it must run before the I/O thread and any producer append start. */ public ObjList readLoadedSymbols() { ObjList out = new ObjList<>(Math.max(size, 1)); From c065ad40c97d9ad6844b17d92815124ebc1337ea Mon Sep 17 00:00:00 2001 From: glasstiger Date: Fri, 10 Jul 2026 23:46:36 +0100 Subject: [PATCH 37/96] Stop split-flush stranding a deferred prefix flushPendingRowsSplit published each table's frame one at a time (deferred, uncommitted) and only checked a frame against the server batch cap after encoding it. An oversized non-first table therefore threw after an earlier table's frame was already on the ring, where a later commit delivered it as a partial batch -- while resetTableBuffersAfterFlush discarded every source row. The caller saw a failure for a batch that had partly committed. Pre-flight every split frame's size before publishing any of them, so the split is all-or-nothing: either every frame fits and all publish, or none publish and it throws with nothing stranded. simBaseline mirrors advanceSentMaxSymbolId, so a measured size equals the frame the publish loop builds; the pass is read-only on the table buffers and touches no delta/persist state. The cost is a second encode pass over the split batch, already the exceptional large-batch path. testOversizedTableSplitStrandsNothing reproduces the stranding -- it fails on the old code with the earlier frame reaching the server -- and passes with the pre-flight. Also add testReconnectPreservesMonotonicDeltaBaseline: it pins that the producer's sent-symbol watermark survives a reconnect, asserting the first post-reconnect data frame carries a delta above the baseline (deltaStart >= 1) rather than re-shipping the whole dictionary from 0. The existing catch-up tests assert only that the final dictionary is complete, which a reset-then-redefine also satisfies. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../qwp/client/QwpWebSocketSender.java | 55 +++++++++++--- .../qwp/client/DeltaDictCatchUpTest.java | 75 ++++++++++++++++++- .../qwp/client/SelfSufficientFramesTest.java | 59 +++++++++++++++ 3 files changed, 174 insertions(+), 15 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index 39db7d41..2fd785a6 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -3552,16 +3552,48 @@ private void flushPendingRowsSplit(ObjList keys, boolean deferComm LOG.debug("Splitting flush across multiple messages [serverMaxBatchSize={}, defer={}]", serverMaxBatchSize, deferCommit); } - // Collect non-empty table indices so we know which is last. + // Collect non-empty table indices so we know which is last, AND pre-flight + // every split frame's size BEFORE publishing any of them. The split hands + // frames to the ring one at a time (all but the last deferred -- appended but + // uncommitted); if a later table's frame were only found oversized + // mid-publish, the already-published prefix would strand on the ring, a + // subsequent commit would deliver it as a partial batch, and + // resetTableBuffersAfterFlush would discard every source row -- a partial + // commit the caller was told (by the throw) had failed. Checking all sizes up + // front makes the split all-or-nothing: either every frame fits and all + // publish, or none publish and we throw with nothing stranded. The cost is a + // second encode pass over the split batch, which is already the exceptional + // large-batch path. encode is read-only on the table buffer, and simBaseline + // mirrors the publish loop's baseline advance (advanceSentMaxSymbolId), so + // each measured size equals the frame the publish loop will build; this pass + // mutates no delta/persist state (the defer-commit flag is a header bit that + // does not change frame size). int nonEmptyCount = 0; + int simBaseline = symbolDeltaBaseline(); for (int i = 0, n = keys.size(); i < n; i++) { CharSequence tableName = keys.getQuick(i); if (tableName == null) { continue; } QwpTableBuffer tableBuffer = tableBuffers.get(tableName); - if (tableBuffer != null && tableBuffer.getRowCount() > 0) { - nonEmptyCount++; + if (tableBuffer == null || tableBuffer.getRowCount() == 0) { + continue; + } + nonEmptyCount++; + encoder.beginMessage(1, globalSymbolDictionary, simBaseline, currentBatchMaxSymbolId); + encoder.addTable(tableBuffer); + int messageSize = encoder.finishMessage(); + if (messageSize > serverMaxBatchSize) { + resetTableBuffersAfterFlush(keys); + throw new LineSenderException("single table batch too large for server batch cap") + .put(" [table=").put(tableName) + .put(", messageSize=").put(messageSize) + .put(", serverMaxBatchSize=").put(serverMaxBatchSize).put(']'); + } + // Mirror advanceSentMaxSymbolId: once the first frame ships the batch's + // new ids, the remaining frames carry an empty delta above the baseline. + if (deltaDictEnabled && currentBatchMaxSymbolId > simBaseline) { + simBaseline = currentBatchMaxSymbolId; } } @@ -3590,14 +3622,15 @@ private void flushPendingRowsSplit(ObjList keys, boolean deferComm encoder.addTable(tableBuffer); int messageSize = encoder.finishMessage(); QwpBufferWriter buffer = encoder.getBuffer(); - - if (messageSize > serverMaxBatchSize) { - resetTableBuffersAfterFlush(keys); - throw new LineSenderException("single table batch too large for server batch cap") - .put(" [table=").put(tableName) - .put(", messageSize=").put(messageSize) - .put(", serverMaxBatchSize=").put(serverMaxBatchSize).put(']'); - } + // The pre-flight pass above already verified every split frame fits the + // cap, so none can be found oversized here -- which is what keeps this + // loop from publishing (and stranding) a deferred prefix before an + // oversized table. The assert guards a future divergence between the two + // passes; it deliberately does NOT reset+throw here, because by this + // point a prefix may already be on the ring. + assert messageSize <= serverMaxBatchSize + : "split frame exceeded serverMaxBatchSize after pre-flight [table=" + tableName + + ", messageSize=" + messageSize + ", serverMaxBatchSize=" + serverMaxBatchSize + ']'; // Write-ahead persist before publish (see flushPendingRows). The // first split frame carries the batch's new symbols; the rest are diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictCatchUpTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictCatchUpTest.java index e7f27db1..0f133443 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictCatchUpTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictCatchUpTest.java @@ -102,6 +102,52 @@ public void testReconnectCatchUpRebuildsDictionary() throws Exception { }); } + @Test + public void testReconnectPreservesMonotonicDeltaBaseline() throws Exception { + // Regression: the producer's sent-symbol watermark (sentMaxSymbolId) must + // SURVIVE a reconnect. resetSymbolDictStateForNewConnection deliberately + // leaves it untouched -- the I/O thread re-registers the whole dictionary via + // a catch-up frame before replay, so the producer keeps shipping deltas ABOVE + // the baseline across the wire boundary. If a regression reset it on + // reconnect, the first post-reconnect data frame would re-ship the whole + // dictionary inline (deltaStart=0), pure wasted bandwidth. The sibling + // testReconnectCatchUpRebuildsDictionary asserts only that the final + // dictionary is complete -- which a reset-then-redefine ALSO satisfies (the + // server tolerates the redefinition) -- so it does NOT catch that regression. + // This pins the baseline survival directly: connection 2's data frame must + // carry a delta starting at id 1 (above alpha), not 0. + assertMemoryLeak(() -> { + CatchUpHandler handler = new CatchUpHandler(); + try (TestWebSocketServer server = new TestWebSocketServer(handler)) { + int port = server.getPort(); + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + + try (Sender sender = Sender.fromConfig("ws::addr=localhost:" + port + ";")) { + // Connection 1 registers alpha (id 0); the server ACKs and drops it. + sender.table("t").symbol("s", "alpha").longColumn("v", 1L).atNow(); + sender.flush(); + waitFor(() -> handler.dictFor(1).size() >= 1, 5_000); + waitFor(() -> handler.conn1Closed, 5_000); + + // Connection 2 (fresh) registers beta (id 1). With the baseline + // preserved, beta ships as a delta ABOVE id 0 (deltaStart=1). + sender.table("t").symbol("s", "beta").longColumn("v", 2L).atNow(); + sender.flush(); + waitFor(() -> handler.connectionsAccepted.get() >= 2 + && handler.dictFor(2).size() >= 2, 5_000); + } + + Assert.assertTrue("connection 2 must re-register the dictionary via a catch-up first", + handler.sawZeroTableFrameOnConn2); + Assert.assertTrue("post-reconnect data frame must ship a delta ABOVE the surviving " + + "baseline (deltaStart >= 1); a reset baseline re-ships the whole " + + "dictionary from deltaStart 0", + handler.conn2SawDeltaAboveBaseline); + } + }); + } + @Test public void testFixedCapNearBoundarySymbolCatchesUpWithoutTerminal() throws Exception { // Regression (homogeneous single cap): a symbol whose length sits just below @@ -332,6 +378,13 @@ private static class CatchUpHandler implements TestWebSocketServer.WebSocketServ // Set from the flags byte of the zero-table catch-up frame on connection 2: // the catch-up carries no rows and must defer its (empty) commit. volatile boolean catchUpDeferredOnConn2; + // Set when connection 2 receives a DATA frame (tableCount > 0) whose delta + // starts ABOVE id 0 (deltaStart >= 1). This can only happen if the producer's + // monotonic baseline SURVIVED the reconnect: a reset would re-ship the whole + // dictionary from deltaStart 0. Robust to replay -- a replayed pre-reconnect + // frame carries its original deltaStart 0, so only a genuinely-above-baseline + // post-reconnect frame trips it. + volatile boolean conn2SawDeltaAboveBaseline; volatile boolean sawZeroTableFrameOnConn2; private final List> dictsByConn = new CopyOnWriteArrayList<>(); private TestWebSocketServer.ClientHandler currentClient; @@ -356,10 +409,24 @@ public synchronized void onBinaryMessage(TestWebSocketServer.ClientHandler clien int connNumber = dictsByConn.size(); List dict = dictsByConn.get(connNumber - 1); accumulate(data, dict); - if (connNumber == 2 && tableCount(data) == 0) { - sawZeroTableFrameOnConn2 = true; - // FLAG_DEFER_COMMIT is bit 0x01 of the flags byte (offset 5). - catchUpDeferredOnConn2 = (data[5] & 0x01) != 0; + if (connNumber == 2) { + if (tableCount(data) == 0) { + sawZeroTableFrameOnConn2 = true; + // FLAG_DEFER_COMMIT is bit 0x01 of the flags byte (offset 5). + catchUpDeferredOnConn2 = (data[5] & 0x01) != 0; + } else if (data.length >= 12 && (data[5] & 0x08) != 0) { + // A post-reconnect DATA frame carrying a delta section + // (FLAG_DELTA_SYMBOL_DICT = 0x08). A deltaStart >= 1 means the + // producer resumed the delta ABOVE the surviving baseline; a reset + // baseline would re-ship from deltaStart 0. Checking any frame (not + // just the first) keeps this robust to a replayed pre-reconnect + // frame arriving ahead of the new one -- that replay carries its + // original deltaStart 0 and does not trip the flag. + int[] pos = {12}; + if (readVarint(data, pos) >= 1) { + conn2SawDeltaAboveBaseline = true; + } + } } try { client.sendBinary(buildAck(nextSeq.getAndIncrement())); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java index cb2b0118..9277addd 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java @@ -25,6 +25,7 @@ package io.questdb.client.test.cutlass.qwp.client; import io.questdb.client.Sender; +import io.questdb.client.cutlass.line.LineSenderException; import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer; import org.junit.Assert; import org.junit.Test; @@ -273,6 +274,64 @@ public void testSplitBatchShipsDeltaOnFirstFrameOnly() throws Exception { }); } + @Test + public void testOversizedTableSplitStrandsNothing() throws Exception { + // Regression: flushPendingRowsSplit publishes each table's frame one at a + // time (all but the last deferred, i.e. appended but uncommitted). If a LATER + // table's frame exceeds the cap, the split must not have already published an + // earlier table's frame -- otherwise that prefix strands on the ring, a later + // commit delivers it as a partial batch, and resetTableBuffersAfterFlush + // discards every source row, all while flush() reported failure to the + // caller. The pre-flight size pass makes the split all-or-nothing: an + // oversized table throws BEFORE any frame is published. Pre-fix, the "small" + // table's frame was published and committed on close, so the server saw it. + assertMemoryLeak(() -> { + CapturingHandler handler = new CapturingHandler(); + try (TestWebSocketServer server = new TestWebSocketServer(handler)) { + server.setAdvertisedMaxBatchSize(200); + int port = server.getPort(); + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + + // auto_flush_bytes=off lets "big" accumulate PAST the cap (byte-based + // auto-flush is otherwise clamped to 90% of the cap and would flush + // first); the row/interval limits are set high so nothing auto-flushes + // during the test. Each row stays under the per-row guard (< cap), but + // 12 rows make "big"'s single frame exceed the cap, which no split can + // shrink. "small" (added first) fits; "big" (added second) does not, so + // the split hits it AFTER publishing "small" pre-fix. + String pad = new String(new char[40]).replace('\0', 'x'); + try (Sender sender = Sender.fromConfig("ws::addr=localhost:" + port + + ";auto_flush_bytes=off;auto_flush_rows=1000000;auto_flush_interval=60000;")) { + sender.table("small").symbol("s", "a").longColumn("v", 1L).atNow(); + for (int i = 0; i < 12; i++) { + sender.table("big").stringColumn("p", pad).longColumn("v", (long) i).atNow(); + } + try { + sender.flush(); + Assert.fail("an oversized single-table split frame must throw"); + } catch (LineSenderException e) { + Assert.assertTrue(e.getMessage(), + e.getMessage().contains("too large for server batch cap")); + } + // close() drains the ring: pre-fix, the stranded "small" frame + // would be sent (and committed) here. + } + + // No DATA frame (tableCount > 0) may have reached the server: the + // oversized-table split published nothing. Pre-fix, "small" arrived. + long dataFrames = 0; + for (byte[] frame : handler.batches) { + if (frame.length >= 8 && (((frame[6] & 0xFF) | ((frame[7] & 0xFF) << 8)) > 0)) { + dataFrames++; + } + } + Assert.assertEquals("an oversized-table split must publish NO data frame -- an " + + "earlier table's frame must not strand on the ring", 0, dataFrames); + } + }); + } + private static int readVarint(byte[] buf, int offset) { // Simple unsigned varint decode — sufficient for small values. int result = 0; From 40c6a28d4ece5d71157ca8dfd6c4c10cb07f7f2e Mon Sep 17 00:00:00 2001 From: glasstiger Date: Sat, 11 Jul 2026 00:05:32 +0100 Subject: [PATCH 38/96] Harden persisted symbol-dict edge paths Fold in low-severity hardening found in review; none changes behavior on any reachable input. PersistedSymbolDict: - appendRawEntries fails loudly (rather than flushing an uninitialised scratch tail and mis-advancing size) if a caller ever passes an (addr, len, count) triple whose entries do not fill len; the sole caller derives both from one beginMessage, so this cannot fire today. - openExisting sets entriesLen alongside the malloc, so the catch frees the right size if the copy walk throws -- the comment already claimed this held. - ensureScratch grows in long so scratchCap*2 cannot overflow negative past ~1 GB, matching ensureSentDictCapacity. - Two varint decode loops gain the shift > 35 bound the other decoders already carry. Close a file-descriptor leak in the rmDir test helpers: Files.walk returns a Stream backed by an open directory handle, so wrap it in try-with-resources (PersistedSymbolDictTest calls rmDir 13 times). Co-Authored-By: Claude Opus 4.8 (1M context) --- .../client/sf/cursor/PersistedSymbolDict.java | 36 ++++++++++++++++--- .../qwp/client/SelfSufficientFramesTest.java | 22 +++++++----- ...CursorWebSocketSendLoopMirrorLeakTest.java | 22 +++++++----- .../sf/cursor/PersistedSymbolDictTest.java | 20 ++++++----- 4 files changed, 70 insertions(+), 30 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java index bb7884bd..3d32aa65 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java @@ -230,6 +230,12 @@ public synchronized void appendRawEntries(long addr, int len, int count) { break; } shift += 7; + if (shift > 35) { + // A canonical entry-length varint is <= 5 bytes; a longer + // continuation run is corrupt. The downstream entryEnd > srcLimit + // check then rejects it. Matches decodeVarint / readVarintAt. + break; + } } long entryEnd = src + symLen; // src is just past the len varint if (entryEnd > srcLimit) { @@ -242,6 +248,16 @@ public synchronized void appendRawEntries(long addr, int len, int count) { dst += wireSpan + CRC_SIZE; src = entryEnd; } + if (src != srcLimit) { + // The count entries did not consume exactly len bytes -- a caller passed an + // inconsistent (addr, len, count) triple. Writing outLen would flush an + // uninitialised scratch tail and mis-advance size, so fail loudly. The sole + // caller derives count and len from one beginMessage, so this cannot fire + // today. + throw new IllegalStateException("raw symbol-dict entries under-filled the buffer to " + + FILE_NAME + " [count=" + count + ", len=" + len + + ", consumed=" + (int) (src - addr) + ']'); + } long written = Files.write(fd, scratchAddr, outLen, appendOffset); if (written != outLen) { throw new IllegalStateException("short write to " + FILE_NAME @@ -503,6 +519,9 @@ private static PersistedSymbolDict openExisting(String filePath, long fileLen) { // open. A second no-alloc walk over the already-validated region. if (wireLen > 0) { entriesAddr = Unsafe.malloc(wireLen, MemoryTag.NATIVE_DEFAULT); + // Record the length alongside the malloc so the catch below frees the + // right size (not 0) if this copy walk ever throws. + entriesLen = wireLen; long dst = entriesAddr; int p = HEADER_SIZE; for (int i = 0; i < count; i++) { @@ -516,6 +535,9 @@ private static PersistedSymbolDict openExisting(String filePath, long fileLen) { break; } shift += 7; + if (shift > 35) { + break; // corrupt run; these entries were CRC-validated above + } } int wireSpan = (vp - p) + (int) symLen; // [len][utf8], no CRC Unsafe.getUnsafe().copyMemory(buf + p, dst, wireSpan); @@ -523,7 +545,6 @@ private static PersistedSymbolDict openExisting(String filePath, long fileLen) { p += wireSpan + CRC_SIZE; // skip the entry's CRC } } - entriesLen = wireLen; Unsafe.free(buf, len, MemoryTag.NATIVE_DEFAULT); buf = 0L; // Drop any torn/stale trailing bytes so a LATER, shorter append cannot @@ -601,8 +622,15 @@ private void ensureScratch(int required) { if (scratchCap >= required) { return; } - int newCap = Math.max(required, Math.max(256, scratchCap * 2)); - scratchAddr = Unsafe.realloc(scratchAddr, scratchCap, newCap, MemoryTag.NATIVE_DEFAULT); - scratchCap = newCap; + // Double in long: scratchCap * 2 as an int overflows negative past ~1 GB and + // would make the realloc size negative. required is bounded by one frame's + // entries (the server batch cap), so this never actually caps -- it mirrors the + // long-math growth in CursorWebSocketSendLoop.ensureSentDictCapacity. + long newCap = Math.max(required, Math.max(256L, (long) scratchCap * 2)); + if (newCap > Integer.MAX_VALUE - 8) { + newCap = Integer.MAX_VALUE - 8; + } + scratchAddr = Unsafe.realloc(scratchAddr, scratchCap, (int) newCap, MemoryTag.NATIVE_DEFAULT); + scratchCap = (int) newCap; } } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java index 9277addd..812a5317 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java @@ -38,6 +38,7 @@ import java.util.Comparator; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; +import java.util.stream.Stream; import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; @@ -351,15 +352,18 @@ private static void rmDir(Path dir) { if (dir == null || !Files.exists(dir)) { return; } - Files.walk(dir) - .sorted(Comparator.reverseOrder()) - .forEach(p -> { - try { - Files.deleteIfExists(p); - } catch (IOException ignored) { - // best-effort - } - }); + // try-with-resources: Files.walk returns a Stream backed by an open + // directory handle that must be closed, or each rmDir leaks a descriptor. + try (Stream walk = Files.walk(dir)) { + walk.sorted(Comparator.reverseOrder()) + .forEach(p -> { + try { + Files.deleteIfExists(p); + } catch (IOException ignored) { + // best-effort + } + }); + } } catch (IOException ignored) { // best-effort } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopMirrorLeakTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopMirrorLeakTest.java index 057e1a5f..38741039 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopMirrorLeakTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopMirrorLeakTest.java @@ -38,6 +38,7 @@ import java.nio.file.Path; import java.util.Comparator; import java.util.concurrent.TimeUnit; +import java.util.stream.Stream; import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; @@ -199,15 +200,18 @@ private static void rmDir(Path dir) throws IOException { if (dir == null || !Files.exists(dir)) { return; } - Files.walk(dir) - .sorted(Comparator.reverseOrder()) - .forEach(p -> { - try { - Files.deleteIfExists(p); - } catch (IOException ignored) { - // best-effort - } - }); + // try-with-resources: Files.walk returns a Stream backed by an open + // directory handle that must be closed, or each rmDir leaks a descriptor. + try (Stream walk = Files.walk(dir)) { + walk.sorted(Comparator.reverseOrder()) + .forEach(p -> { + try { + Files.deleteIfExists(p); + } catch (IOException ignored) { + // best-effort + } + }); + } } private static class SilentHandler implements TestWebSocketServer.WebSocketServerHandler { diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java index cfcf4cf5..79a9a221 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java @@ -37,6 +37,7 @@ import java.nio.file.Path; import java.nio.file.StandardOpenOption; import java.util.Comparator; +import java.util.stream.Stream; import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; @@ -503,14 +504,17 @@ private static void rmDir(Path dir) { if (dir == null || !Files.exists(dir)) { return; } - Files.walk(dir) - .sorted(Comparator.reverseOrder()) - .forEach(p -> { - try { - Files.deleteIfExists(p); - } catch (IOException ignored) { - } - }); + // try-with-resources: Files.walk returns a Stream backed by an open + // directory handle that must be closed, or each rmDir leaks a descriptor. + try (Stream walk = Files.walk(dir)) { + walk.sorted(Comparator.reverseOrder()) + .forEach(p -> { + try { + Files.deleteIfExists(p); + } catch (IOException ignored) { + } + }); + } } catch (IOException ignored) { } } From 94e3b1d2a9907e38d20ddc3f06ae702132f83891 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Sat, 11 Jul 2026 22:50:50 +0100 Subject: [PATCH 39/96] Guard the symbol-dict reopen against a >2GB file open() cast the on-disk file length to int before reading the whole file into one native buffer. A .symbol-dict that grew past 2GB (an extreme ~100M+ distinct symbols on one slot) therefore reopened with a negative malloc, a malloc(0) followed by a 4-byte out-of-bounds read, or a small-positive prefix whose validLen --- .../client/sf/cursor/PersistedSymbolDict.java | 94 ++++++--- .../sf/cursor/PersistedSymbolDictTest.java | 186 ++++++++++++++++++ 2 files changed, 254 insertions(+), 26 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java index 3d32aa65..1d545f10 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java @@ -27,7 +27,7 @@ import io.questdb.client.cutlass.qwp.client.GlobalSymbolDictionary; import io.questdb.client.cutlass.qwp.client.NativeBufferWriter; import io.questdb.client.std.Crc32c; -import io.questdb.client.std.Files; +import io.questdb.client.std.FilesFacade; import io.questdb.client.std.MemoryTag; import io.questdb.client.std.ObjList; import io.questdb.client.std.QuietCloseable; @@ -120,6 +120,10 @@ public final class PersistedSymbolDict implements QuietCloseable { static final byte VERSION = 2; // v2 appended the per-entry CRC-32C private static final Logger LOG = LoggerFactory.getLogger(PersistedSymbolDict.class); private final int fd; + // Filesystem seam. Production is FilesFacade.INSTANCE (straight to Files); + // tests inject a fault facade to exercise recovery I/O failures (a truncate + // that cannot drop a torn tail, a short write) without a real broken disk. + private final FilesFacade ff; private long appendOffset; private boolean closed; // In-memory copy of the entry region [len][utf8]... exactly as on disk, @@ -136,7 +140,8 @@ public final class PersistedSymbolDict implements QuietCloseable { private int scratchCap; private int size; - private PersistedSymbolDict(int fd, long appendOffset, int size, long loadedEntriesAddr, int loadedEntriesLen) { + private PersistedSymbolDict(FilesFacade ff, int fd, long appendOffset, int size, long loadedEntriesAddr, int loadedEntriesLen) { + this.ff = ff; this.fd = fd; this.appendOffset = appendOffset; this.size = size; @@ -153,17 +158,40 @@ private PersistedSymbolDict(int fd, long appendOffset, int size, long loadedEntr * so a broken side-file degrades gracefully rather than aborting the sender. */ public static PersistedSymbolDict open(String slotDir) { + return open(FilesFacade.INSTANCE, slotDir); + } + + /** + * Facade-aware variant of {@link #open(String)}. Production passes + * {@link FilesFacade#INSTANCE}; tests inject a fault facade to drive recovery + * I/O failures (e.g. a truncate that cannot drop a torn tail). + */ + public static PersistedSymbolDict open(FilesFacade ff, String slotDir) { String filePath = slotDir + "/" + FILE_NAME; - long existing = Files.exists(filePath) ? Files.length(filePath) : -1L; + long existing = ff.exists(filePath) ? ff.length(filePath) : -1L; + // A dictionary that legitimately grew past Integer.MAX_VALUE cannot be + // reopened: openExisting reads it into ONE int-sized native buffer, and + // the (int) cast of a >2GB length is either negative (malloc rejects it), + // exactly zero (getInt then reads 4 bytes past a zero-size allocation), or + // a small positive prefix (whose validLen < len branch would then + // DESTRUCTIVELY truncate the multi-GB file). Recreate empty instead -- + // fail-clean, exactly like every other unreadable-file case here, so the + // sender falls back to full self-sufficient frames. Reaching this needs + // ~100M+ distinct symbols on one slot (far past realistic symbol + // cardinality); the guard keeps the read/write size boundary safe anyway. + if (existing > Integer.MAX_VALUE) { + LOG.warn("symbol dict {} too large ({} bytes) to reopen; recreating empty", filePath, existing); + return openFresh(ff, filePath); + } if (existing >= HEADER_SIZE) { - PersistedSymbolDict d = openExisting(filePath, existing); + PersistedSymbolDict d = openExisting(ff, filePath, existing); if (d != null) { return d; } // Fall through to a clean re-create: a header/parse failure on an // existing file means it cannot be trusted for delta replay. } - return openFresh(filePath); + return openFresh(ff, filePath); } /** @@ -183,7 +211,14 @@ public static PersistedSymbolDict open(String slotDir) { * does. */ public static PersistedSymbolDict openClean(String slotDir) { - return openFresh(slotDir + "/" + FILE_NAME); + return openClean(FilesFacade.INSTANCE, slotDir); + } + + /** + * Facade-aware variant of {@link #openClean(String)}. + */ + public static PersistedSymbolDict openClean(FilesFacade ff, String slotDir) { + return openFresh(ff, slotDir + "/" + FILE_NAME); } /** @@ -194,7 +229,14 @@ public static PersistedSymbolDict openClean(String slotDir) { * failed delete cannot leave a stale dictionary a new session would trust. */ public static void removeOrphan(String slotDir) { - Files.remove(slotDir + "/" + FILE_NAME); + removeOrphan(FilesFacade.INSTANCE, slotDir); + } + + /** + * Facade-aware variant of {@link #removeOrphan(String)}. + */ + public static void removeOrphan(FilesFacade ff, String slotDir) { + ff.remove(slotDir + "/" + FILE_NAME); } /** @@ -258,7 +300,7 @@ public synchronized void appendRawEntries(long addr, int len, int count) { + FILE_NAME + " [count=" + count + ", len=" + len + ", consumed=" + (int) (src - addr) + ']'); } - long written = Files.write(fd, scratchAddr, outLen, appendOffset); + long written = ff.write(fd, scratchAddr, outLen, appendOffset); if (written != outLen) { throw new IllegalStateException("short write to " + FILE_NAME + " [expected=" + outLen + ", actual=" + written + ']'); @@ -290,7 +332,7 @@ public synchronized void appendSymbol(CharSequence symbol) { Utf8s.strCpyUtf8(symbol, p, utf8Len); } Unsafe.getUnsafe().putInt(scratchAddr + wireLen, Crc32c.update(Crc32c.INIT, scratchAddr, wireLen)); - long written = Files.write(fd, scratchAddr, recLen, appendOffset); + long written = ff.write(fd, scratchAddr, recLen, appendOffset); if (written != recLen) { throw new IllegalStateException("short write to " + FILE_NAME + " [expected=" + recLen + ", actual=" + written + ']'); @@ -333,7 +375,7 @@ public synchronized void appendSymbols(GlobalSymbolDictionary dict, int from, in Unsafe.getUnsafe().putInt(entryStart + wireLen, Crc32c.update(Crc32c.INIT, entryStart, wireLen)); len += wireLen + CRC_SIZE; } - long written = Files.write(fd, scratchAddr, len, appendOffset); + long written = ff.write(fd, scratchAddr, len, appendOffset); if (written != len) { throw new IllegalStateException("short write to " + FILE_NAME + " [expected=" + len + ", actual=" + written + ']'); @@ -362,7 +404,7 @@ public synchronized void close() { scratchCap = 0; } if (fd >= 0) { - Files.close(fd); + ff.close(fd); } } @@ -456,8 +498,8 @@ private static long[] decodeVarint(long buf, int pos, int limit) { return null; } - private static PersistedSymbolDict openExisting(String filePath, long fileLen) { - int fd = Files.openRW(filePath); + private static PersistedSymbolDict openExisting(FilesFacade ff, String filePath, long fileLen) { + int fd = ff.openRW(filePath); if (fd < 0) { LOG.warn("symbol dict {} could not be opened (rc={}); recreating", filePath, fd); return null; @@ -468,13 +510,13 @@ private static PersistedSymbolDict openExisting(String filePath, long fileLen) { try { int len = (int) fileLen; buf = Unsafe.malloc(len, MemoryTag.NATIVE_DEFAULT); - long read = Files.read(fd, buf, len, 0); + long read = ff.read(fd, buf, len, 0); if (read != len || Unsafe.getUnsafe().getInt(buf) != FILE_MAGIC || Unsafe.getUnsafe().getByte(buf + 4) != VERSION) { LOG.warn("symbol dict {} unreadable, bad magic or unknown version; recreating", filePath); Unsafe.free(buf, len, MemoryTag.NATIVE_DEFAULT); - Files.close(fd); + ff.close(fd); return null; } // Parse complete, CRC-valid entries after the header; stop at the first @@ -555,15 +597,15 @@ private static PersistedSymbolDict openExisting(String filePath, long fileLen) { // open() then recreates it empty (fail-clean) rather than risk a silent // misattribution. long validLen = HEADER_SIZE + diskConsumed; - if (validLen < len && !Files.truncate(fd, validLen)) { + if (validLen < len && !ff.truncate(fd, validLen)) { LOG.warn("symbol dict {} could not drop its torn/stale tail (truncate failed); recreating", filePath); if (entriesAddr != 0L) { Unsafe.free(entriesAddr, entriesLen, MemoryTag.NATIVE_DEFAULT); } - Files.close(fd); + ff.close(fd); return null; } - return new PersistedSymbolDict(fd, validLen, count, entriesAddr, entriesLen); + return new PersistedSymbolDict(ff, fd, validLen, count, entriesAddr, entriesLen); } catch (Throwable t) { if (buf != 0L) { Unsafe.free(buf, (int) fileLen, MemoryTag.NATIVE_DEFAULT); @@ -575,14 +617,14 @@ private static PersistedSymbolDict openExisting(String filePath, long fileLen) { if (entriesAddr != 0L) { Unsafe.free(entriesAddr, entriesLen, MemoryTag.NATIVE_DEFAULT); } - Files.close(fd); + ff.close(fd); LOG.warn("symbol dict {} recovery failed ({}); recreating", filePath, String.valueOf(t)); return null; } } - private static PersistedSymbolDict openFresh(String filePath) { - int fd = Files.openCleanRW(filePath); + private static PersistedSymbolDict openFresh(FilesFacade ff, String filePath) { + int fd = ff.openCleanRW(filePath); if (fd < 0) { LOG.warn("symbol dict {} could not be created (rc={}); proceeding without it", filePath, fd); return null; @@ -595,10 +637,10 @@ private static PersistedSymbolDict openFresh(String filePath) { Unsafe.getUnsafe().putByte(hdr + 5, (byte) 0); Unsafe.getUnsafe().putByte(hdr + 6, (byte) 0); Unsafe.getUnsafe().putByte(hdr + 7, (byte) 0); - long written = Files.write(fd, hdr, HEADER_SIZE, 0); + long written = ff.write(fd, hdr, HEADER_SIZE, 0); if (written != HEADER_SIZE) { - Files.close(fd); - Files.remove(filePath); + ff.close(fd); + ff.remove(filePath); LOG.warn("symbol dict {} header write failed; proceeding without it", filePath); return null; } @@ -607,7 +649,7 @@ private static PersistedSymbolDict openFresh(String filePath) { // throwing; the Unsafe puts target a valid 8-byte buffer and an 8-byte // malloc cannot realistically OOM), but close the fd against a future // edit so it cannot leak -- mirroring openExisting's error handling. - Files.close(fd); + ff.close(fd); LOG.warn("symbol dict {} creation failed ({}); proceeding without it", filePath, String.valueOf(t)); return null; } finally { @@ -615,7 +657,7 @@ private static PersistedSymbolDict openFresh(String filePath) { Unsafe.free(hdr, HEADER_SIZE, MemoryTag.NATIVE_DEFAULT); } } - return new PersistedSymbolDict(fd, HEADER_SIZE, 0, 0L, 0); + return new PersistedSymbolDict(ff, fd, HEADER_SIZE, 0, 0L, 0); } private void ensureScratch(int required) { diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java index 79a9a221..966cdffd 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java @@ -26,6 +26,7 @@ import io.questdb.client.cutlass.qwp.client.GlobalSymbolDictionary; import io.questdb.client.cutlass.qwp.client.sf.cursor.PersistedSymbolDict; +import io.questdb.client.std.FilesFacade; import io.questdb.client.std.ObjList; import io.questdb.client.test.tools.TestUtils; import org.junit.Assert; @@ -499,6 +500,53 @@ public void testTornTrailingEntrySelfHeals() throws Exception { }); } + @Test + public void testTruncateFailureRecreatesEmpty() throws Exception { + // A host crash can leave a torn/stale tail past the last complete entry. + // open() drops it with a truncate; if that truncate FAILS (a read-only + // remount, a Windows share lock), the file still exposes the stale bytes, + // whose self-consistent per-entry CRC a later shifted parse could accept as + // a real symbol. So a failed truncate must make the file UNTRUSTED -- + // open() recreates it empty (fail-clean) rather than returning a dict laid + // over stale bytes. Drive the truncate failure with a facade and assert the + // reopened dictionary is empty, not the [one, two] prefix. + assertMemoryLeak(() -> { + Path dir = Files.createTempDirectory("qwp-symdict"); + try { + PersistedSymbolDict d = PersistedSymbolDict.open(dir.toString()); + d.appendSymbol("one"); + d.appendSymbol("two"); + d.close(); + + // Append a torn trailing record (length prefix 5, only 2 bytes) so + // the reopen parses [one, two], then finds validLen < len and tries + // to truncate the tail -- the branch under test. + Path f = dir.resolve(".symbol-dict"); + long cleanLen = Files.size(f); // header + "one" + "two", no tail + Files.write(f, new byte[]{(byte) 5, (byte) 'x', (byte) 'y'}, StandardOpenOption.APPEND); + Assert.assertEquals("torn tail present before reopen", cleanLen + 3, Files.size(f)); + + // Reopen through a facade whose truncate() fails. + PersistedSymbolDict re = PersistedSymbolDict.open(new FailingTruncateFacade(), dir.toString()); + Assert.assertNotNull(re); + try { + // The failed truncate made the file untrusted, so open() recreated + // it empty rather than trusting the [one, two] prefix over a stale + // tail: size()==0, not 2, and no recovered symbols. + Assert.assertEquals("failed truncate must recreate the dictionary empty", 0, re.size()); + Assert.assertEquals(0, re.readLoadedSymbols().size()); + // openFresh rewrote a bare 8-byte header (magic + version), so both + // the two entries and the torn tail are gone. + Assert.assertEquals("recreated file is a bare header", 8L, Files.size(f)); + } finally { + re.close(); + } + } finally { + rmDir(dir); + } + }); + } + private static void rmDir(Path dir) { try { if (dir == null || !Files.exists(dir)) { @@ -518,4 +566,142 @@ private static void rmDir(Path dir) { } catch (IOException ignored) { } } + + /** + * A {@link FilesFacade} that delegates every call to {@link FilesFacade#INSTANCE} + * except {@link #truncate(int, long)}, which always fails -- reproducing a + * host where the torn/stale-tail truncate cannot succeed (read-only remount, + * Windows share lock) so {@code open()}'s fail-clean recreate path runs. + */ + private static final class FailingTruncateFacade implements FilesFacade { + @Override + public long allocNativePath(String path) { + return INSTANCE.allocNativePath(path); + } + + @Override + public boolean allocate(int fd, long size) { + return INSTANCE.allocate(fd, size); + } + + @Override + public int close(int fd) { + return INSTANCE.close(fd); + } + + @Override + public boolean exists(String path) { + return INSTANCE.exists(path); + } + + @Override + public void findClose(long findPtr) { + INSTANCE.findClose(findPtr); + } + + @Override + public long findFirst(String dir) { + return INSTANCE.findFirst(dir); + } + + @Override + public long findName(long findPtr) { + return INSTANCE.findName(findPtr); + } + + @Override + public int findNext(long findPtr) { + return INSTANCE.findNext(findPtr); + } + + @Override + public int findType(long findPtr) { + return INSTANCE.findType(findPtr); + } + + @Override + public void freeNativePath(long pathPtr) { + INSTANCE.freeNativePath(pathPtr); + } + + @Override + public int fsync(int fd) { + return INSTANCE.fsync(fd); + } + + @Override + public long length(int fd) { + return INSTANCE.length(fd); + } + + @Override + public long length(String path) { + return INSTANCE.length(path); + } + + @Override + public long length(long pathPtr) { + return INSTANCE.length(pathPtr); + } + + @Override + public int lock(int fd) { + return INSTANCE.lock(fd); + } + + @Override + public int mkdir(String path, int mode) { + return INSTANCE.mkdir(path, mode); + } + + @Override + public int openCleanRW(String path) { + return INSTANCE.openCleanRW(path); + } + + @Override + public int openCleanRW(long pathPtr) { + return INSTANCE.openCleanRW(pathPtr); + } + + @Override + public int openRW(String path) { + return INSTANCE.openRW(path); + } + + @Override + public int openRW(long pathPtr) { + return INSTANCE.openRW(pathPtr); + } + + @Override + public long read(int fd, long addr, long len, long offset) { + return INSTANCE.read(fd, addr, len, offset); + } + + @Override + public boolean remove(String path) { + return INSTANCE.remove(path); + } + + @Override + public boolean remove(long pathPtr) { + return INSTANCE.remove(pathPtr); + } + + @Override + public int rename(String oldPath, String newPath) { + return INSTANCE.rename(oldPath, newPath); + } + + @Override + public boolean truncate(int fd, long size) { + return false; // the fault under test + } + + @Override + public long write(int fd, long addr, long len, long offset) { + return INSTANCE.write(fd, addr, len, offset); + } + } } From 6c6dc40d6000c561e81dbda8bab0e61628a7dc8f Mon Sep 17 00:00:00 2001 From: glasstiger Date: Sat, 11 Jul 2026 23:02:12 +0100 Subject: [PATCH 40/96] Latch a terminal on sent-dict mirror overflow ensureSentDictCapacity threw a bare LineSenderException when the lifetime-monotonic sent-dictionary mirror would exceed MAX_SENT_DICT_BYTES. accumulateSentDict runs after the frame's sendBinary, so that throw unwound to ioLoop -> fail() -> connectLoop with running still true, which reconnected and replayed the same frame; the mirror never shrinks, so the replay re-overflowed and the loop reconnected again -- an unbounded livelock instead of the loud failure the ceiling promises. recordFatal now flips running=false before the throw, so connectLoop's !running guard winds the loop down and checkError() surfaces the terminal to the producer. This matches sendCatchUpChunk's guard for the same ceiling. The throw stays, to unwind past the pending copyMemory. The path needs ~100M+ distinct symbols on one connection to reach, so it is defensive; the producer's GlobalSymbolDictionary would OOM first. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../client/sf/cursor/CursorWebSocketSendLoop.java | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java index a855cc50..a24776d3 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java @@ -2137,8 +2137,19 @@ private void ensureSentDictCapacity(long required) { return; } if (required > MAX_SENT_DICT_BYTES) { - throw new LineSenderException("symbol dictionary mirror exceeds the maximum size [" + // Latch a terminal, do NOT just throw: accumulateSentDict runs AFTER + // the frame's sendBinary, so a bare throw unwinds to ioLoop -> fail() + // -> connectLoop, which (running still true) reconnects and replays the + // same frame, which re-overflows the never-shrinking mirror -- an + // unbounded reconnect livelock rather than the "fails loudly" the + // MAX_SENT_DICT_BYTES ceiling promises. recordFatal flips running=false + // so connectLoop's !running guard winds the loop down and checkError() + // surfaces the terminal, matching sendCatchUpChunk's guard for the same + // ceiling. The throw still unwinds past the pending copyMemory. + LineSenderException err = new LineSenderException("symbol dictionary mirror exceeds the maximum size [" + "required=" + required + ", max=" + MAX_SENT_DICT_BYTES + ']'); + recordFatal(err); + throw err; } // Grow in long to avoid the capacity*2 int overflow (negative) that would // otherwise degrade the doubling near 1 GB; clamp to the int ceiling. From ff2e841af2e2d10e9568737806dd119f075be17f Mon Sep 17 00:00:00 2001 From: glasstiger Date: Mon, 13 Jul 2026 00:16:03 +0100 Subject: [PATCH 41/96] Guard torn-dict resume and skip split re-encode Two hardening fixes to the delta symbol-dictionary path. Torn-dictionary resume (data integrity): the persisted .symbol-dict is not fsync'd, so a host crash can lose its newest entries while the segment frames that introduced those ids survive and replay. A resumed producer, seeded from the shorter dictionary, would reuse ids the surviving frames already define and silently misattribute symbols. The send loop's replay guard only catches a gap (deltaStart > mirror); the contiguous tear self-heals the mirror and leaves the producer diverged. CursorSendEngine now records the highest symbol id the recovered frames reference via a read-only MmapSegment/SegmentRing walk, and seedGlobalDictionaryFromPersisted fails clean when it reaches the recovered dictionary size -- the affected data must be resent. Only the resuming producer refuses; the orphan drainer still self-heals and drains the slot. testTornDictSubsetFailsCleanOnResume covers it. Split-flush sizing (performance): flushPendingRowsSplit re-encoded the whole batch a second time just to size each split frame. It now derives each frame size arithmetically from the combined encode flushPendingRows already performed, capturing each table's body bytes and the delta-entry length. Body encoding is context-free, so the sizes are exact; the publish loop's assert cross-checks them. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../qwp/client/QwpWebSocketSender.java | 81 ++++++++++++++++--- .../client/sf/cursor/CursorSendEngine.java | 31 +++++++ .../qwp/client/sf/cursor/MmapSegment.java | 66 +++++++++++++++ .../qwp/client/sf/cursor/SegmentRing.java | 18 +++++ .../qwp/client/DeltaDictRecoveryTest.java | 69 ++++++++++++++++ 5 files changed, 253 insertions(+), 12 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index 2fd785a6..1f948a72 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -57,6 +57,7 @@ import io.questdb.client.std.Decimal128; import io.questdb.client.std.Decimal256; import io.questdb.client.std.Decimal64; +import io.questdb.client.std.IntList; import io.questdb.client.std.Misc; import io.questdb.client.std.Numbers; import io.questdb.client.std.NumericException; @@ -169,6 +170,11 @@ public class QwpWebSocketSender implements Sender { // behind a drainer's endpoint walk. private final ReentrantLock connectWalkLock = new ReentrantLock(); private final QwpHostHealthTracker hostTracker; + // Per-table encoded body byte counts captured during flushPendingRows' combined + // encode, reused by flushPendingRowsSplit to size each split frame arithmetically + // instead of re-encoding the batch a second time. Cleared and repopulated on every + // flush; only consumed on the (exceptional) split path. Reused to stay zero-GC. + private final IntList splitFrameBodyBytes = new IntList(); private final CharSequenceObjHashMap tableBuffers; // null means plain text (no TLS) private final ClientTlsConfiguration tlsConfig; @@ -3474,6 +3480,15 @@ private void flushPendingRows(boolean deferCommit) { encoder.setDeferCommit(deferCommit); encoder.beginMessage(tableCount, globalSymbolDictionary, symbolDeltaBaseline(), currentBatchMaxSymbolId); + // Record each table's encoded body size (position delta across addTable) so + // the split path can size its per-table frames arithmetically rather than + // re-encoding the whole batch. Body encoding is context-free (a table's bytes + // don't depend on its siblings or the delta section), so the size a table + // takes here equals the size it takes in its own split frame. Only consumed + // when the batch overflows the cap; the capture is a couple of int ops per + // table on the common path. + splitFrameBodyBytes.clear(); + int bodyStart = encoder.getBuffer().getPosition(); for (int i = 0, n = keys.size(); i < n; i++) { CharSequence tableName = keys.getQuick(i); if (tableName == null) { @@ -3490,12 +3505,18 @@ private void flushPendingRows(boolean deferCommit) { } encoder.addTable(tableBuffer); + int bodyEnd = encoder.getBuffer().getPosition(); + splitFrameBodyBytes.add(bodyEnd - bodyStart); + bodyStart = bodyEnd; } int messageSize = encoder.finishMessage(); QwpBufferWriter buffer = encoder.getBuffer(); if (serverMaxBatchSize > 0 && messageSize > serverMaxBatchSize) { - flushPendingRowsSplit(keys, deferCommit); + // The combined frame's delta-entry bytes are byte-identical to the first + // split frame's (same baseline + batch max), so capture the length now + // for the arithmetic frame-sizing in flushPendingRowsSplit. + flushPendingRowsSplit(keys, deferCommit, encoder.getDeltaEntriesLen()); return; } @@ -3547,7 +3568,7 @@ private void flushPendingRows(boolean deferCommit) { * carry FLAG_DEFER_COMMIT. When false, only the * last message omits the flag. */ - private void flushPendingRowsSplit(ObjList keys, boolean deferCommit) { + private void flushPendingRowsSplit(ObjList keys, boolean deferCommit, int combinedDeltaEntriesLen) { if (LOG.isDebugEnabled()) { LOG.debug("Splitting flush across multiple messages [serverMaxBatchSize={}, defer={}]", serverMaxBatchSize, deferCommit); } @@ -3561,15 +3582,19 @@ private void flushPendingRowsSplit(ObjList keys, boolean deferComm // resetTableBuffersAfterFlush would discard every source row -- a partial // commit the caller was told (by the throw) had failed. Checking all sizes up // front makes the split all-or-nothing: either every frame fits and all - // publish, or none publish and we throw with nothing stranded. The cost is a - // second encode pass over the split batch, which is already the exceptional - // large-batch path. encode is read-only on the table buffer, and simBaseline - // mirrors the publish loop's baseline advance (advanceSentMaxSymbolId), so - // each measured size equals the frame the publish loop will build; this pass - // mutates no delta/persist state (the defer-commit flag is a header bit that - // does not change frame size). + // publish, or none publish and we throw with nothing stranded. + // + // Each split frame's size is derived ARITHMETICALLY from the combined encode + // flushPendingRows already performed -- header + the two delta-section varints + // + the delta entries (byte-identical to the combined frame's when this frame + // carries them) + the table's own body bytes (captured in splitFrameBodyBytes, + // context-free so identical here and in its solo frame) -- rather than + // re-encoding every table a second time. simBaseline mirrors the publish loop's + // baseline advance (advanceSentMaxSymbolId), so each size equals the frame the + // publish loop will build; this pass mutates no delta/persist state. int nonEmptyCount = 0; int simBaseline = symbolDeltaBaseline(); + int bodyIdx = 0; for (int i = 0, n = keys.size(); i < n; i++) { CharSequence tableName = keys.getQuick(i); if (tableName == null) { @@ -3580,9 +3605,14 @@ private void flushPendingRowsSplit(ObjList keys, boolean deferComm continue; } nonEmptyCount++; - encoder.beginMessage(1, globalSymbolDictionary, simBaseline, currentBatchMaxSymbolId); - encoder.addTable(tableBuffer); - int messageSize = encoder.finishMessage(); + int deltaStart = simBaseline + 1; + int deltaCount = Math.max(0, currentBatchMaxSymbolId - simBaseline); + int messageSize = QwpConstants.HEADER_SIZE + + NativeBufferWriter.varintSize(deltaStart) + + NativeBufferWriter.varintSize(deltaCount) + + (deltaCount > 0 ? combinedDeltaEntriesLen : 0) + + splitFrameBodyBytes.getQuick(bodyIdx); + bodyIdx++; if (messageSize > serverMaxBatchSize) { resetTableBuffersAfterFlush(keys); throw new LineSenderException("single table batch too large for server batch cap") @@ -3821,11 +3851,38 @@ private void resetSymbolDictStateForNewConnection() { * dictionary shorter than {@code pd.size()} and desyncing * {@code sentMaxSymbolId} from the mirror's {@code sentDictCount = pd.size()}, * which silently misattributes later symbols after a reconnect. + *

+ * Host-crash tear guard. The persisted dictionary is NOT fsync'd (see + * {@code PersistedSymbolDict}), so a host/power crash can lose its + * most-recently-written (highest-id) entries while the segment frames that + * introduced those ids survive -- and those newest frames, being the least + * likely to be acked, replay on recovery. The send loop's catch-up mirror then + * rebuilds the missing ids from those frames' own delta bytes, but THIS producer + * -- seeded only from the shorter dictionary -- would assign its next new symbol + * an id the surviving frames already define, putting two symbols on one id and + * silently misattributing values. The send loop's replay guard only catches a + * GAP ({@code deltaStart > sentDictCount}); a frame that introduces exactly the + * torn-off id ({@code deltaStart == pd.size()}) slips through and self-heals the + * mirror, leaving only this producer diverged. Detect it here -- the surviving + * frames reference an id at or beyond the recovered dictionary size -- and fail + * clean: the affected data must be resent, matching the design's torn-dict + * "resend required" contract. The recovered data is not lost; the background + * drainer still drains this slot (its mirror self-heals from the frames). Only a + * host crash reaches this -- a process crash keeps the page cache, so the + * write-ahead ordering keeps the dictionary a superset of the frames. */ private void seedGlobalDictionaryFromPersisted(PersistedSymbolDict pd) { if (pd == null || pd.size() == 0) { return; } + if (cursorEngine != null && cursorEngine.recoveredMaxSymbolId() >= pd.size()) { + throw new LineSenderException( + "recovered store-and-forward symbol dictionary is a subset of the surviving frames " + + "(likely a host crash tore its unsynced tail): frames reference symbol id " + + cursorEngine.recoveredMaxSymbolId() + " but the recovered dictionary holds only " + + pd.size() + " id(s); resuming would reuse ids the frames already define -- " + + "resend the affected data"); + } ObjList symbols = pd.readLoadedSymbols(); for (int i = 0, n = symbols.size(); i < n; i++) { globalSymbolDictionary.addRecoveredSymbol(symbols.getQuick(i)); diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java index 6c34124d..eb59633a 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java @@ -96,6 +96,15 @@ public final class CursorSendEngine implements QuietCloseable { // covers them. Read by the sender's close-time drain to avoid waiting on // acks that cannot arrive. private long recoveredCommitBoundaryFsn = -1L; + // Highest symbol id any recovered delta frame references, or -1 for + // fresh/memory rings (and recovered rings with no symbol-bearing frame). A + // resuming producer seeds its dictionary baseline from the persisted + // .symbol-dict; if that dictionary was torn below this id by a host crash + // (the side-file is not fsync'd), the producer would re-use ids the surviving + // frames already define. seedGlobalDictionaryFromPersisted compares this + // against the recovered dictionary size to fail clean instead. Computed once + // in the constructor's recovery branch; -1 elsewhere. + private long recoveredMaxSymbolId = -1L; // FSN of the last frame of a recovered orphaned deferred tail, or -1 when // the recovered ring has no such tail. When >= 0, frames // [recoveredCommitBoundaryFsn + 1 .. recoveredOrphanTipFsn] all carry @@ -312,6 +321,18 @@ private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager man publishedFsn - Math.max(recoveredCommitBoundaryFsn, -1L), recoveredCommitBoundaryFsn, publishedFsn); } + // Highest symbol id the surviving frames reference. A resuming + // producer compares this against its recovered dictionary size + // (seedGlobalDictionaryFromPersisted) to detect a host-crash tear: + // if a frame references an id the (unsynced, torn) .symbol-dict no + // longer holds, resuming would re-use it. maxSymbolDeltaEnd returns + // 0 when no frame carries a symbol, yielding -1 here. Computed + // before the I/O loop or producer append; single-threaded here. + this.recoveredMaxSymbolId = recovered.maxSymbolDeltaEnd( + io.questdb.client.cutlass.qwp.protocol.QwpConstants.MAGIC_MESSAGE, + io.questdb.client.cutlass.qwp.protocol.QwpConstants.HEADER_OFFSET_FLAGS, + io.questdb.client.cutlass.qwp.protocol.QwpConstants.FLAG_DELTA_SYMBOL_DICT, + io.questdb.client.cutlass.qwp.protocol.QwpConstants.HEADER_SIZE) - 1L; } else { // Fresh start with no recovered segments. Any stale // watermark from a prior fully-drained session refers @@ -714,6 +735,16 @@ public long recoveredCommitBoundaryFsn() { return recoveredCommitBoundaryFsn; } + /** + * Highest symbol id any recovered delta frame references, or {@code -1} for + * fresh/memory rings (and recovered rings with no symbol-bearing frame). A + * resuming producer compares this against its recovered dictionary size to + * detect a host-crash tear of the persisted {@code .symbol-dict}. + */ + public long recoveredMaxSymbolId() { + return recoveredMaxSymbolId; + } + /** * FSN of the last frame of a recovered orphaned deferred tail, or * {@code -1} when none. See {@link #recoveredCommitBoundaryFsn()}: the diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java index 78e5db9f..04f2ac93 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java @@ -522,6 +522,72 @@ public long findLastFrameFsnWithoutPayloadFlag(int flagsOffset, int flagMask, in return best; } + /** + * Highest {@code deltaStart + deltaCount} (one past the highest symbol id) that + * any symbol-dict delta frame in this segment references, or {@code 0} when no + * such frame carries a symbol. Read-only walk over the recovered frames, used + * once at recovery to detect a persisted {@code .symbol-dict} torn (host crash, + * out-of-order page loss) below the ids the surviving frames still reference: if + * the max here reaches at or beyond the recovered dictionary size, a resuming + * producer -- seeded from the shorter dictionary -- would re-use ids the frames + * already define. The frame layout mirrors {@link #findLastFrameFsnWithoutPayloadFlag}: + * the QWP message header ({@code qwpHeaderSize} bytes) is followed by the delta + * section {@code [deltaStart varint][deltaCount varint]...}. + * + * @param headerMagic the QWP message magic identifying a well-formed frame + * @param flagsOffset byte offset of the flags field within the QWP header + * @param flagDeltaMask the FLAG_DELTA_SYMBOL_DICT bit + * @param qwpHeaderSize the QWP message header size (delta section starts past it) + */ + public long maxSymbolDeltaEnd(int headerMagic, int flagsOffset, int flagDeltaMask, int qwpHeaderSize) { + long maxEnd = 0L; + long off = HEADER_SIZE; + long frames = frameCount; + for (long i = 0; i < frames; i++) { + int payloadLen = Unsafe.getUnsafe().getInt(mmapAddress + off + 4); + long payload = mmapAddress + off + FRAME_HEADER_SIZE; + if (payloadLen >= qwpHeaderSize + && payloadLen > flagsOffset + && Unsafe.getUnsafe().getInt(payload) == headerMagic + && (Unsafe.getUnsafe().getByte(payload + flagsOffset) & flagDeltaMask) != 0) { + long p = payload + qwpHeaderSize; + long limit = payload + payloadLen; + long deltaStart = 0L; + int shift = 0; + while (p < limit) { + byte b = Unsafe.getUnsafe().getByte(p++); + deltaStart |= (long) (b & 0x7F) << shift; + if ((b & 0x80) == 0) { + break; + } + shift += 7; + if (shift > 35) { + break; // corrupt run; recovered frames are CRC-valid, so defensive only + } + } + long deltaCount = 0L; + shift = 0; + while (p < limit) { + byte b = Unsafe.getUnsafe().getByte(p++); + deltaCount |= (long) (b & 0x7F) << shift; + if ((b & 0x80) == 0) { + break; + } + shift += 7; + if (shift > 35) { + break; + } + } + long end = deltaStart + deltaCount; + if (end > maxEnd) { + maxEnd = end; + } + } + off += FRAME_HEADER_SIZE + payloadLen; + } + return maxEnd; + } + /** * Number of frames written since {@link #create} (or recovered by * {@link #openExisting}). Used by {@code SegmentRing} to compute diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java index c8516c4c..2b53e956 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java @@ -546,6 +546,24 @@ public synchronized long findLastFsnWithoutPayloadFlag(int flagsOffset, int flag return Math.max(best, fsn); } + /** + * Highest {@code deltaStart + deltaCount} any symbol-dict delta frame in the + * ring references (0 when none). See {@link MmapSegment#maxSymbolDeltaEnd}; + * used once at recovery to detect a persisted dictionary torn below the ids + * the surviving frames reference. + */ + public synchronized long maxSymbolDeltaEnd(int headerMagic, int flagsOffset, int flagDeltaMask, int qwpHeaderSize) { + long maxEnd = 0L; + for (int i = 0, n = sealedSegments.size(); i < n; i++) { + long end = sealedSegments.get(i).maxSymbolDeltaEnd(headerMagic, flagsOffset, flagDeltaMask, qwpHeaderSize); + if (end > maxEnd) { + maxEnd = end; + } + } + long end = active.maxSymbolDeltaEnd(headerMagic, flagsOffset, flagDeltaMask, qwpHeaderSize); + return Math.max(maxEnd, end); + } + public MmapSegment getActive() { return active; } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java index e46d0148..21023c04 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java @@ -771,6 +771,75 @@ public void testRecoverySeedKeepsUtf8CollidingSymbolsInLockstep() throws Excepti }); } + @Test + public void testTornDictSubsetFailsCleanOnResume() throws Exception { + // C1 regression: the persisted .symbol-dict is NOT fsync'd, so a host crash + // can lose its highest-id tail entries while the segment frames that + // introduced those ids survive (out-of-order page loss). Recovery then seeds + // the producer from the SHORTER dictionary, but the I/O-thread catch-up mirror + // rebuilds the missing ids from those surviving frames -- so a resumed producer + // would assign its next new symbol an id the frames already define, silently + // misattributing symbol values on the wire. The send loop's replay guard only + // catches a GAP (deltaStart > mirror); a frame introducing exactly the + // torn-off id slips through and self-heals the mirror, leaving only this + // producer diverged. seedGlobalDictionaryFromPersisted must detect that the + // surviving frames reference an id at/beyond the recovered dictionary size and + // fail clean. Without the fix the resume succeeds and the reuse corrupts + // silently -- so the build below would NOT throw and the test's fail() fires. + assertMemoryLeak(() -> { + // Phase 1: record three delta frames (a@0, b@1, c@2) to disk with nothing + // acked (silent server), so all three survive and replay on recovery. + try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) { + int port = silent.getPort(); + silent.start(); + Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS)); + String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + + ";close_flush_timeout_millis=0;"; + Sender s1 = Sender.fromConfig(cfg); + try { + s1.table("m").symbol("s", "a").longColumn("v", 0).atNow(); + s1.flush(); + s1.table("m").symbol("s", "b").longColumn("v", 1).atNow(); + s1.flush(); + s1.table("m").symbol("s", "c").longColumn("v", 2).atNow(); + s1.flush(); + } finally { + s1.close(); + } + } + + // Simulate the host-crash tear: rewrite the persisted dictionary to drop + // its highest entry (c@2), keeping a@0,b@1 -- while the frame referencing + // c@2 stays on disk. openClean truncates; the two appends leave the exact + // two-entry dictionary a torn tail would recover to. + String slotDir = Paths.get(sfDir, "default").toString(); + try (PersistedSymbolDict torn = PersistedSymbolDict.openClean(slotDir)) { + Assert.assertNotNull(torn); + torn.appendSymbol("a"); + torn.appendSymbol("b"); + Assert.assertEquals(2, torn.size()); + } + + // Phase 2: a resuming sender must refuse -- the surviving frames reference + // id 2 but the recovered dictionary holds only 2 id(s) [0,1]. + try (TestWebSocketServer good = new TestWebSocketServer(new DictReconstructingHandler())) { + int port = good.getPort(); + good.start(); + Assert.assertTrue(good.awaitStart(5, TimeUnit.SECONDS)); + String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";"; + try { + Sender resumed = Sender.fromConfig(cfg); + resumed.close(); + Assert.fail("resume on a torn (subset) dictionary must fail clean"); + } catch (LineSenderException expected) { + Assert.assertTrue("message must name the torn-dict/resend failure: " + expected.getMessage(), + expected.getMessage().contains("subset of the surviving frames") + && expected.getMessage().contains("resend")); + } + } + }); + } + private static int globalDictSize(Sender sender) throws Exception { Field f = sender.getClass().getDeclaredField("globalSymbolDictionary"); f.setAccessible(true); From 50fc21814f24eca96514fcea7d220431aeb6bd9d Mon Sep 17 00:00:00 2001 From: glasstiger Date: Mon, 13 Jul 2026 00:37:08 +0100 Subject: [PATCH 42/96] Give the catch-up cap gap a settle budget Two hardening fixes to the split/catch-up paths for heterogeneous / rolling-cap clusters. Catch-up cap-gap settle budget (M1): when a symbol-dict catch-up entry is too large for the fresh server's advertised batch cap, sendDictCatchUp latched a terminal on first sight. A homogeneous cluster never trips this (an entry that fit its data frame under a cap always fits its bare catch-up frame under that same cap), but a failover to a smaller-cap node could kill the sender for an entry an earlier node accepted. It now retries across MAX_CATCHUP_CAP_GAP_ATTEMPTS consecutive reconnects (throwing a retriable CatchUpSendException, no recordFatal), riding out the transient window until a larger-cap node returns; only a persistent gap latches the terminal. A successful catch-up resets the budget, and a transient reconnect never reaches the catch-up so it neither increments nor burns it. Matches the orphan drainer's durable-ack settle budget. Split-flush cap snapshot (M2): flushPendingRows now snapshots the volatile serverMaxBatchSize once and threads it into flushPendingRowsSplit for both the pre-flight sizing and the publish-loop assert. A mid-flush failover (the I/O thread lowers the cap) could previously make the two passes size against different caps, breaking the all-or-nothing guarantee and firing the assert on a legitimate race. The next flush picks up the new cap. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../qwp/client/QwpWebSocketSender.java | 32 ++++++---- .../sf/cursor/CursorWebSocketSendLoop.java | 63 ++++++++++++++----- .../qwp/client/DeltaDictCatchUpTest.java | 17 +++-- ...WebSocketSendLoopCatchUpAlignmentTest.java | 56 +++++++++++++++++ 4 files changed, 136 insertions(+), 32 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index 1f948a72..3765ce87 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -3512,11 +3512,19 @@ private void flushPendingRows(boolean deferCommit) { int messageSize = encoder.finishMessage(); QwpBufferWriter buffer = encoder.getBuffer(); - if (serverMaxBatchSize > 0 && messageSize > serverMaxBatchSize) { + // Snapshot the volatile cap ONCE for this whole flush. The I/O thread lowers + // serverMaxBatchSize on a mid-stream failover to a smaller-cap node + // (applyServerBatchSizeLimit); if the split pre-flight and the publish loop + // re-read the field independently, a failover between them would size frames + // against two different caps -- breaking the all-or-nothing guarantee and + // firing the publish-loop assert on a legitimate race. Both use this snapshot; + // the next flush picks up the new cap. + int cap = serverMaxBatchSize; + if (cap > 0 && messageSize > cap) { // The combined frame's delta-entry bytes are byte-identical to the first // split frame's (same baseline + batch max), so capture the length now // for the arithmetic frame-sizing in flushPendingRowsSplit. - flushPendingRowsSplit(keys, deferCommit, encoder.getDeltaEntriesLen()); + flushPendingRowsSplit(keys, deferCommit, encoder.getDeltaEntriesLen(), cap); return; } @@ -3568,9 +3576,9 @@ private void flushPendingRows(boolean deferCommit) { * carry FLAG_DEFER_COMMIT. When false, only the * last message omits the flag. */ - private void flushPendingRowsSplit(ObjList keys, boolean deferCommit, int combinedDeltaEntriesLen) { + private void flushPendingRowsSplit(ObjList keys, boolean deferCommit, int combinedDeltaEntriesLen, int cap) { if (LOG.isDebugEnabled()) { - LOG.debug("Splitting flush across multiple messages [serverMaxBatchSize={}, defer={}]", serverMaxBatchSize, deferCommit); + LOG.debug("Splitting flush across multiple messages [serverMaxBatchSize={}, defer={}]", cap, deferCommit); } // Collect non-empty table indices so we know which is last, AND pre-flight @@ -3613,12 +3621,12 @@ private void flushPendingRowsSplit(ObjList keys, boolean deferComm + (deltaCount > 0 ? combinedDeltaEntriesLen : 0) + splitFrameBodyBytes.getQuick(bodyIdx); bodyIdx++; - if (messageSize > serverMaxBatchSize) { + if (messageSize > cap) { resetTableBuffersAfterFlush(keys); throw new LineSenderException("single table batch too large for server batch cap") .put(" [table=").put(tableName) .put(", messageSize=").put(messageSize) - .put(", serverMaxBatchSize=").put(serverMaxBatchSize).put(']'); + .put(", serverMaxBatchSize=").put(cap).put(']'); } // Mirror advanceSentMaxSymbolId: once the first frame ships the batch's // new ids, the remaining frames carry an empty delta above the baseline. @@ -3655,12 +3663,14 @@ private void flushPendingRowsSplit(ObjList keys, boolean deferComm // The pre-flight pass above already verified every split frame fits the // cap, so none can be found oversized here -- which is what keeps this // loop from publishing (and stranding) a deferred prefix before an - // oversized table. The assert guards a future divergence between the two - // passes; it deliberately does NOT reset+throw here, because by this - // point a prefix may already be on the ring. - assert messageSize <= serverMaxBatchSize + // oversized table. Both passes size against the SAME snapshot cap, so a + // mid-flush failover cannot make them disagree; the assert therefore only + // catches a genuine divergence between the pre-flight arithmetic and the + // real encode (a future bug), not a cap race. It deliberately does NOT + // reset+throw here, because by this point a prefix may already be on the ring. + assert messageSize <= cap : "split frame exceeded serverMaxBatchSize after pre-flight [table=" + tableName - + ", messageSize=" + messageSize + ", serverMaxBatchSize=" + serverMaxBatchSize + ']'; + + ", messageSize=" + messageSize + ", serverMaxBatchSize=" + cap + ']'; // Write-ahead persist before publish (see flushPendingRows). The // first split frame carries the batch's new symbols; the rest are diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java index a24776d3..1b4a7d6f 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java @@ -133,6 +133,18 @@ public final class CursorWebSocketSendLoop implements QuietCloseable { */ public static final long DEFAULT_POISON_MIN_ESCALATION_WINDOW_MILLIS = 5_000L; private static final Logger LOG = LoggerFactory.getLogger(CursorWebSocketSendLoop.class); + // Settle budget for the symbol-dict catch-up cap gap: how many CONSECUTIVE + // reconnect attempts may find a single dictionary entry too large for the fresh + // server's advertised batch cap before the sender latches a terminal. A + // homogeneous cluster never trips it -- an entry that fit its data frame under a + // cap always fits its bare catch-up frame under that same cap -- so this only + // affects a heterogeneous / rolling-cap cluster, where a failover to a + // smaller-cap node can hit it for an entry an earlier node accepted. Retrying + // rides out the transient window until a larger-cap node returns; only a + // persistent gap (every reachable node too small for this many attempts) latches + // terminal, matching the orphan drainer's DEFAULT_MAX_DURABLE_ACK_MISMATCH_ATTEMPTS. + // A successful catch-up resets the counter (see sendDictCatchUp). + private static final int MAX_CATCHUP_CAP_GAP_ATTEMPTS = 16; // Hard ceiling for the lifetime-monotonic sent-dictionary mirror. The mirror // fields are int, so it cannot exceed Integer.MAX_VALUE bytes; reaching even // this needs ~200M+ distinct symbols on a single connection, far past any real @@ -206,6 +218,12 @@ public final class CursorWebSocketSendLoop implements QuietCloseable { // for the connection's lifetime (a reconnect may need the whole dictionary at // any moment), so it cannot be dropped; it is an intentional cost of the feature. private final boolean deltaDictEnabled; + // Consecutive reconnect attempts whose symbol-dict catch-up found an entry too + // large for the fresh server's batch cap (see MAX_CATCHUP_CAP_GAP_ATTEMPTS). A + // successful catch-up resets it; it is NOT reset per connection -- it measures + // the cap-gap episode across reconnects so a persistent gap eventually latches. + // I/O-thread-only. + private int catchUpCapGapAttempts; // True once a real ring frame (data or commit) has been sent on the CURRENT // connection, as opposed to only the dictionary catch-up. The catch-up // consumes wire sequences (nextWireSeq), so nextWireSeq > 0 no longer implies @@ -2243,24 +2261,36 @@ private int sendDictCatchUp() { + NativeBufferWriter.varintSize(1) + entryBytes; if (soloFrameLen > frameLimit) { - // Non-retriable: the entry will not shrink and the same cluster - // re-advertises the same cap, so reconnecting would livelock. - // Latch a terminal (the data must be resent after the cap is - // raised) rather than calling fail() -- which, from inside the - // catch-up, would re-enter connectLoop (see CatchUpSendException). + // Cap gap: this entry cannot be re-registered under the fresh + // server's advertised cap. A HOMOGENEOUS cluster never reaches here + // (an entry that fit its data frame under a cap always fits its bare + // catch-up frame under that same cap), so the only way in is a + // heterogeneous / rolling-cap failover to a smaller-cap node. // - // Tradeoff (heterogeneous / rolling-cap clusters): a symbol - // accepted under a larger/absent cap can hit this on failover to a - // smaller-cap node, and the hard terminal does NOT self-recover if - // a later node advertises a larger cap -- the producer must be - // resumed after the data is resent (or the cap raised). Bounding - // symbol size at ingest, or a settle budget across reconnects - // before latching, would relax this, but both are larger changes; - // the terminal keeps the homogeneous common case livelock-free. + // Give the cluster a settle budget instead of latching on first + // sight: a larger-cap node may return, so retry across reconnects + // and only latch a terminal after MAX_CATCHUP_CAP_GAP_ATTEMPTS + // consecutive attempts still find it too large. Under budget the + // throw is RETRIABLE (no recordFatal) -- connectLoop reconnects with + // backoff and re-runs the catch-up, which resets the counter on a + // node that accepts it. A transient reconnect (connect/upgrade + // failure, role reject) never reaches the catch-up, so it neither + // increments nor burns this budget. On exhaustion latch via + // recordFatal, NOT fail() -- failing from inside the catch-up would + // re-enter connectLoop (see CatchUpSendException); the data must be + // resent after the cap is raised. + catchUpCapGapAttempts++; + boolean exhausted = catchUpCapGapAttempts >= MAX_CATCHUP_CAP_GAP_ATTEMPTS; LineSenderException err = new LineSenderException( "symbol dictionary entry too large for the server batch cap during catch-up [" - + "frameLen=" + soloFrameLen + ", cap=" + cap + ']'); - recordFatal(err); + + "frameLen=" + soloFrameLen + ", cap=" + cap + ", attempt=" + + catchUpCapGapAttempts + '/' + MAX_CATCHUP_CAP_GAP_ATTEMPTS + ']' + + (exhausted + ? "; the data must be resent after the cap is raised" + : "; retrying -- a larger-cap node may return")); + if (exhausted) { + recordFatal(err); + } throw new CatchUpSendException(err); } if (chunkSymbols > 0 && chunkBytes + entryBytes > budget) { @@ -2279,6 +2309,9 @@ private int sendDictCatchUp() { sendCatchUpChunk(chunkStartId, chunkSymbols, chunkStartAddr, (int) chunkBytes); framesSent++; } + // The whole dictionary re-registered without a cap gap: this node accepts + // every entry, so the cap-gap episode (if any) is over -- reset the budget. + catchUpCapGapAttempts = 0; return framesSent; } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictCatchUpTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictCatchUpTest.java index 0f133443..132f2c06 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictCatchUpTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictCatchUpTest.java @@ -212,11 +212,15 @@ public void testFixedCapNearBoundarySymbolCatchesUpWithoutTerminal() throws Exce @Test public void testCatchUpEntryTooLargeForCapFailsTerminally() throws Exception { // A dictionary entry that exceeds the reconnect server's per-chunk budget - // (cap - HEADER_SIZE - 16) cannot be shipped as a catch-up chunk. - // sendDictCatchUp must latch a clean terminal ("... during catch-up") - // rather than call fail(): pre-fix the oversized entry drove an endless - // reconnect loop (the entry never shrinks and the same cluster - // re-advertises the same cap) and re-entered connectLoop from the catch-up. + // (cap - HEADER_SIZE - 16) cannot be shipped as a catch-up chunk. Every + // reachable node re-advertises the same small cap here, so the gap never + // resolves: sendDictCatchUp must retry across the settle budget and then + // latch a clean terminal ("... during catch-up") rather than call fail() + // (which from inside the catch-up re-enters connectLoop). Pre-fix it latched + // on the FIRST cap gap; the settle budget (MAX_CATCHUP_CAP_GAP_ATTEMPTS) + // rides out a transient smaller-cap window first (see the retry sibling in + // CursorWebSocketSendLoopCatchUpAlignmentTest), and only a persistent gap + // exhausts it. Small reconnect backoffs keep the budgeted attempts fast. // // Connection 1 advertises no cap, so the ~202-byte symbol registers and // enters the sent-dictionary mirror. The handler then shrinks the @@ -234,7 +238,8 @@ public void testCatchUpEntryTooLargeForCapFailsTerminally() throws Exception { String bigSymbol = TestUtils.repeat("x", 200); // ~202-byte dict entry LineSenderException terminal = null; - Sender sender = Sender.fromConfig("ws::addr=localhost:" + port + ";"); + Sender sender = Sender.fromConfig("ws::addr=localhost:" + port + + ";reconnect_initial_backoff_millis=10;reconnect_max_backoff_millis=50;"); try { sender.table("t").symbol("s", bigSymbol).longColumn("v", 1L).atNow(); sender.flush(); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java index 3814d826..315e3d75 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java @@ -26,6 +26,7 @@ import io.questdb.client.DefaultHttpClientConfiguration; import io.questdb.client.cutlass.http.client.WebSocketClient; +import io.questdb.client.cutlass.line.LineSenderException; import io.questdb.client.cutlass.qwp.client.WebSocketResponse; import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine; import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorWebSocketSendLoop; @@ -265,6 +266,61 @@ public void testCatchUpChunkFrameSizeOverflowFailsLoud() throws Exception { }); } + @Test + public void testCatchUpCapGapRetriesUntilBudgetThenLatches() throws Exception { + // M1: an entry too large for the fresh server's cap during catch-up (a + // heterogeneous / rolling-cap failover to a smaller-cap node) must NOT latch + // on first sight. sendDictCatchUp throws a RETRIABLE CatchUpSendException so + // the reconnect loop rides it out -- a larger-cap node may return -- and only + // after MAX_CATCHUP_CAP_GAP_ATTEMPTS consecutive cap gaps does it recordFatal. + // Pre-fix the first cap gap latched a terminal, so one transient failover to a + // smaller-cap node killed the sender. (A successful catch-up resets the budget; + // the other catch-up tests, which use a fitting cap, never trip it.) + TestUtils.assertMemoryLeak(() -> { + Field maxField = CursorWebSocketSendLoop.class.getDeclaredField("MAX_CATCHUP_CAP_GAP_ATTEMPTS"); + maxField.setAccessible(true); + int maxAttempts = maxField.getInt(null); + // cap 160 => catch-up budget is below a ~216-byte solo frame for a 200-char symbol. + CatchUpCapturingClient client = new CatchUpCapturingClient(160); + try (CursorSendEngine engine = newEngine()) { + CursorWebSocketSendLoop loop = newLoop(engine, client); + try { + seedMirror(loop, TestUtils.repeat("x", 200)); + // Attempts 1 .. max-1 are retriable: no terminal is latched. + for (int i = 1; i < maxAttempts; i++) { + try { + invokeSetWireBaselineWithCatchUp(loop, engine.ackedFsn() + 1L); + fail("cap gap must raise a retriable CatchUpSendException (attempt " + i + ')'); + } catch (InvocationTargetException e) { + assertEquals("CatchUpSendException", e.getCause().getClass().getSimpleName()); + assertTrue("attempt " + i + " must name the catch-up cap gap: " + + e.getCause().getMessage(), + e.getCause().getMessage().contains("during catch-up")); + } + loop.checkError(); // under budget => retriable => no terminal + } + // The exhausting attempt still throws, and now latches the terminal. + try { + invokeSetWireBaselineWithCatchUp(loop, engine.ackedFsn() + 1L); + fail("the exhausting cap gap must still raise CatchUpSendException"); + } catch (InvocationTargetException e) { + assertEquals("CatchUpSendException", e.getCause().getClass().getSimpleName()); + } + try { + loop.checkError(); + fail("exhausting the cap-gap settle budget must latch a terminal"); + } catch (LineSenderException terminal) { + assertTrue("terminal must name the exhausted catch-up cap gap: " + terminal.getMessage(), + terminal.getMessage().contains("during catch-up") + && terminal.getMessage().contains("must be resent")); + } + } finally { + loop.close(); + } + } + }); + } + private static void appendFrames(CursorSendEngine engine, int count) { long buf = Unsafe.malloc(16, MemoryTag.NATIVE_DEFAULT); try { From 7880b7a2730304a42c79ba37ef15e4dcca4951f9 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Mon, 13 Jul 2026 14:13:24 +0100 Subject: [PATCH 43/96] Cover defensive dict guards and tidy test helpers Round of minor cleanups on the delta symbol-dictionary work. Tests for previously-uncovered defensive guards (all proven to fail without their guard): PersistedSymbolDict.open recreates empty on a bad version byte (a valid entry behind a corrupted version must not parse) and on a > Integer.MAX_VALUE length (a >2GB dictionary can't map into one int buffer -- driven by a fault facade); and the send loop's ensureSentDictCapacity latches a terminal instead of overflowing the int capacity math when the mirror would exceed MAX_SENT_DICT_BYTES. openExisting nulls buf after freeing it on the bad-magic/bad-version path, so the catch block cannot double-free if ff.close ever threw. PersistedSymbolDict.appendSymbol is marked @TestOnly -- production persists a frame's whole new-symbol range via appendSymbols / appendRawEntries; only tests append one symbol at a time. Documented that symbol-dict catch-up frames sit outside the poison-frame detector: a deterministically-NACKed catch-up recycles forever (paced), which is fine since a catch-up only re-registers already-accepted symbols. Consolidated four reinvented recursive rmDir/rmDirRec test helpers into one TestUtils.removeTmpDirRec (the existing removeTmpDir is flat and can't clean the store-and-forward slot subdirectory layout), and removed the imports that freed up. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../sf/cursor/CursorWebSocketSendLoop.java | 7 + .../client/sf/cursor/PersistedSymbolDict.java | 7 + .../qwp/client/DeltaDictRecoveryTest.java | 30 +---- .../qwp/client/SelfSufficientFramesTest.java | 23 +--- ...WebSocketSendLoopCatchUpAlignmentTest.java | 41 ++++++ ...CursorWebSocketSendLoopMirrorLeakTest.java | 19 +-- .../sf/cursor/PersistedSymbolDictTest.java | 124 ++++++++++++++---- .../questdb/client/test/tools/TestUtils.java | 30 +++++ 8 files changed, 188 insertions(+), 93 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java index 1b4a7d6f..f55f47c8 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java @@ -2911,6 +2911,13 @@ private void handleServerRejection(long wireSeq) { // path, where engine.acknowledge() no-ops at or below ackedFsn. A // real replayed data frame is at fsn > ackedFsn, so it is never // caught here. + // + // Catch-up frames therefore sit OUTSIDE the poison detector: a + // deterministically-NACKed catch-up recycles forever (paced, so no + // busy-loop). That is acceptable -- a catch-up only re-registers + // symbols the cluster already accepted, so a persistent NACK of one + // is a server bug, not a poison-frame the client can quarantine, and + // Invariant B's "retry a transient outage forever" applies. handlePreSendRejection(wireSeq, status, category, policy); return; } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java index 1d545f10..9930a239 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java @@ -33,6 +33,7 @@ import io.questdb.client.std.QuietCloseable; import io.questdb.client.std.Unsafe; import io.questdb.client.std.str.Utf8s; +import org.jetbrains.annotations.TestOnly; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -317,7 +318,12 @@ public synchronized void appendRawEntries(long addr, int len, int count) { * (the entry's position). Writes {@code [len varint][utf8][crc32c]}, the CRC * covering the {@code [len][utf8]} bytes so a torn/stale entry is detected on * recovery. + *

+ * Test-only: production persists a frame's whole new-symbol range in one write + * via {@link #appendSymbols} / {@link #appendRawEntries}. Tests use this to + * build a dictionary one entry at a time. */ + @TestOnly public synchronized void appendSymbol(CharSequence symbol) { if (closed) { return; @@ -516,6 +522,7 @@ private static PersistedSymbolDict openExisting(FilesFacade ff, String filePath, || Unsafe.getUnsafe().getByte(buf + 4) != VERSION) { LOG.warn("symbol dict {} unreadable, bad magic or unknown version; recreating", filePath); Unsafe.free(buf, len, MemoryTag.NATIVE_DEFAULT); + buf = 0L; // null after free so the catch below cannot double-free if ff.close throws ff.close(fd); return null; } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java index 21023c04..c7d4de13 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java @@ -27,7 +27,6 @@ import io.questdb.client.Sender; import io.questdb.client.cutlass.line.LineSenderException; import io.questdb.client.cutlass.qwp.client.sf.cursor.PersistedSymbolDict; -import io.questdb.client.std.Files; import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer; import io.questdb.client.test.tools.TestUtils; import org.junit.After; @@ -76,9 +75,7 @@ public void setUp() { @After public void tearDown() { - if (sfDir != null) { - rmDirRec(sfDir); - } + TestUtils.removeTmpDirRec(sfDir); } @Test @@ -879,31 +876,6 @@ private static int readVarint(byte[] buf, int[] pos) { throw new IllegalStateException("varint truncated"); } - private static void rmDirRec(String dir) { - if (!Files.exists(dir)) { - return; - } - long find = Files.findFirst(dir); - if (find > 0) { - try { - int rc = 1; - while (rc > 0) { - String name = Files.utf8ToString(Files.findName(find)); - if (name != null && !".".equals(name) && !"..".equals(name)) { - String child = dir + "/" + name; - if (!Files.remove(child)) { - rmDirRec(child); - } - } - rc = Files.findNext(find); - } - } finally { - Files.findClose(find); - } - } - Files.remove(dir); - } - /** * Reconstructs the per-connection symbol dictionary from delta sections, * mirroring the server's {@code setQuick(deltaStart + i)} + null-padding. diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java index 812a5317..524063a1 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java @@ -27,6 +27,7 @@ import io.questdb.client.Sender; import io.questdb.client.cutlass.line.LineSenderException; import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer; +import io.questdb.client.test.tools.TestUtils; import org.junit.Assert; import org.junit.Test; @@ -35,10 +36,8 @@ import java.nio.ByteOrder; import java.nio.file.Files; import java.nio.file.Path; -import java.util.Comparator; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; -import java.util.stream.Stream; import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; @@ -348,25 +347,7 @@ private static int readVarint(byte[] buf, int offset) { } private static void rmDir(Path dir) { - try { - if (dir == null || !Files.exists(dir)) { - return; - } - // try-with-resources: Files.walk returns a Stream backed by an open - // directory handle that must be closed, or each rmDir leaks a descriptor. - try (Stream walk = Files.walk(dir)) { - walk.sorted(Comparator.reverseOrder()) - .forEach(p -> { - try { - Files.deleteIfExists(p); - } catch (IOException ignored) { - // best-effort - } - }); - } - } catch (IOException ignored) { - // best-effort - } + TestUtils.removeTmpDirRec(dir == null ? null : dir.toString()); } private static void waitFor(BoolCondition cond, long timeoutMillis) { diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java index 315e3d75..54fd645c 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java @@ -321,6 +321,47 @@ public void testCatchUpCapGapRetriesUntilBudgetThenLatches() throws Exception { }); } + @Test + public void testMirrorOverflowFailsLoud() throws Exception { + // ensureSentDictCapacity must latch a terminal -- not silently overflow the + // int capacity math into a heap-corrupting copyMemory -- when the sent-dict + // mirror would exceed MAX_SENT_DICT_BYTES. Unreachable at real cardinality + // (~200M+ symbols on one connection), so drive the guard directly with an + // oversized required, mirroring testCatchUpChunkFrameSizeOverflowFailsLoud. + TestUtils.assertMemoryLeak(() -> { + Field maxField = CursorWebSocketSendLoop.class.getDeclaredField("MAX_SENT_DICT_BYTES"); + maxField.setAccessible(true); + long overCeiling = (long) maxField.getInt(null) + 1L; + CatchUpCapturingClient client = new CatchUpCapturingClient(0); + try (CursorSendEngine engine = newEngine()) { + CursorWebSocketSendLoop loop = newLoop(engine, client); + try { + Method m = CursorWebSocketSendLoop.class.getDeclaredMethod("ensureSentDictCapacity", long.class); + m.setAccessible(true); + try { + m.invoke(loop, overCeiling); + fail("a mirror capacity past MAX_SENT_DICT_BYTES must fail loud, not overflow"); + } catch (InvocationTargetException e) { + assertEquals("overflow must surface as LineSenderException", + "LineSenderException", e.getCause().getClass().getSimpleName()); + assertTrue("message must name the mirror ceiling: " + e.getCause().getMessage(), + e.getCause().getMessage().contains("mirror exceeds the maximum size")); + } + // recordFatal (not a bare throw) latched the terminal, so the loop + // winds down instead of reconnecting into the same overflow. + try { + loop.checkError(); + fail("mirror overflow must latch a terminal"); + } catch (LineSenderException terminal) { + assertTrue(terminal.getMessage().contains("mirror exceeds the maximum size")); + } + } finally { + loop.close(); + } + } + }); + } + private static void appendFrames(CursorSendEngine engine, int count) { long buf = Unsafe.malloc(16, MemoryTag.NATIVE_DEFAULT); try { diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopMirrorLeakTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopMirrorLeakTest.java index 38741039..5cff81ba 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopMirrorLeakTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopMirrorLeakTest.java @@ -29,6 +29,7 @@ import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorWebSocketSendLoop; import io.questdb.client.cutlass.qwp.client.sf.cursor.PersistedSymbolDict; import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer; +import io.questdb.client.test.tools.TestUtils; import org.junit.Assert; import org.junit.Test; @@ -36,9 +37,7 @@ import java.lang.reflect.Field; import java.nio.file.Files; import java.nio.file.Path; -import java.util.Comparator; import java.util.concurrent.TimeUnit; -import java.util.stream.Stream; import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; @@ -197,21 +196,7 @@ private static int readInt(CursorWebSocketSendLoop loop, String name) throws Exc } private static void rmDir(Path dir) throws IOException { - if (dir == null || !Files.exists(dir)) { - return; - } - // try-with-resources: Files.walk returns a Stream backed by an open - // directory handle that must be closed, or each rmDir leaks a descriptor. - try (Stream walk = Files.walk(dir)) { - walk.sorted(Comparator.reverseOrder()) - .forEach(p -> { - try { - Files.deleteIfExists(p); - } catch (IOException ignored) { - // best-effort - } - }); - } + TestUtils.removeTmpDirRec(dir == null ? null : dir.toString()); } private static class SilentHandler implements TestWebSocketServer.WebSocketServerHandler { diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java index 966cdffd..eb79e016 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java @@ -32,13 +32,10 @@ import org.junit.Assert; import org.junit.Test; -import java.io.IOException; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.StandardOpenOption; -import java.util.Comparator; -import java.util.stream.Stream; import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; @@ -230,6 +227,43 @@ public void testBadMagicIsRecreatedEmpty() throws Exception { }); } + @Test + public void testBadVersionIsRecreatedEmpty() throws Exception { + // A file with correct 'SYD1' magic and a VALID entry but an unknown version + // byte must be recreated empty -- its entries belong to a foreign/future + // format and must not be parsed as v2. Covers the version sub-condition + // specifically (testBadMagicIsRecreatedEmpty covers the magic one). Because + // the seeded "a" is a valid v2 entry, without the version check it would parse + // back (size 1), so this fails. + assertMemoryLeak(() -> { + Path dir = Files.createTempDirectory("qwp-symdict"); + try { + Path f = dir.resolve(".symbol-dict"); + // Build a real v2 dictionary with one valid entry... + PersistedSymbolDict seed = PersistedSymbolDict.open(dir.toString()); + Assert.assertNotNull(seed); + seed.appendSymbol("a"); + seed.close(); + // ...then corrupt ONLY the version byte (offset 4) to an unknown value. + byte[] bytes = Files.readAllBytes(f); + bytes[4] = (byte) 99; + Files.write(f, bytes); + + PersistedSymbolDict d = PersistedSymbolDict.open(dir.toString()); + Assert.assertNotNull(d); + try { + Assert.assertEquals("bad-version file recreated empty (entries must not parse)", 0, d.size()); + d.appendSymbol("X"); + Assert.assertEquals(1, d.size()); + } finally { + d.close(); + } + } finally { + rmDir(dir); + } + }); + } + @Test public void testCloseNullsLoadedEntries() throws Exception { // close() must null loadedEntriesAddr/Len after freeing them (like @@ -450,6 +484,38 @@ public void testRemoveOrphanDeletesFile() throws Exception { }); } + @Test + public void testTooLargeToReopenRecreatesEmpty() throws Exception { + // A dictionary that legitimately grew past Integer.MAX_VALUE cannot be read + // into one int-sized buffer; open() must recreate it empty (fail-clean, like + // every other unreadable-file case) rather than truncate the multi-GB file + // via a negative/zero (int) length cast. A fault facade reports the huge + // length without needing a real 2GB file on disk. + assertMemoryLeak(() -> { + Path dir = Files.createTempDirectory("qwp-symdict"); + try { + // Seed a small real file so exists() is true and the facade's huge + // length() drives the > Integer.MAX_VALUE reopen guard. + PersistedSymbolDict seed = PersistedSymbolDict.open(dir.toString()); + Assert.assertNotNull(seed); + seed.appendSymbol("a"); + seed.close(); + + PersistedSymbolDict d = PersistedSymbolDict.open(new HugeLengthFacade(), dir.toString()); + Assert.assertNotNull(d); + try { + Assert.assertEquals("a >2GB dictionary must recreate empty", 0, d.size()); + d.appendSymbol("X"); + Assert.assertEquals(1, d.size()); + } finally { + d.close(); + } + } finally { + rmDir(dir); + } + }); + } + @Test public void testTornTrailingEntrySelfHeals() throws Exception { assertMemoryLeak(() -> { @@ -548,32 +614,14 @@ public void testTruncateFailureRecreatesEmpty() throws Exception { } private static void rmDir(Path dir) { - try { - if (dir == null || !Files.exists(dir)) { - return; - } - // try-with-resources: Files.walk returns a Stream backed by an open - // directory handle that must be closed, or each rmDir leaks a descriptor. - try (Stream walk = Files.walk(dir)) { - walk.sorted(Comparator.reverseOrder()) - .forEach(p -> { - try { - Files.deleteIfExists(p); - } catch (IOException ignored) { - } - }); - } - } catch (IOException ignored) { - } + TestUtils.removeTmpDirRec(dir == null ? null : dir.toString()); } /** - * A {@link FilesFacade} that delegates every call to {@link FilesFacade#INSTANCE} - * except {@link #truncate(int, long)}, which always fails -- reproducing a - * host where the torn/stale-tail truncate cannot succeed (read-only remount, - * Windows share lock) so {@code open()}'s fail-clean recreate path runs. + * A {@link FilesFacade} that delegates every call to {@link FilesFacade#INSTANCE}. + * Subclasses inject a single fault; every other call hits the real filesystem. */ - private static final class FailingTruncateFacade implements FilesFacade { + private static class DelegatingFilesFacade implements FilesFacade { @Override public long allocNativePath(String path) { return INSTANCE.allocNativePath(path); @@ -696,7 +744,7 @@ public int rename(String oldPath, String newPath) { @Override public boolean truncate(int fd, long size) { - return false; // the fault under test + return INSTANCE.truncate(fd, size); } @Override @@ -704,4 +752,28 @@ public long write(int fd, long addr, long len, long offset) { return INSTANCE.write(fd, addr, len, offset); } } + + /** + * Fails every {@link #truncate(int, long)} -- reproducing a host where the + * torn/stale-tail truncate cannot succeed (read-only remount, Windows share + * lock) so {@code open()}'s fail-clean recreate path runs. + */ + private static final class FailingTruncateFacade extends DelegatingFilesFacade { + @Override + public boolean truncate(int fd, long size) { + return false; + } + } + + /** + * Reports a dictionary length past {@link Integer#MAX_VALUE} -- reproducing a + * dictionary that legitimately grew beyond 2GB, which {@code open()} cannot read + * into one int-sized buffer and must recreate empty. + */ + private static final class HugeLengthFacade extends DelegatingFilesFacade { + @Override + public long length(String path) { + return (long) Integer.MAX_VALUE + 1L; + } + } } diff --git a/core/src/test/java/io/questdb/client/test/tools/TestUtils.java b/core/src/test/java/io/questdb/client/test/tools/TestUtils.java index 7572880b..39755426 100644 --- a/core/src/test/java/io/questdb/client/test/tools/TestUtils.java +++ b/core/src/test/java/io/questdb/client/test/tools/TestUtils.java @@ -215,6 +215,36 @@ public static void removeTmpDir(String tmpDir) { Files.remove(tmpDir); } + /** + * Recursive counterpart to {@link #removeTmpDir(String)} for tests whose temp + * directory has subdirectories -- e.g. the store-and-forward slot layout + * {@code

/default/...}, which the flat variant cannot clean up. A + * {@code null} argument is a no-op, so it is safe from {@code tearDown} before + * {@code setUp} ran. Uses {@code java.nio.file} (fully qualified to avoid the + * {@code io.questdb.client.std.Files} import clash) so subdirectories delete + * bottom-up. + */ + public static void removeTmpDirRec(String tmpDir) { + if (tmpDir == null) { + return; + } + java.nio.file.Path root = java.nio.file.Paths.get(tmpDir); + if (!java.nio.file.Files.exists(root)) { + return; + } + // try-with-resources: the walk Stream holds an open directory handle that + // must be closed, or each call leaks a descriptor. + try (java.util.stream.Stream walk = java.nio.file.Files.walk(root)) { + walk.sorted(java.util.Comparator.reverseOrder()).forEach(p -> { + try { + java.nio.file.Files.deleteIfExists(p); + } catch (java.io.IOException ignored) { + } + }); + } catch (java.io.IOException ignored) { + } + } + /** * Java 8 stand-in for {@code String.repeat(int)} (added in Java 11). */ From 2ad8f5018788549dd75ba64a8d4249032f8a6d5b Mon Sep 17 00:00:00 2001 From: glasstiger Date: Mon, 13 Jul 2026 22:09:58 +0100 Subject: [PATCH 44/96] Fail clean on a total symbol-dict tear on resume seedGlobalDictionaryFromPersisted returned early when the persisted .symbol-dict recovered empty (size 0), skipping the torn-dictionary guard on the line just below. A host crash -- or a prior full-dict-mode session whose openClean failed and never wrote the side-file -- can leave a size-0 dictionary while the segment frames that introduced its ids survive. Resuming then seeds the producer with an empty dictionary, so the next new symbol reuses id 0; once that frame is trimmed and the connection recycles, the catch-up re-registers the recovered ids and the producer's rows are silently misattributed. The send loop's replay guard cannot catch this: a deltaStart=0 frame self-heals the catch-up mirror rather than tripping the gap check, so the seed-time guard is the only defense. Run it before the size-0 short-circuit. A genuinely empty slot (no symbol-bearing frames) still has recoveredMaxSymbolId == -1, so -1 >= 0 is false and the empty case returns without throwing. Add testTornDictTotalLossFailsCleanOnResume covering the size-0 total tear (frames replay from deltaStart=0, so only the seed-time guard can reject it). Repoint two tests that had reached the I/O-thread guard through the bypassed early return: testTornDictionaryFailsCleanly... now expects the build-time guard, and testTornDictionaryOneIdGap... reaches the I/O guard through the unopenable-dictionary path instead. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../qwp/client/QwpWebSocketSender.java | 14 +- .../qwp/client/DeltaDictRecoveryTest.java | 146 ++++++++++++------ 2 files changed, 116 insertions(+), 44 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index 3765ce87..4f7c374b 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -3882,9 +3882,18 @@ private void resetSymbolDictStateForNewConnection() { * write-ahead ordering keeps the dictionary a superset of the frames. */ private void seedGlobalDictionaryFromPersisted(PersistedSymbolDict pd) { - if (pd == null || pd.size() == 0) { + if (pd == null) { return; } + // Run the torn-dictionary guard BEFORE the empty-dictionary short-circuit + // below: a TOTAL tear (pd.size() == 0) with surviving symbol-bearing frames + // must fail clean too, not slip through. Such a frame starts at deltaStart=0 + // and self-heals the I/O-thread catch-up mirror, so the send loop's replay + // guard (deltaStart > sentDictCount) never fires -- this seed-time guard is + // then the only defense against the producer resuming unseeded and silently + // reusing ids the frames already define. A genuinely empty slot (no + // symbol-bearing frames) has recoveredMaxSymbolId() == -1, so -1 >= 0 is false + // and the pd.size() == 0 return below still fires. if (cursorEngine != null && cursorEngine.recoveredMaxSymbolId() >= pd.size()) { throw new LineSenderException( "recovered store-and-forward symbol dictionary is a subset of the surviving frames " @@ -3893,6 +3902,9 @@ private void seedGlobalDictionaryFromPersisted(PersistedSymbolDict pd) { + pd.size() + " id(s); resuming would reuse ids the frames already define -- " + "resend the affected data"); } + if (pd.size() == 0) { + return; + } ObjList symbols = pd.readLoadedSymbols(); for (int i = 0, n = symbols.size(); i < n; i++) { globalSymbolDictionary.addRecoveredSymbol(symbols.getQuick(i)); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java index c7d4de13..fb355b82 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java @@ -171,19 +171,21 @@ public void testTornDictionaryFailsCleanlyInsteadOfCorrupting() throws Exception } } - // Simulate a host/power crash: the segment frames survive but the persisted - // dictionary is lost, and the ack watermark was left mid-stream. Truncate - // .symbol-dict to its 8-byte header (0 symbols) and stamp the watermark at - // FSN 2, so recovery replays from FSN 3 -- a frame with deltaStart=3. + // Simulate a host/power crash: the segment frames survive but the WHOLE + // persisted dictionary is lost (truncate .symbol-dict to its 8-byte header, + // 0 symbols), and the ack watermark was left mid-stream (FSN 2). The + // surviving frames still reference the lost ids. java.nio.file.Path slot = Paths.get(sfDir, "default"); java.nio.file.Path dict = slot.resolve(".symbol-dict"); byte[] header = Arrays.copyOf(java.nio.file.Files.readAllBytes(dict), 8); java.nio.file.Files.write(dict, header); writeAckWatermark(slot.resolve(".ack-watermark"), 2); - // Phase 2: recover against a fresh counting server. The replay guard must - // fire (frame deltaStart 3 > recovered dictionary size 0) and fail terminally - // rather than send a gapped frame that would corrupt the table. + // Phase 2: recover against a fresh counting server. With the whole + // dictionary lost (size 0) while the surviving frames still reference its + // ids, seedGlobalDictionaryFromPersisted must refuse the resume at BUILD -- + // its size==0 short-circuit must NOT skip the torn-dict guard -- rather than + // let the producer resume unseeded and silently misattribute reused ids. CountingHandler handler = new CountingHandler(); try (TestWebSocketServer good = new TestWebSocketServer(handler)) { int port = good.getPort(); @@ -192,47 +194,34 @@ public void testTornDictionaryFailsCleanlyInsteadOfCorrupting() throws Exception String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";"; LineSenderException terminal = null; - Sender s2 = Sender.fromConfig(cfg); try { - // Poll for the replay guard to fire (it recordFatal's on the I/O - // thread); flush() surfaces the latched terminal to the producer. - // A bounded poll replaces a fixed sleep and captures it as soon as - // it fires; close() below is the fallback if it surfaces only there. - long deadline = System.currentTimeMillis() + 10_000; - while (System.currentTimeMillis() < deadline && terminal == null) { - try { - s2.flush(); - Thread.sleep(20); - } catch (LineSenderException e) { - terminal = e; - } - } - } finally { - try { - s2.close(); - } catch (LineSenderException e) { - if (terminal == null) { - terminal = e; - } - } + Sender s2 = Sender.fromConfig(cfg); + s2.close(); + } catch (LineSenderException e) { + terminal = e; } Assert.assertEquals("no frame may be replayed to a fresh server with a torn dictionary", 0, handler.frames.get()); Assert.assertNotNull("a torn dictionary must surface a terminal error", terminal); Assert.assertTrue(terminal.getMessage(), - terminal.getMessage().contains("symbol dictionary is incomplete")); + terminal.getMessage().contains("subset of the surviving frames") + && terminal.getMessage().contains("resend")); } }); } @Test public void testTornDictionaryOneIdGapFailsCleanly() throws Exception { - // Boundary variant of testTornDictionaryFailsCleanlyInsteadOfCorrupting: the - // recovered dictionary is short by EXACTLY ONE id -- the tightest gap the - // guard must still reject (deltaStart == recoveredSize + 1). A one-entry-short - // tail is the common host-crash outcome, so this pins the guard's - // false-negative edge: a "deltaStart > size + 1" mutation would let this - // single-id gap through and null-pad the missing symbol on the server. + // One-id-gap boundary variant of + // testUnopenablePersistedDictStillGuardsAgainstReplayingDeltaFrames: the first + // replayed frame starts EXACTLY ONE id past the empty mirror -- the tightest + // gap the I/O-thread replay guard must still reject (deltaStart == + // sentDictCount + 1). It reaches that guard via the unopenable-dictionary path + // (deltaDictEnabled=false), NOT a size-0 openable dict, which now fails clean + // earlier in seedGlobalDictionaryFromPersisted. A one-entry-short tail is the + // common host-crash outcome, so this pins the guard's false-negative edge: a + // "deltaStart > size + 1" mutation would let this single-id gap through and + // null-pad the missing symbol on the server. assertMemoryLeak(() -> { // Phase 1: each row introduces a new symbol (frame i carries deltaStart=i). try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) { @@ -249,15 +238,17 @@ public void testTornDictionaryOneIdGapFailsCleanly() throws Exception { } } - // Lose the whole dictionary (truncate to the 8-byte header, size 0) but - // stamp the watermark at FSN 0, so recovery replays from FSN 1 -- a frame - // with deltaStart=1. The dictionary covers no ids, so id 0 is the single - // missing entry: deltaStart(1) == recoveredSize(0) + 1. Because size is 0 - // no catch-up is sent, so any counted frame would be the gapped data frame. + // Make the persisted dictionary UNOPENABLE (replace .symbol-dict with a + // directory, so both openRW and openCleanRW fail and the engine reports + // deltaDictEnabled=false), then stamp the watermark at FSN 0 so recovery + // replays from FSN 1 -- a frame with deltaStart=1. The mirror is unseeded + // (size 0), so id 0 is the single missing entry: deltaStart(1) == + // sentDictCount(0) + 1. No catch-up is sent, so any counted frame would be + // the gapped data frame. java.nio.file.Path slot = Paths.get(sfDir, "default"); java.nio.file.Path dict = slot.resolve(".symbol-dict"); - byte[] header = Arrays.copyOf(java.nio.file.Files.readAllBytes(dict), 8); - java.nio.file.Files.write(dict, header); + java.nio.file.Files.delete(dict); + java.nio.file.Files.createDirectory(dict); writeAckWatermark(slot.resolve(".ack-watermark"), 0); // Phase 2: recover against a fresh counting server. The guard must fire on @@ -837,6 +828,75 @@ public void testTornDictSubsetFailsCleanOnResume() throws Exception { }); } + @Test + public void testTornDictTotalLossFailsCleanOnResume() throws Exception { + // C1 regression (TOTAL-loss variant of testTornDictSubsetFailsCleanOnResume): + // a host crash can lose the ENTIRE persisted .symbol-dict (size 0) while the + // frames that introduced its ids survive; equivalently, a prior session whose + // openClean failed ran full-dict mode and never wrote the side-file, so this + // session recovers those frames against a FRESH EMPTY dictionary. The recovered + // frames start at deltaStart=0 and self-heal the I/O-thread catch-up mirror, so + // the send loop's replay guard (deltaStart > mirror) never fires -- only + // seedGlobalDictionaryFromPersisted can catch it. Its pd.size()==0 early return + // must NOT skip the torn-dict guard: the surviving frames reference id 2 while + // the recovered dictionary holds 0 id(s), so resuming unseeded would reuse those + // ids and silently misattribute symbol values on the wire. Without the fix the + // build below does NOT throw (the guard is bypassed) and the test's fail() fires. + assertMemoryLeak(() -> { + // Phase 1: record three delta frames (a@0, b@1, c@2) with nothing acked, so + // all three survive and replay -- from FSN 0 (deltaStart=0), the case the + // I/O-thread guard self-heals rather than rejects. + try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) { + int port = silent.getPort(); + silent.start(); + Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS)); + String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + + ";close_flush_timeout_millis=0;"; + Sender s1 = Sender.fromConfig(cfg); + try { + s1.table("m").symbol("s", "a").longColumn("v", 0).atNow(); + s1.flush(); + s1.table("m").symbol("s", "b").longColumn("v", 1).atNow(); + s1.flush(); + s1.table("m").symbol("s", "c").longColumn("v", 2).atNow(); + s1.flush(); + } finally { + s1.close(); + } + } + + // Total dictionary loss: openClean truncates .symbol-dict to its bare header + // (size 0) with zero appends, while every frame -- including the ones + // referencing a@0,b@1,c@2 -- stays on disk. This is the fresh-empty-dict + // state a full-dict-mode prior session or a total host-crash tear leaves. + String slotDir = Paths.get(sfDir, "default").toString(); + try (PersistedSymbolDict torn = PersistedSymbolDict.openClean(slotDir)) { + Assert.assertNotNull(torn); + Assert.assertEquals(0, torn.size()); + } + + // Phase 2: a resuming sender must refuse at build -- the surviving frames + // reference id 2 but the recovered dictionary holds 0 id(s). The I/O-thread + // guard cannot help (deltaStart 0 self-heals the mirror), so the seed-time + // guard is the sole defense. + try (TestWebSocketServer good = new TestWebSocketServer(new DictReconstructingHandler())) { + int port = good.getPort(); + good.start(); + Assert.assertTrue(good.awaitStart(5, TimeUnit.SECONDS)); + String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";"; + try { + Sender resumed = Sender.fromConfig(cfg); + resumed.close(); + Assert.fail("resume on a totally lost (size 0) dictionary must fail clean"); + } catch (LineSenderException expected) { + Assert.assertTrue("message must name the torn-dict/resend failure: " + expected.getMessage(), + expected.getMessage().contains("subset of the surviving frames") + && expected.getMessage().contains("resend")); + } + } + }); + } + private static int globalDictSize(Sender sender) throws Exception { Field f = sender.getClass().getDeclaredField("globalSymbolDictionary"); f.setAccessible(true); From 76fe4f1d34b2ae5d49e863abbfc17952b09bf012 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Mon, 13 Jul 2026 22:43:47 +0100 Subject: [PATCH 45/96] Bound recovered symbol id to committed frames CursorSendEngine.recoveredMaxSymbolId walked every recovered frame, including the aborted orphan-deferred tail -- the frames trySendOne retires without ever transmitting. Those frames typically introduce the highest symbol ids, so a host crash that tore the persisted .symbol-dict down to the committed ids while an orphan-tail frame referenced a higher one inflated recoveredMaxSymbolId and made seedGlobalDictionaryFromPersisted reject an otherwise fully recoverable slot. The producer never reuses an orphan id on the wire -- the tail retires before its new frames send -- so the rejection was a false positive. Bound the maxSymbolDeltaEnd walk to recoveredCommitBoundaryFsn so only committed frames count. MmapSegment and SegmentRing take a maxFsnInclusive argument; a frame whose fsn (baseSeq + i) exceeds it is skipped, and the walk still advances past it. A slot with no orphan tail is unchanged: the boundary equals publishedFsn, so every frame still counts. Add testRecoveredMaxSymbolIdExcludesOrphanTailFrames: a committed delta frame registering ids 0,1 plus a deferred (orphan) frame registering id 2 must recover recoveredMaxSymbolId == 1, not 2. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../client/sf/cursor/CursorSendEngine.java | 20 ++++--- .../qwp/client/sf/cursor/MmapSegment.java | 29 ++++++---- .../qwp/client/sf/cursor/SegmentRing.java | 16 +++--- ...CursorWebSocketSendLoopOrphanTailTest.java | 53 +++++++++++++++++++ 4 files changed, 94 insertions(+), 24 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java index eb59633a..dde24142 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java @@ -321,18 +321,22 @@ private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager man publishedFsn - Math.max(recoveredCommitBoundaryFsn, -1L), recoveredCommitBoundaryFsn, publishedFsn); } - // Highest symbol id the surviving frames reference. A resuming - // producer compares this against its recovered dictionary size - // (seedGlobalDictionaryFromPersisted) to detect a host-crash tear: - // if a frame references an id the (unsynced, torn) .symbol-dict no - // longer holds, resuming would re-use it. maxSymbolDeltaEnd returns - // 0 when no frame carries a symbol, yielding -1 here. Computed - // before the I/O loop or producer append; single-threaded here. + // Highest symbol id the surviving COMMITTED frames reference. A + // resuming producer compares this against its recovered dictionary + // size (seedGlobalDictionaryFromPersisted) to detect a host-crash + // tear: if a committed frame references an id the (unsynced, torn) + // .symbol-dict no longer holds, resuming would re-use it. The walk is + // bounded to recoveredCommitBoundaryFsn so the aborted orphan-deferred + // tail -- retired without ever being transmitted -- does not inflate + // this and over-reject an otherwise-recoverable slot. maxSymbolDeltaEnd + // returns 0 when no such frame carries a symbol, yielding -1 here. + // Computed before the I/O loop or producer append; single-threaded. this.recoveredMaxSymbolId = recovered.maxSymbolDeltaEnd( io.questdb.client.cutlass.qwp.protocol.QwpConstants.MAGIC_MESSAGE, io.questdb.client.cutlass.qwp.protocol.QwpConstants.HEADER_OFFSET_FLAGS, io.questdb.client.cutlass.qwp.protocol.QwpConstants.FLAG_DELTA_SYMBOL_DICT, - io.questdb.client.cutlass.qwp.protocol.QwpConstants.HEADER_SIZE) - 1L; + io.questdb.client.cutlass.qwp.protocol.QwpConstants.HEADER_SIZE, + recoveredCommitBoundaryFsn) - 1L; } else { // Fresh start with no recovered segments. Any stale // watermark from a prior fully-drained session refers diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java index 04f2ac93..a7ee42ec 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java @@ -530,23 +530,34 @@ public long findLastFrameFsnWithoutPayloadFlag(int flagsOffset, int flagMask, in * out-of-order page loss) below the ids the surviving frames still reference: if * the max here reaches at or beyond the recovered dictionary size, a resuming * producer -- seeded from the shorter dictionary -- would re-use ids the frames - * already define. The frame layout mirrors {@link #findLastFrameFsnWithoutPayloadFlag}: - * the QWP message header ({@code qwpHeaderSize} bytes) is followed by the delta - * section {@code [deltaStart varint][deltaCount varint]...}. + * already define. Only frames at or below {@code maxFsnInclusive} count: frames + * above it are the aborted orphan-deferred tail, which {@code trySendOne} retires + * without ever transmitting, so their ids must not inflate the result (a resuming + * producer never reuses them on the wire). The frame layout mirrors + * {@link #findLastFrameFsnWithoutPayloadFlag}: the QWP message header + * ({@code qwpHeaderSize} bytes) is followed by the delta section + * {@code [deltaStart varint][deltaCount varint]...}. * - * @param headerMagic the QWP message magic identifying a well-formed frame - * @param flagsOffset byte offset of the flags field within the QWP header - * @param flagDeltaMask the FLAG_DELTA_SYMBOL_DICT bit - * @param qwpHeaderSize the QWP message header size (delta section starts past it) + * @param headerMagic the QWP message magic identifying a well-formed frame + * @param flagsOffset byte offset of the flags field within the QWP header + * @param flagDeltaMask the FLAG_DELTA_SYMBOL_DICT bit + * @param qwpHeaderSize the QWP message header size (delta section starts past it) + * @param maxFsnInclusive highest frame FSN to consider; frames above it are the + * retired orphan-deferred tail -- pass + * {@code recoveredCommitBoundaryFsn} */ - public long maxSymbolDeltaEnd(int headerMagic, int flagsOffset, int flagDeltaMask, int qwpHeaderSize) { + public long maxSymbolDeltaEnd(int headerMagic, int flagsOffset, int flagDeltaMask, int qwpHeaderSize, long maxFsnInclusive) { long maxEnd = 0L; long off = HEADER_SIZE; long frames = frameCount; for (long i = 0; i < frames; i++) { + long fsn = baseSeq + i; int payloadLen = Unsafe.getUnsafe().getInt(mmapAddress + off + 4); long payload = mmapAddress + off + FRAME_HEADER_SIZE; - if (payloadLen >= qwpHeaderSize + // Skip the orphan-deferred tail (fsn > maxFsnInclusive): retired, never + // sent, so its ids must not count. off still advances below. + if (fsn <= maxFsnInclusive + && payloadLen >= qwpHeaderSize && payloadLen > flagsOffset && Unsafe.getUnsafe().getInt(payload) == headerMagic && (Unsafe.getUnsafe().getByte(payload + flagsOffset) & flagDeltaMask) != 0) { diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java index 2b53e956..21cfa9d1 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java @@ -547,20 +547,22 @@ public synchronized long findLastFsnWithoutPayloadFlag(int flagsOffset, int flag } /** - * Highest {@code deltaStart + deltaCount} any symbol-dict delta frame in the - * ring references (0 when none). See {@link MmapSegment#maxSymbolDeltaEnd}; - * used once at recovery to detect a persisted dictionary torn below the ids - * the surviving frames reference. + * Highest {@code deltaStart + deltaCount} any symbol-dict delta frame at or below + * {@code maxFsnInclusive} references (0 when none). See + * {@link MmapSegment#maxSymbolDeltaEnd}; used once at recovery to detect a + * persisted dictionary torn below the ids the surviving committed frames + * reference. Frames above {@code maxFsnInclusive} are the retired orphan-deferred + * tail and are excluded. */ - public synchronized long maxSymbolDeltaEnd(int headerMagic, int flagsOffset, int flagDeltaMask, int qwpHeaderSize) { + public synchronized long maxSymbolDeltaEnd(int headerMagic, int flagsOffset, int flagDeltaMask, int qwpHeaderSize, long maxFsnInclusive) { long maxEnd = 0L; for (int i = 0, n = sealedSegments.size(); i < n; i++) { - long end = sealedSegments.get(i).maxSymbolDeltaEnd(headerMagic, flagsOffset, flagDeltaMask, qwpHeaderSize); + long end = sealedSegments.get(i).maxSymbolDeltaEnd(headerMagic, flagsOffset, flagDeltaMask, qwpHeaderSize, maxFsnInclusive); if (end > maxEnd) { maxEnd = end; } } - long end = active.maxSymbolDeltaEnd(headerMagic, flagsOffset, flagDeltaMask, qwpHeaderSize); + long end = active.maxSymbolDeltaEnd(headerMagic, flagsOffset, flagDeltaMask, qwpHeaderSize, maxFsnInclusive); return Math.max(maxEnd, end); } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopOrphanTailTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopOrphanTailTest.java index ff710fe1..ed3989ee 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopOrphanTailTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopOrphanTailTest.java @@ -75,7 +75,9 @@ public class CursorWebSocketSendLoopOrphanTailTest { private static final int FLAG_DEFER_COMMIT = 0x01; + private static final int FLAG_DELTA_SYMBOL_DICT = 0x08; private static final int HEADER_OFFSET_FLAGS = 5; + private static final int HEADER_SIZE = 12; private static final int MAGIC_MESSAGE = 0x31505751; // "QWP1" little-endian private String tmpDir; @@ -221,6 +223,36 @@ public void testSlowPathReplaysBelowTailThenRetiresAndRecyclesOnce() throws Exce }); } + @Test + public void testRecoveredMaxSymbolIdExcludesOrphanTailFrames() throws Exception { + // recoveredMaxSymbolId must reflect only COMMITTED (transmitted) frames, not + // the aborted orphan-tail frames trySendOne retires without ever sending. A + // host crash that tears the persisted dictionary down to the committed ids + // while an orphan-tail frame introduced a HIGHER id must NOT over-reject the + // resume: the producer never reuses an orphan id on the wire (the tail retires + // first), so counting it would inflate recoveredMaxSymbolId and make + // seedGlobalDictionaryFromPersisted fail a fully-recoverable slot. The + // maxSymbolDeltaEnd walk is therefore bounded to recoveredCommitBoundaryFsn. + TestUtils.assertMemoryLeak(() -> { + try (CursorSendEngine engine = newEngine()) { + // fsn 0: commit-bearing delta frame registering ids 0,1. + appendDeltaFrame(engine, false, 0, 2); + // fsn 1: DEFERRED delta frame registering id 2 -- the orphan tail. + appendDeltaFrame(engine, true, 2, 1); + } + try (CursorSendEngine engine = newEngine()) { + assertTrue(engine.wasRecoveredFromDisk()); + assertEquals("last commit-bearing frame", 0L, engine.recoveredCommitBoundaryFsn()); + assertEquals("orphan tail tip", 1L, engine.recoveredOrphanTipFsn()); + // Only the committed frame's ids (0,1) count -> highest id 1. The + // orphan-tail frame's id 2 is excluded, so a resume whose recovered + // dictionary holds ids 0,1 (size 2) is NOT over-rejected. + assertEquals("orphan-tail id 2 must be excluded from recoveredMaxSymbolId", + 1L, engine.recoveredMaxSymbolId()); + } + }); + } + // --------------------------------------------------------------------- // harness // --------------------------------------------------------------------- @@ -291,6 +323,27 @@ private static void appendFrame(CursorSendEngine engine, boolean defer) { } } + // Appends a QWP frame carrying a symbol-dict delta section ([deltaStart varint] + // [deltaCount varint]) so the recovery walk's maxSymbolDeltaEnd counts it. + // deltaStart/deltaCount stay < 128 so each encodes in a single LEB128 byte. + private static void appendDeltaFrame(CursorSendEngine engine, boolean defer, int deltaStart, int deltaCount) { + int size = HEADER_SIZE + 2; + long buf = Unsafe.malloc(size, MemoryTag.NATIVE_DEFAULT); + try { + for (int i = 0; i < size; i++) { + Unsafe.getUnsafe().putByte(buf + i, (byte) 0); + } + Unsafe.getUnsafe().putInt(buf, MAGIC_MESSAGE); + Unsafe.getUnsafe().putByte(buf + HEADER_OFFSET_FLAGS, + (byte) (FLAG_DELTA_SYMBOL_DICT | (defer ? FLAG_DEFER_COMMIT : 0))); + Unsafe.getUnsafe().putByte(buf + HEADER_SIZE, (byte) deltaStart); + Unsafe.getUnsafe().putByte(buf + HEADER_SIZE + 1, (byte) deltaCount); + engine.appendBlocking(buf, size); + } finally { + Unsafe.free(buf, size, MemoryTag.NATIVE_DEFAULT); + } + } + private static void awaitAckedFsn(CursorSendEngine engine, long target) throws InterruptedException { long deadline = System.nanoTime() + 10_000_000_000L; while (engine.ackedFsn() < target) { From 084f93f23fdc0a5cf3109dd6d0aae0e06e3c4f3f Mon Sep 17 00:00:00 2001 From: glasstiger Date: Mon, 13 Jul 2026 22:44:01 +0100 Subject: [PATCH 46/96] Null loaded dict entries after a failed truncate PersistedSymbolDict.openExisting frees the loaded-entries buffer in the truncate-failure branch, then calls ff.close(fd). If that close threw, the enclosing catch re-freed the same buffer -- a double-free -- because, unlike the bad-magic branch, this branch did not null the pointer first. Null entriesAddr/entriesLen right after freeing them, matching the buf discipline above, so the catch's guard skips the second free. Correct the catch comment, which wrongly claimed nothing between the malloc and the return could throw -- the truncate branch's free-then-close is exactly such a point. This is defensive hardening: FilesFacade.close returns a status code and never throws (the native close never raises), so the path is unreachable today, but the nulling keeps it safe against a future facade or edit. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../qwp/client/sf/cursor/PersistedSymbolDict.java | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java index 9930a239..70cadb19 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java @@ -608,6 +608,10 @@ private static PersistedSymbolDict openExisting(FilesFacade ff, String filePath, LOG.warn("symbol dict {} could not drop its torn/stale tail (truncate failed); recreating", filePath); if (entriesAddr != 0L) { Unsafe.free(entriesAddr, entriesLen, MemoryTag.NATIVE_DEFAULT); + // Null after freeing (like buf above) so the catch below cannot + // double-free entriesAddr if the following ff.close throws. + entriesAddr = 0L; + entriesLen = 0; } ff.close(fd); return null; @@ -617,10 +621,11 @@ private static PersistedSymbolDict openExisting(FilesFacade ff, String filePath, if (buf != 0L) { Unsafe.free(buf, (int) fileLen, MemoryTag.NATIVE_DEFAULT); } - // entriesAddr is transferred to the returned dict on the success path, - // and the catch only runs before that return, so freeing it here cannot - // double-free. Unreachable today (nothing between its malloc and the - // return throws), but keeps the error path leak-free against a future edit. + // Free entriesAddr if it was allocated and not yet handed off. The success + // path transfers it to the returned dict, and every path that frees it + // earlier (the truncate-failure branch above) also nulls it, so this cannot + // double-free. Keeps the error path leak-free on any throw between its + // malloc and the return. if (entriesAddr != 0L) { Unsafe.free(entriesAddr, entriesLen, MemoryTag.NATIVE_DEFAULT); } From 8facb8d2627f41cc51f206f68d901486f6988cfa Mon Sep 17 00:00:00 2001 From: glasstiger Date: Mon, 13 Jul 2026 22:44:16 +0100 Subject: [PATCH 47/96] Test the catch-up NACK guard at the ack boundary The catch-up-NACK routing guard (fsn <= ackedFsn) had its equality case untested: the common single-frame catch-up maps its only catch-up frame to fsn == ackedFsn exactly, yet the existing test only exercised the strictly-below case (fsn = -2 < ackedFsn = -1). A "<=" -> "<" mutation therefore survived the suite while routing the single (or last) catch-up frame's NACK onto the poison-strike path and laundering the detector. Add testPostSendCatchUpNackAtAckedFsnBoundaryDoesNotStrike, which drives fsn == ackedFsn == -1 and asserts poisonStrikes stays 0. poisonStrikes, not poisonFsn, is the discriminator here: a wrongly-charged strike sets poisonFsn to the rejected fsn -1, which is the "no suspect" sentinel, so a poisonFsn check is blind at this boundary. The mutation charges the strike (0 -> 1) and the test fails; the production guard is unchanged. Co-Authored-By: Claude Opus 4.8 (1M context) --- ...ursorWebSocketSendLoopPoisonFrameTest.java | 42 +++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopPoisonFrameTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopPoisonFrameTest.java index d413d82e..d63887db 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopPoisonFrameTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopPoisonFrameTest.java @@ -696,6 +696,48 @@ public void testPostSendCatchUpNackDoesNotStrikeOrLaunderPoisonState() throws Ex }); } + @Test + public void testPostSendCatchUpNackAtAckedFsnBoundaryDoesNotStrike() throws Exception { + // Boundary companion to testPostSendCatchUpNackDoesNotStrikeOrLaunderPoisonState: + // the COMMON single-frame catch-up maps its only catch-up frame to + // fsn == ackedFsn EXACTLY (the last catch-up frame lands on replayStart-1 == + // ackedFsn). The pre-send routing guard is "fsn <= ackedFsn", so this equality + // case must also skip the poison strike. Without pinning it a "<=" -> "<" + // mutation would slip the single/last catch-up frame's NACK onto the post-send + // poison-strike path and launder the detector -- exactly the hazard the guard + // closes -- yet the strictly-below test above would still pass. + TestUtils.assertMemoryLeak(() -> { + List clients = new ArrayList<>(); + try (CursorSendEngine engine = newEngine()) { + appendFrames(engine, 2); + CursorWebSocketSendLoop loop = newDurableLoop(engine, clients); + // 1 catch-up frame (wire seq 0) + 1 data frame (wire seq 1), nothing + // acked (ackedFsn = -1), so fsnAtZero = replayStart(0) - catchUpFrames(1) + // = -1. A NACK of catch-up wire seq 0 maps to fsn -1 == ackedFsn exactly. + setPostCatchUpDataFrameBaseline(loop, -1L, 2L); + assertEquals("precondition: no strikes yet", + 0L, getLongField(loop, "poisonStrikes")); + + deliverRetriableNack(loop, 0, "disk full"); + + // poisonStrikes -- NOT poisonFsn -- is the discriminator at this exact + // boundary: a wrongly-charged strike calls recordHeadRejectionStrike(-1), + // which sets poisonFsn to the rejected fsn -1 -- the "no suspect" + // sentinel -- so a poisonFsn check cannot see the laundering here. It + // still bumps poisonStrikes 0 -> 1, so the correct pre-send routing (no + // strike) shows as poisonStrikes staying 0. A "<=" -> "<" mutation of the + // fsn-vs-ackedFsn guard charges the strike and fails this. + assertEquals("catch-up NACK at fsn == ackedFsn must not charge a poison strike", + 0L, getLongField(loop, "poisonStrikes")); + assertEquals("... and must leave the poison sentinel intact", + -1L, getLongField(loop, "poisonFsn")); + loop.checkError(); + } finally { + closeAll(clients); + } + }); + } + @Test public void testPostSendNotWritableNackNeverEscalatesToPoisonTerminal() throws Exception { // RETRIABLE_OTHER (NOT_WRITABLE) is a node-state verdict, not a frame From 031777e27b4452f8f9c228fe7a5d6e19c3849d4d Mon Sep 17 00:00:00 2001 From: glasstiger Date: Mon, 13 Jul 2026 23:00:04 +0100 Subject: [PATCH 48/96] Cover the cap-gap budget reset on success The cap-gap settle budget (catchUpCapGapAttempts) counts consecutive catch-up cap gaps across reconnects and resets to 0 on a successful catch-up, so gaps interspersed with successes -- a rolling-cap cluster -- never accumulate to a spurious terminal. The existing latch test only accrues gaps under one fixed cap with no success interleaved, so a dropped reset would have gone unnoticed. Add testSuccessfulCatchUpResetsCapGapBudget: accrue max-1 cap gaps, let a larger-cap node accept the dictionary, then assert the budget is back to 0 and that max-1 further gaps still latch no terminal. Make the stub client's advertised cap mutable so the test can model the rolling cap. Co-Authored-By: Claude Opus 4.8 (1M context) --- ...WebSocketSendLoopCatchUpAlignmentTest.java | 61 ++++++++++++++++++- 1 file changed, 60 insertions(+), 1 deletion(-) diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java index 54fd645c..c21b7d82 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java @@ -321,6 +321,63 @@ public void testCatchUpCapGapRetriesUntilBudgetThenLatches() throws Exception { }); } + @Test + public void testSuccessfulCatchUpResetsCapGapBudget() throws Exception { + // The cap-gap settle budget (catchUpCapGapAttempts) counts CONSECUTIVE cap + // gaps across reconnects; a successful catch-up ends the episode and MUST reset + // it to 0 (sendDictCatchUp's final line). Otherwise cap gaps interspersed with + // successful catch-ups -- a rolling-cap cluster where a larger-cap node comes + // and goes -- would accumulate to a spurious terminal over a long-lived sender. + // testCatchUpCapGapRetriesUntilBudgetThenLatches only accrues gaps under one + // fixed cap with no success interleaved, so it cannot pin the reset. + TestUtils.assertMemoryLeak(() -> { + Field maxField = CursorWebSocketSendLoop.class.getDeclaredField("MAX_CATCHUP_CAP_GAP_ATTEMPTS"); + maxField.setAccessible(true); + int maxAttempts = maxField.getInt(null); + CatchUpCapturingClient client = new CatchUpCapturingClient(160); // too small for a 200-char symbol + try (CursorSendEngine engine = newEngine()) { + CursorWebSocketSendLoop loop = newLoop(engine, client); + try { + seedMirror(loop, TestUtils.repeat("x", 200)); + // Accrue max-1 consecutive cap gaps (each retriable, no terminal). + for (int i = 1; i < maxAttempts; i++) { + try { + invokeSetWireBaselineWithCatchUp(loop, engine.ackedFsn() + 1L); + fail("cap gap must raise a retriable CatchUpSendException (attempt " + i + ')'); + } catch (InvocationTargetException e) { + assertEquals("CatchUpSendException", e.getCause().getClass().getSimpleName()); + } + } + assertEquals("precondition: budget accrued to max-1", + maxAttempts - 1, readInt(loop, "catchUpCapGapAttempts")); + + // A larger-cap node returns: the whole dictionary re-registers with + // no cap gap, so the settle budget must reset to 0. + client.cap = 0; // no cap => the 200-char symbol fits one frame + invokeSetWireBaselineWithCatchUp(loop, engine.ackedFsn() + 1L); + assertEquals("a successful catch-up must reset the cap-gap settle budget", + 0, readInt(loop, "catchUpCapGapAttempts")); + + // Behavioural proof the budget is genuinely fresh: max-1 more cap + // gaps still latch NO terminal (they would if the counter had stayed + // at max-1 -- one more gap would have hit the cap and killed the sender). + client.cap = 160; + for (int i = 1; i < maxAttempts; i++) { + try { + invokeSetWireBaselineWithCatchUp(loop, engine.ackedFsn() + 1L); + fail("post-reset cap gap must be retriable (attempt " + i + ')'); + } catch (InvocationTargetException e) { + assertEquals("CatchUpSendException", e.getCause().getClass().getSimpleName()); + } + loop.checkError(); // fresh budget => still under max => no terminal + } + } finally { + loop.close(); + } + } + }); + } + @Test public void testMirrorOverflowFailsLoud() throws Exception { // ensureSentDictCapacity must latch a terminal -- not silently overflow the @@ -539,7 +596,9 @@ private static long writeVarint(long addr, long value) { // when throwOnSend is set -- raises a transient wire error to model the fresh // connection dropping mid-catch-up. private static final class CatchUpCapturingClient extends WebSocketClient { - private final int cap; + // Mutable so a test can model a rolling-cap cluster: raise it for a node that + // accepts the dictionary, lower it for a smaller-cap node that cap-gaps. + private int cap; private final boolean throwOnSend; private int framesSent; From 0c0b9dc55b1685e4c7219f4fe9fde5b91cae777d Mon Sep 17 00:00:00 2001 From: glasstiger Date: Mon, 13 Jul 2026 23:00:17 +0100 Subject: [PATCH 49/96] Cover file-mode split write-ahead dict persist flushPendingRowsSplit's write-ahead dictionary persist (the appendRawEntries fast path) runs only in the exceptional over-cap split path, and both split tests were memory-mode, so the split-times-persist path was never exercised with a live PersistedSymbolDict -- a recovered slot relies on it to rebuild the delta frames. Add testFileModeSplitPersistsDictBeforePublish: a file-mode two-table batch that exceeds the server cap splits into two frames, and the first frame's persist must record both new symbols in .symbol-dict. Dropping the split persist leaves the file empty and fails the test; the non-split file persist test is unaffected. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../qwp/client/SelfSufficientFramesTest.java | 49 +++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java index 524063a1..684f0239 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java @@ -332,6 +332,55 @@ public void testOversizedTableSplitStrandsNothing() throws Exception { }); } + @Test + public void testFileModeSplitPersistsDictBeforePublish() throws Exception { + // File-mode store-and-forward + a SPLIT flush: a two-table batch whose combined + // size exceeds the server cap splits into one frame per table + // (flushPendingRowsSplit). The first split frame's write-ahead persist + // (persistNewSymbolsBeforePublish, the appendRawEntries fast path) must record + // the batch's new symbols in .symbol-dict BEFORE the frames publish, so a + // recovered / orphan-drained slot can rebuild what the delta frames reference. + // The other split tests run in memory mode, so this is the only coverage of the + // split x persist path with a live PersistedSymbolDict. + Path sfDir = Files.createTempDirectory("qwp-sf-split-persist"); + try { + assertMemoryLeak(() -> { + CapturingHandler handler = new CapturingHandler(); + try (TestWebSocketServer server = new TestWebSocketServer(handler)) { + server.setAdvertisedMaxBatchSize(150); // forces the two-table batch to split + int port = server.getPort(); + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + + String config = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";"; + Path dictFile = sfDir.resolve("default").resolve(".symbol-dict"); + String pad = new String(new char[60]).replace('\0', 'x'); + try (Sender sender = Sender.fromConfig(config)) { + // Two tables, two new symbols, ONE flush -> the combined message + // exceeds cap 150 and splits into two frames. + sender.table("t1").symbol("s", "alpha").stringColumn("p", pad).longColumn("v", 1L).atNow(); + sender.table("t2").symbol("s", "bravo").stringColumn("p", pad).longColumn("v", 2L).atNow(); + sender.flush(); + waitFor(() -> handler.batches.size() >= 2, 5_000); + + // Check .symbol-dict while the sender is live: a fully-drained + // close would unlink it. The split's first-frame write-ahead + // persist must have recorded BOTH new symbols. + Assert.assertTrue("persisted dictionary file exists", Files.exists(dictFile)); + byte[] dict = Files.readAllBytes(dictFile); + Assert.assertTrue("split-flush persist must record alpha", containsUtf8(dict, "alpha")); + Assert.assertTrue("split-flush persist must record bravo", containsUtf8(dict, "bravo")); + } + + Assert.assertEquals("the oversized two-table batch must split into 2 frames", + 2, handler.batches.size()); + } + }); + } finally { + rmDir(sfDir); + } + } + private static int readVarint(byte[] buf, int offset) { // Simple unsigned varint decode — sufficient for small values. int result = 0; From bbc01a82da19ca9097b8a81c6e1617fc22729bc8 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Tue, 14 Jul 2026 02:00:11 +0100 Subject: [PATCH 50/96] Cover split-preflight and torn-dict guard gaps Address review findings on the delta symbol-dictionary change. Tests (close two coverage gaps): - SelfSufficientFramesTest pins the split-flush pre-flight baseline advance: a large-delta two-table split whose second frame fits only with an empty delta. Removing the simBaseline advance mis-sizes that frame with the delta and wrongly rejects a shippable batch. - CursorWebSocketSendLoopOrphanTailTest pins that a zero-count delta frame anchors recoveredMaxSymbolId at its baseline, not 0. Skipping it would under-strand a torn dictionary and risk a silent id shift. Docs (correct load-bearing recovery semantics): - seedGlobalDictionaryFromPersisted no longer claims the drainer always self-heals: it does so only when a surviving frame straddles the tear; a higher-baseline survivor quarantines the slot, a deliberate conservative over-strand. - maxSymbolDeltaEnd documents that a zero-count delta frame contributes its deltaStart, not 0. - The catch-up cap-gap budget documents that a transient reconnect never increments or resets the counter (only a successful catch-up resets it), so a transient cannot burn the terminal budget. Hardening: - writeVarint and putVarint assert a non-negative value: the signed loop writes one truncated byte for a negative long while varintSize returns 10. - PersistedSymbolDict.open diverts a file at exactly Integer.MAX_VALUE to a clean re-create instead of a doomed 2GB malloc (> becomes >=). - readLoadedSymbols bounds its varint decode (shift > 35) like its sibling parsers. Production behavior changes are limited to the two varint asserts, the >= boundary, and the defensive varint bound; the rest is tests and documentation. Full delta-dict suite green on JDK 25 with -ea. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../qwp/client/NativeBufferWriter.java | 7 ++ .../qwp/client/QwpWebSocketSender.java | 20 +++++- .../sf/cursor/CursorWebSocketSendLoop.java | 46 ++++++++----- .../qwp/client/sf/cursor/MmapSegment.java | 13 +++- .../client/sf/cursor/PersistedSymbolDict.java | 31 ++++++--- .../qwp/client/SelfSufficientFramesTest.java | 69 +++++++++++++++++++ ...CursorWebSocketSendLoopOrphanTailTest.java | 47 +++++++++++++ 7 files changed, 200 insertions(+), 33 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/NativeBufferWriter.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/NativeBufferWriter.java index cb6ebb92..86343772 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/NativeBufferWriter.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/NativeBufferWriter.java @@ -81,8 +81,14 @@ public static int varintSize(long value) { * {@code addr} and returns the address just past the last byte. The canonical * raw-address varint writer shared by the SF cursor's persisted dictionary and * catch-up frame builder. + *

+ * {@code value} must be non-negative: the signed {@code value > 0x7F} loop emits + * a SINGLE truncated byte for a negative long, whereas {@link #varintSize} + * returns 10 for it -- a size/write mismatch that would corrupt the stream. All + * callers pass ids/lengths/counts (non-negative); the assert pins that contract. */ public static long writeVarint(long addr, long value) { + assert value >= 0 : "unsigned LEB128 varint requires a non-negative value: " + value; while (value > 0x7F) { Unsafe.getUnsafe().putByte(addr++, (byte) ((value & 0x7F) | 0x80)); value >>>= 7; @@ -320,6 +326,7 @@ public void putUtf8(CharSequence value) { */ @Override public void putVarint(long value) { + assert value >= 0 : "unsigned LEB128 varint requires a non-negative value: " + value; ensureCapacity(10); // max varint bytes long addr = bufferPtr + position; while (value > 0x7F) { diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index 4f7c374b..70a7e877 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -3876,9 +3876,23 @@ private void resetSymbolDictStateForNewConnection() { * mirror, leaving only this producer diverged. Detect it here -- the surviving * frames reference an id at or beyond the recovered dictionary size -- and fail * clean: the affected data must be resent, matching the design's torn-dict - * "resend required" contract. The recovered data is not lost; the background - * drainer still drains this slot (its mirror self-heals from the frames). Only a - * host crash reaches this -- a process crash keeps the page cache, so the + * "resend required" contract. + *

+ * The background drainer self-heals the mirror ONLY when a surviving frame + * STRADDLES the tear ({@code deltaStart <= pd.size() < deltaStart + deltaCount}): + * such a frame carries the torn-off ids in its own delta and + * {@code accumulateSentDict} re-registers them, so the drainer drains the slot. + * But when the symbol-introducing frames were already acked and trimmed and only + * a HIGHER-baseline frame survives ({@code deltaStart > pd.size()} -- e.g. a + * commit or a symbol-reusing frame, since {@code beginMessage} always sets the + * delta flag), the drainer's own replay guard ({@code deltaStart > sentDictCount}) + * fires too and quarantines the slot: the recorded bytes are not silently lost, + * but the slot is NOT auto-drained -- it must be resent. That is a deliberate + * CONSERVATIVE over-strand -- the guard keys on {@code deltaStart}, not on the + * frame's actual highest referenced id, to avoid parsing row data at recovery, + * so it may reject a frame whose rows reference only ids the truncated + * dictionary still holds. It fails clean rather than risk a silent id shift. + * Only a host crash reaches this -- a process crash keeps the page cache, so the * write-ahead ordering keeps the dictionary a superset of the frames. */ private void seedGlobalDictionaryFromPersisted(PersistedSymbolDict pd) { diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java index f55f47c8..1ea87523 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java @@ -133,17 +133,30 @@ public final class CursorWebSocketSendLoop implements QuietCloseable { */ public static final long DEFAULT_POISON_MIN_ESCALATION_WINDOW_MILLIS = 5_000L; private static final Logger LOG = LoggerFactory.getLogger(CursorWebSocketSendLoop.class); - // Settle budget for the symbol-dict catch-up cap gap: how many CONSECUTIVE - // reconnect attempts may find a single dictionary entry too large for the fresh - // server's advertised batch cap before the sender latches a terminal. A - // homogeneous cluster never trips it -- an entry that fit its data frame under a - // cap always fits its bare catch-up frame under that same cap -- so this only - // affects a heterogeneous / rolling-cap cluster, where a failover to a - // smaller-cap node can hit it for an entry an earlier node accepted. Retrying - // rides out the transient window until a larger-cap node returns; only a - // persistent gap (every reachable node too small for this many attempts) latches - // terminal, matching the orphan drainer's DEFAULT_MAX_DURABLE_ACK_MISMATCH_ATTEMPTS. - // A successful catch-up resets the counter (see sendDictCatchUp). + // Settle budget for the symbol-dict catch-up cap gap: how many cap-gap attempts + // -- catch-ups that reached a fresh server and found a single dictionary entry + // too large for its advertised batch cap -- may occur, with no intervening + // SUCCESSFUL catch-up, before the sender latches a terminal. This is a SANCTIONED + // terminal (a genuine cluster batch-size capability gap), the connect-time analog + // of the orphan drainer's durable-ack capability gap + // (DEFAULT_MAX_DURABLE_ACK_MISMATCH_ATTEMPTS). A homogeneous cluster never trips + // it -- an entry that fit its data frame under a cap always fits its bare catch-up + // frame under that same cap -- so it only affects a heterogeneous / rolling-cap + // cluster, where a failover to a smaller-cap node can hit it for an entry an + // earlier node accepted. Retrying rides out the transient window until a + // larger-cap node returns; only a persistent gap (this many cap gaps with no + // successful catch-up in between) latches. + // + // Budget accounting (satisfies "a transient must never burn the terminal + // budget"): catchUpCapGapAttempts increments ONLY inside sendDictCatchUp when a + // node is reached and an entry is oversized, and resets ONLY when a catch-up fully + // succeeds. A TRANSIENT reconnect (connect refuse, upgrade/role failure) never + // reaches the catch-up, so it NEITHER increments nor resets the counter -- it only + // lengthens the wall-clock settle window. The terminal therefore always requires + // this many GENUINE cap gaps; a transient can never inflate it. Deliberately NOT + // reset on a mere successful RECONNECT: a reconnect to the small-cap node itself + // produces the cap gap, so resetting there would stop a persistent gap from ever + // latching -- the reset must gate on a successful CATCH-UP, not on connecting. private static final int MAX_CATCHUP_CAP_GAP_ATTEMPTS = 16; // Hard ceiling for the lifetime-monotonic sent-dictionary mirror. The mirror // fields are int, so it cannot exceed Integer.MAX_VALUE bytes; reaching even @@ -218,11 +231,12 @@ public final class CursorWebSocketSendLoop implements QuietCloseable { // for the connection's lifetime (a reconnect may need the whole dictionary at // any moment), so it cannot be dropped; it is an intentional cost of the feature. private final boolean deltaDictEnabled; - // Consecutive reconnect attempts whose symbol-dict catch-up found an entry too - // large for the fresh server's batch cap (see MAX_CATCHUP_CAP_GAP_ATTEMPTS). A - // successful catch-up resets it; it is NOT reset per connection -- it measures - // the cap-gap episode across reconnects so a persistent gap eventually latches. - // I/O-thread-only. + // Cap-gap attempts -- catch-ups that reached a node and found an entry too large + // for its batch cap -- since the last SUCCESSFUL catch-up (see + // MAX_CATCHUP_CAP_GAP_ATTEMPTS for the full budget accounting). A successful + // catch-up resets it (sendDictCatchUp); a transient reconnect neither increments + // nor resets it. NOT reset per connection -- it measures the cap-gap episode + // across reconnects so a persistent gap eventually latches. I/O-thread-only. private int catchUpCapGapAttempts; // True once a real ring frame (data or commit) has been sent on the CURRENT // connection, as opposed to only the dictionary catch-up. The catch-up diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java index a7ee42ec..50fc18ff 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java @@ -523,9 +523,16 @@ public long findLastFrameFsnWithoutPayloadFlag(int flagsOffset, int flagMask, in } /** - * Highest {@code deltaStart + deltaCount} (one past the highest symbol id) that - * any symbol-dict delta frame in this segment references, or {@code 0} when no - * such frame carries a symbol. Read-only walk over the recovered frames, used + * Highest {@code deltaStart + deltaCount} (one past the highest symbol id) any + * delta-flagged frame in this segment carries, or {@code 0} only when NO + * delta-flagged frame is present. A frame with {@code deltaCount == 0} (a commit + * or a symbol-reusing frame -- the encoder always sets the delta flag) still + * contributes its {@code deltaStart}, i.e. the producer's baseline at encode + * time, NOT 0. That is deliberate: it anchors the torn-dict guard at that + * baseline so a dictionary torn below it is detected -- a conservative + * over-strand (it may over-reject a frame whose rows reference only surviving + * ids), never an under-strand that could silently shift the dense id map. + * Read-only walk over the recovered frames, used * once at recovery to detect a persisted {@code .symbol-dict} torn (host crash, * out-of-order page loss) below the ids the surviving frames still reference: if * the max here reaches at or beyond the recovered dictionary size, a resuming diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java index 70cadb19..51c1b19b 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java @@ -170,17 +170,19 @@ public static PersistedSymbolDict open(String slotDir) { public static PersistedSymbolDict open(FilesFacade ff, String slotDir) { String filePath = slotDir + "/" + FILE_NAME; long existing = ff.exists(filePath) ? ff.length(filePath) : -1L; - // A dictionary that legitimately grew past Integer.MAX_VALUE cannot be - // reopened: openExisting reads it into ONE int-sized native buffer, and - // the (int) cast of a >2GB length is either negative (malloc rejects it), - // exactly zero (getInt then reads 4 bytes past a zero-size allocation), or - // a small positive prefix (whose validLen < len branch would then - // DESTRUCTIVELY truncate the multi-GB file). Recreate empty instead -- - // fail-clean, exactly like every other unreadable-file case here, so the - // sender falls back to full self-sufficient frames. Reaching this needs - // ~100M+ distinct symbols on one slot (far past realistic symbol - // cardinality); the guard keeps the read/write size boundary safe anyway. - if (existing > Integer.MAX_VALUE) { + // A dictionary that grew to or past Integer.MAX_VALUE cannot be reopened: + // openExisting reads it into ONE int-sized native buffer. PAST 2GiB the + // (int) cast is either negative (malloc rejects it), exactly zero (getInt + // then reads 4 bytes past a zero-size allocation), or a small positive prefix + // (whose validLen < len branch would then DESTRUCTIVELY truncate the multi-GB + // file); AT exactly Integer.MAX_VALUE the cast is exact but the ~2GB malloc is + // doomed to OutOfMemoryError. The >= guard short-circuits every case to a + // clean re-create instead of the doomed allocation -- fail-clean, exactly like + // every other unreadable-file case here, so the sender falls back to full + // self-sufficient frames. Reaching this needs ~100M+ distinct symbols on one + // slot (far past realistic symbol cardinality); the guard keeps the read/write + // size boundary safe anyway. + if (existing >= Integer.MAX_VALUE) { LOG.warn("symbol dict {} too large ({} bytes) to reopen; recreating empty", filePath, existing); return openFresh(ff, filePath); } @@ -463,6 +465,13 @@ public ObjList readLoadedSymbols() { break; } shift += 7; + if (shift > 35) { + // Bound the varint like decodeVarint / appendRawEntries / + // readVarintAt: a canonical length is <= 5 bytes. open() already + // CRC-validated these bytes, so this is defensive only; the + // p + len > limit check below then rejects the over-long run. + break; + } } if (p + len > limit) { break; // defensive: torn tail (should not happen past parse in open) diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java index 684f0239..5729aa44 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java @@ -381,6 +381,75 @@ public void testFileModeSplitPersistsDictBeforePublish() throws Exception { } } + @Test + public void testSplitPreflightAdvancesBaselineSoLaterFramesArentSizedWithTheDelta() throws Exception { + // Regression for the split pre-flight baseline advance in flushPendingRowsSplit + // (the "Mirror advanceSentMaxSymbolId" step). Only the FIRST split frame ships + // the batch's symbol-dict delta; the rest ship an EMPTY delta and reference ids + // the first frame already registered. The pre-flight size pass must advance + // simBaseline after the first table so it STOPS adding combinedDeltaEntriesLen + // to the later frames' estimated sizes. Without that advance, a later table + // whose real (empty-delta) frame fits the cap is mis-estimated as still carrying + // the whole delta and wrongly rejected with "single table batch too large" -- + // discarding a legitimately shippable batch (fail-closed data loss). + // + // Shape (memory mode, delta enabled): a LARGE combined delta (two ~64-char + // symbols) rides only the first split frame. The first table (added first, so + // the first split frame) has a tiny body, so delta + body1 fits the cap. The + // second table has a big body: body2 alone fits the cap, but delta + body2 does + // NOT. The real code splits and ships both frames; the un-advanced pre-flight + // would size the second frame WITH the delta and throw. Table order is + // insertion order (CharSequenceObjHashMap.keys()), so t1 is the delta frame. + assertMemoryLeak(() -> { + CapturingHandler handler = new CapturingHandler(); + try (TestWebSocketServer server = new TestWebSocketServer(handler)) { + server.setAdvertisedMaxBatchSize(200); + int port = server.getPort(); + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + + // Two long symbols => a large combined delta section (~130 bytes) that + // rides ONLY the first split frame. The symbol STRINGS live in the delta, + // not in either table body (the body carries only the varint global id). + String longSymA = new String(new char[64]).replace('\0', 'a'); + String longSymB = new String(new char[64]).replace('\0', 'b'); + String bigPad = new String(new char[100]).replace('\0', 'x'); + // auto_flush off so both rows batch into one flush (byte-based auto-flush + // is otherwise clamped under the cap and would flush before the split). + try (Sender sender = Sender.fromConfig("ws::addr=localhost:" + port + + ";auto_flush_bytes=off;auto_flush_rows=1000000;auto_flush_interval=60000;")) { + // t1 (added first -> first split frame): carries the whole delta but a + // tiny body, so delta + body1 fits the 200-byte cap. + sender.table("t1").symbol("s", longSymA).longColumn("v", 1L).atNow(); + // t2 (second split frame): empty delta but a big body. body2 alone + // fits the cap; delta + body2 does NOT -- the mis-size the advance + // prevents. + sender.table("t2").symbol("s", longSymB).stringColumn("p", bigPad).longColumn("v", 2L).atNow(); + // Must NOT throw: with the baseline advanced, t2's frame is sized + // WITHOUT the delta and fits. A broken advance throws "too large" here. + sender.flush(); + waitFor(() -> handler.batches.size() >= 2, 5_000); + } + + Assert.assertEquals("the batch must split into 2 frames, neither spuriously rejected", + 2, handler.batches.size()); + byte[] f1 = handler.batches.get(0); + byte[] f2 = handler.batches.get(1); + // First split frame ships the whole delta (both new symbols, ids 0 and 1). + Assert.assertEquals("first split frame deltaStart must be 0", + 0, readVarint(f1, DELTA_START_OFFSET)); + Assert.assertEquals("first split frame ships both new symbols", + 2, readVarint(f1, DELTA_START_OFFSET + 1)); + // Second split frame carries an EMPTY delta above the advanced baseline -- + // the whole point: it is not re-sized (or re-sent) with the delta. + Assert.assertEquals("second split frame deltaStart must be 2 (baseline advanced)", + 2, readVarint(f2, DELTA_START_OFFSET)); + Assert.assertEquals("second split frame carries no new symbols", + 0, readVarint(f2, DELTA_START_OFFSET + 1)); + } + }); + } + private static int readVarint(byte[] buf, int offset) { // Simple unsigned varint decode — sufficient for small values. int result = 0; diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopOrphanTailTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopOrphanTailTest.java index ed3989ee..bf16f3b0 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopOrphanTailTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopOrphanTailTest.java @@ -253,6 +253,53 @@ public void testRecoveredMaxSymbolIdExcludesOrphanTailFrames() throws Exception }); } + @Test + public void testZeroCountDeltaFrameAnchorsRecoveredMaxSymbolIdAtItsBaseline() throws Exception { + // A committed delta frame that introduces NO new symbol (deltaCount == 0 -- a + // commit frame, or one whose rows only re-use existing ids) still carries + // deltaStart == the producer's baseline at encode time, because beginMessage + // ALWAYS sets FLAG_DELTA_SYMBOL_DICT. maxSymbolDeltaEnd counts it as + // deltaStart + deltaCount == deltaStart (NOT 0), so recoveredMaxSymbolId == + // deltaStart - 1 even though the frame introduces nothing. + // + // This is the mechanism behind the torn-dict guard's deliberate CONSERVATIVE + // over-strand (see seedGlobalDictionaryFromPersisted): if a host crash tears + // the persisted dictionary below such a frame's baseline while its + // symbol-introducing predecessors were already acked and trimmed, both the + // seed-time guard (recoveredMaxSymbolId >= pd.size()) and the drainer's replay + // guard (deltaStart > sentDictCount) fire and quarantine the slot -- fail-clean + // "resend required" -- even though the frame's rows may reference only ids the + // truncated dictionary still holds. + // + // Counting the zero-count frame's baseline is load-bearing SAFETY: a "fix" that + // skipped zero-count frames (returning 0 for them) would UNDER-strand and let a + // genuinely torn dictionary through, silently shifting the dense id map. This + // pins that a zero-count delta frame is anchored at its baseline, not skipped. + TestUtils.assertMemoryLeak(() -> { + try (CursorSendEngine engine = newEngine()) { + // fsn 0: commit-bearing delta frame that genuinely registers ids 0..4. + appendDeltaFrame(engine, false, 0, 5); + // fsn 1: commit-bearing delta frame with deltaStart 10, deltaCount 0 -- + // introduces NOTHING, but its baseline (10) sits ABOVE every id any + // surviving frame actually introduces (max 4). Models a commit / + // symbol-reusing frame emitted after ids 5..9 were registered by + // predecessor frames that have since been acked and trimmed away. + appendDeltaFrame(engine, false, 10, 0); + } + try (CursorSendEngine engine = newEngine()) { + assertTrue(engine.wasRecoveredFromDisk()); + assertEquals("both frames are commit-bearing", 1L, engine.recoveredCommitBoundaryFsn()); + // The zero-count frame drives recoveredMaxSymbolId to 9 (its baseline + // 10, minus 1), NOT to 4 (the highest id any surviving frame actually + // introduces) and NOT to 0 (which skipping it would yield). This + // inflation is exactly what makes seedGlobalDictionaryFromPersisted + // over-reject a dictionary holding ids 0..4 (size 5). + assertEquals("a zero-count delta frame anchors recoveredMaxSymbolId at its baseline-1", + 9L, engine.recoveredMaxSymbolId()); + } + }); + } + // --------------------------------------------------------------------- // harness // --------------------------------------------------------------------- From 39e9e52fd3ce0323cc5fafac87d3280a0a0dde47 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Tue, 14 Jul 2026 12:22:39 +0100 Subject: [PATCH 51/96] Cover the persisted symbol-dict short-write path The failed-publish regression tests exercise a persist that SUCCEEDED (the publish is what fails), so nothing pinned the persist-FAILURE trigger: a short write (disk full / quota) mid-persist. Add a ShortWriteOnceFacade that lands one entry append short and two tests -- one per production persist path (appendRawEntries fast path, appendSymbols slow path) -- asserting the load-bearing idempotency invariant the write-ahead persist relies on: a short write throws WITHOUT advancing size()/appendOffset, so a retry keyed off pd.size() re-persists the same range at the same offset and recovers a gap-free, duplicate-free dictionary. Verified as regression guards: reordering the production code to advance size before the written==len check fails both tests with "short write must NOT advance size expected:<0> but was:<2>". Co-Authored-By: Claude Opus 4.8 (1M context) --- .../sf/cursor/PersistedSymbolDictTest.java | 146 ++++++++++++++++++ 1 file changed, 146 insertions(+) diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java index eb79e016..677e12e2 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java @@ -146,6 +146,78 @@ public void testAppendRawEntriesMatchesAppendSymbols() throws Exception { }); } + @Test + public void testAppendRawEntriesShortWriteThrowsWithoutAdvancingIsIdempotentOnRetry() throws Exception { + // The producer's FAST path persists a frame's already-encoded delta bytes + // via appendRawEntries. A short write (disk full / quota) mid-persist must + // throw WITHOUT advancing size()/appendOffset, so a retry keyed off size() + // re-writes the identical bytes at the same offset and recovers a gap-free, + // duplicate-free dictionary. The failed-PUBLISH regressions + // (DeltaDictRecoveryTest) exercise a persist that SUCCEEDED; only this pins + // the persist-FAILURE trigger, whose idempotency the write-ahead + // (persistNewSymbolsBeforePublish, resuming from pd.size()) relies on. + assertMemoryLeak(() -> { + Path src = Files.createTempDirectory("qwp-symdict-src"); + Path dst = Files.createTempDirectory("qwp-symdict-dst"); + try { + GlobalSymbolDictionary dict = new GlobalSymbolDictionary(); + dict.getOrAddSymbol("AAPL"); // id 0 + dict.getOrAddSymbol("MSFT"); // id 1 + + // Encode the range once to obtain its on-disk [len][utf8]... bytes. + PersistedSymbolDict encoded = PersistedSymbolDict.open(src.toString()); + encoded.appendSymbols(dict, 0, 1); + encoded.close(); + + PersistedSymbolDict source = PersistedSymbolDict.open(src.toString()); + try { + long addr = source.loadedEntriesAddr(); + int rawLen = source.loadedEntriesLen(); + int count = source.size(); + + ShortWriteOnceFacade ff = new ShortWriteOnceFacade(); + PersistedSymbolDict d = PersistedSymbolDict.open(ff, dst.toString()); + Assert.assertNotNull(d); + try { + ff.armed = true; // the next entry append lands short + try { + d.appendRawEntries(addr, rawLen, count); + Assert.fail("a short write must throw"); + } catch (IllegalStateException expected) { + Assert.assertTrue("short-write error: " + expected.getMessage(), + expected.getMessage().contains("short write")); + } + // The throw preceded the size/offset advance: nothing persisted. + Assert.assertEquals("short write must NOT advance size", 0, d.size()); + + // Retry the SAME bytes (the facade auto-disarmed): the write + // lands at the unchanged offset, overwriting the torn prefix. + d.appendRawEntries(addr, rawLen, count); + Assert.assertEquals(2, d.size()); + } finally { + d.close(); + } + } finally { + source.close(); + } + + // Recovery sees a gap-free, duplicate-free dictionary (size 2, not 3). + PersistedSymbolDict reopened = PersistedSymbolDict.open(dst.toString()); + try { + Assert.assertEquals("retry must not duplicate or gap the dictionary", 2, reopened.size()); + ObjList symbols = reopened.readLoadedSymbols(); + Assert.assertEquals("AAPL", symbols.getQuick(0)); + Assert.assertEquals("MSFT", symbols.getQuick(1)); + } finally { + reopened.close(); + } + } finally { + rmDir(src); + rmDir(dst); + } + }); + } + @Test public void testAppendSymbolsBatchWritesDenseRange() throws Exception { // appendSymbols persists a whole id range in one write (the hot-path @@ -204,6 +276,60 @@ public void testAppendSymbolsBatchWritesDenseRange() throws Exception { }); } + @Test + public void testAppendSymbolsShortWriteThrowsWithoutAdvancingIsIdempotentOnRetry() throws Exception { + // The producer's SLOW path (re-encode the [pd.size()..batchMax] suffix after + // a prior partial persist) writes via appendSymbols. A short write (disk full + // / quota) must throw WITHOUT advancing size()/appendOffset, so a retry keyed + // off size() re-persists the SAME range at the SAME offset and recovers a + // gap-free, duplicate-free dictionary. A regression that advanced size before + // the written==len check would strand a torn/duplicated dictionary here. + assertMemoryLeak(() -> { + Path dir = Files.createTempDirectory("qwp-symdict"); + try { + GlobalSymbolDictionary dict = new GlobalSymbolDictionary(); + dict.getOrAddSymbol("AAPL"); // id 0 + dict.getOrAddSymbol("GOOG"); // id 1 + + ShortWriteOnceFacade ff = new ShortWriteOnceFacade(); + PersistedSymbolDict d = PersistedSymbolDict.open(ff, dir.toString()); + Assert.assertNotNull(d); + try { + ff.armed = true; // the next entry append lands short + try { + d.appendSymbols(dict, 0, 1); + Assert.fail("a short write must throw"); + } catch (IllegalStateException expected) { + Assert.assertTrue("short-write error: " + expected.getMessage(), + expected.getMessage().contains("short write")); + } + // The throw preceded the size/offset advance: nothing persisted. + Assert.assertEquals("short write must NOT advance size", 0, d.size()); + + // Retry the SAME range (the facade auto-disarmed): re-writes at + // the unchanged offset, overwriting the torn bytes. + d.appendSymbols(dict, 0, 1); + Assert.assertEquals(2, d.size()); + } finally { + d.close(); + } + + // Recovery sees a gap-free, duplicate-free dictionary (size 2, not 3). + PersistedSymbolDict reopened = PersistedSymbolDict.open(dir.toString()); + try { + Assert.assertEquals("retry must not duplicate or gap the dictionary", 2, reopened.size()); + ObjList symbols = reopened.readLoadedSymbols(); + Assert.assertEquals("AAPL", symbols.getQuick(0)); + Assert.assertEquals("GOOG", symbols.getQuick(1)); + } finally { + reopened.close(); + } + } finally { + rmDir(dir); + } + }); + } + @Test public void testBadMagicIsRecreatedEmpty() throws Exception { assertMemoryLeak(() -> { @@ -776,4 +902,24 @@ public long length(String path) { return (long) Integer.MAX_VALUE + 1L; } } + + /** + * Lands ONE armed entry append short -- writes {@code len-1} of the {@code len} + * requested bytes and reports {@code len-1} -- reproducing a disk-full / quota + * short write mid-persist. Fires only on an entry append (offset past the + * 8-byte header), never the header write, and disarms after firing so the retry + * writes cleanly. + */ + private static final class ShortWriteOnceFacade extends DelegatingFilesFacade { + boolean armed; + + @Override + public long write(int fd, long addr, long len, long offset) { + if (armed && offset > 0 && len > 1) { + armed = false; + return INSTANCE.write(fd, addr, len - 1, offset); + } + return INSTANCE.write(fd, addr, len, offset); + } + } } From 265b703468890f402f8b3d3cd46816d55b5e4bc0 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Tue, 14 Jul 2026 12:45:08 +0100 Subject: [PATCH 52/96] Harden the symbol-dict fd close and split sizing Three review follow-ups on the delta symbol-dictionary change: - PersistedSymbolDict relinquishes fd (sets it to -1) before each in-branch ff.close and guards the catch-block closes with fd >= 0, so a throwing in-branch close cannot double-close via the catch. Completes the null-before-free discipline the buffers already use. - QwpWebSocketEncoderTest pins the split path's context-free-body invariant: a table's body bytes must encode identically solo and combined, since flushPendingRowsSplit sizes each frame from the combined encode's splitFrameBodyBytes rather than re-measuring. A future column encoder that carried cross-table state now fails here instead of silently stranding a deferred prefix. - CursorSendEngine imports QwpConstants instead of spelling it out fully-qualified eight times, matching its sibling send loop. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../client/sf/cursor/CursorSendEngine.java | 17 ++--- .../client/sf/cursor/PersistedSymbolDict.java | 20 ++++-- .../qwp/client/QwpWebSocketEncoderTest.java | 68 +++++++++++++++++++ 3 files changed, 92 insertions(+), 13 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java index dde24142..2c9746af 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java @@ -24,6 +24,7 @@ package io.questdb.client.cutlass.qwp.client.sf.cursor; +import io.questdb.client.cutlass.qwp.protocol.QwpConstants; import io.questdb.client.std.Compat; import io.questdb.client.std.Files; import io.questdb.client.std.ObjList; @@ -307,10 +308,10 @@ private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager man // commit would resurrect half a transaction -- see the WARN // below. Computed before the I/O loop or producer append. this.recoveredCommitBoundaryFsn = recovered.findLastFsnWithoutPayloadFlag( - io.questdb.client.cutlass.qwp.protocol.QwpConstants.HEADER_OFFSET_FLAGS, - io.questdb.client.cutlass.qwp.protocol.QwpConstants.FLAG_DEFER_COMMIT, - io.questdb.client.cutlass.qwp.protocol.QwpConstants.MAGIC_MESSAGE, - io.questdb.client.cutlass.qwp.protocol.QwpConstants.HEADER_SIZE + QwpConstants.HEADER_OFFSET_FLAGS, + QwpConstants.FLAG_DEFER_COMMIT, + QwpConstants.MAGIC_MESSAGE, + QwpConstants.HEADER_SIZE ); if (publishedFsn >= 0 && recoveredCommitBoundaryFsn < publishedFsn) { this.recoveredOrphanTipFsn = publishedFsn; @@ -332,10 +333,10 @@ private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager man // returns 0 when no such frame carries a symbol, yielding -1 here. // Computed before the I/O loop or producer append; single-threaded. this.recoveredMaxSymbolId = recovered.maxSymbolDeltaEnd( - io.questdb.client.cutlass.qwp.protocol.QwpConstants.MAGIC_MESSAGE, - io.questdb.client.cutlass.qwp.protocol.QwpConstants.HEADER_OFFSET_FLAGS, - io.questdb.client.cutlass.qwp.protocol.QwpConstants.FLAG_DELTA_SYMBOL_DICT, - io.questdb.client.cutlass.qwp.protocol.QwpConstants.HEADER_SIZE, + QwpConstants.MAGIC_MESSAGE, + QwpConstants.HEADER_OFFSET_FLAGS, + QwpConstants.FLAG_DELTA_SYMBOL_DICT, + QwpConstants.HEADER_SIZE, recoveredCommitBoundaryFsn) - 1L; } else { // Fresh start with no recovered segments. Any stale diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java index 51c1b19b..83371201 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java @@ -532,7 +532,9 @@ private static PersistedSymbolDict openExisting(FilesFacade ff, String filePath, LOG.warn("symbol dict {} unreadable, bad magic or unknown version; recreating", filePath); Unsafe.free(buf, len, MemoryTag.NATIVE_DEFAULT); buf = 0L; // null after free so the catch below cannot double-free if ff.close throws - ff.close(fd); + int fdToClose = fd; + fd = -1; // relinquish before close so the catch cannot double-close if close throws + ff.close(fdToClose); return null; } // Parse complete, CRC-valid entries after the header; stop at the first @@ -622,7 +624,9 @@ private static PersistedSymbolDict openExisting(FilesFacade ff, String filePath, entriesAddr = 0L; entriesLen = 0; } - ff.close(fd); + int fdToClose = fd; + fd = -1; // relinquish before close so the catch cannot double-close if close throws + ff.close(fdToClose); return null; } return new PersistedSymbolDict(ff, fd, validLen, count, entriesAddr, entriesLen); @@ -638,7 +642,9 @@ private static PersistedSymbolDict openExisting(FilesFacade ff, String filePath, if (entriesAddr != 0L) { Unsafe.free(entriesAddr, entriesLen, MemoryTag.NATIVE_DEFAULT); } - ff.close(fd); + if (fd >= 0) { // a branch that already closed fd relinquished it to -1 + ff.close(fd); + } LOG.warn("symbol dict {} recovery failed ({}); recreating", filePath, String.valueOf(t)); return null; } @@ -660,7 +666,9 @@ private static PersistedSymbolDict openFresh(FilesFacade ff, String filePath) { Unsafe.getUnsafe().putByte(hdr + 7, (byte) 0); long written = ff.write(fd, hdr, HEADER_SIZE, 0); if (written != HEADER_SIZE) { - ff.close(fd); + int fdToClose = fd; + fd = -1; // relinquish before close so the catch cannot double-close if close throws + ff.close(fdToClose); ff.remove(filePath); LOG.warn("symbol dict {} header write failed; proceeding without it", filePath); return null; @@ -670,7 +678,9 @@ private static PersistedSymbolDict openFresh(FilesFacade ff, String filePath) { // throwing; the Unsafe puts target a valid 8-byte buffer and an 8-byte // malloc cannot realistically OOM), but close the fd against a future // edit so it cannot leak -- mirroring openExisting's error handling. - ff.close(fd); + if (fd >= 0) { // the header-write branch relinquished fd to -1 before closing + ff.close(fd); + } LOG.warn("symbol dict {} creation failed ({}); proceeding without it", filePath, String.valueOf(t)); return null; } finally { diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketEncoderTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketEncoderTest.java index 4d7d7931..b21d2e25 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketEncoderTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketEncoderTest.java @@ -1284,6 +1284,74 @@ public void testReset() throws Exception { }); } + @Test + public void testTableBodyEncodingIsContextFree() throws Exception { + // The split-flush path (QwpWebSocketSender.flushPendingRowsSplit) sizes each + // per-table frame ARITHMETICALLY from splitFrameBodyBytes -- the body byte + // count captured during the COMBINED encode in flushPendingRows -- instead of + // re-encoding to measure. That is sound only while a table's body bytes are + // context-free: identical whether the table is encoded solo or as the k-th + // table after other tables and the delta section. Today every column encoder + // is stateless and symbol cells carry absolute global ids, so the property + // holds; this pins it so a future column encoder that ever carried + // cross-table state (which would break the arithmetic sizing and could strand + // a deferred prefix mid-split) fails loudly here rather than silently in + // production. + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + QwpTableBuffer t0 = new QwpTableBuffer("alpha"); + QwpTableBuffer t1 = new QwpTableBuffer("bravo"); + QwpTableBuffer t2 = new QwpTableBuffer("charlie")) { + GlobalSymbolDictionary dict = new GlobalSymbolDictionary(); + int aapl = dict.getOrAddSymbol("AAPL"); // 0 + int goog = dict.getOrAddSymbol("GOOG"); // 1 + int msft = dict.getOrAddSymbol("MSFT"); // 2 + + // Distinct schemas + a symbol column (absolute global ids) so a + // positional / cross-table encoder bug would shift the body bytes. + t0.getOrCreateColumn("sym", TYPE_SYMBOL, false).addSymbolWithGlobalId("AAPL", aapl); + t0.getOrCreateColumn("v", TYPE_LONG, false).addLong(1L); + t0.nextRow(); + + t1.getOrCreateColumn("sym", TYPE_SYMBOL, false).addSymbolWithGlobalId("GOOG", goog); + t1.getOrCreateColumn("d", TYPE_DOUBLE, false).addDouble(2.5); + t1.getOrCreateColumn("s", TYPE_VARCHAR, true).addString("hello"); + t1.nextRow(); + + t2.getOrCreateColumn("sym", TYPE_SYMBOL, false).addSymbolWithGlobalId("MSFT", msft); + t2.getOrCreateDesignatedTimestampColumn(TYPE_TIMESTAMP).addLong(1_000_000L); + t2.nextRow(); + + QwpTableBuffer[] tables = {t0, t1, t2}; + int confirmedMaxId = -1; + int batchMaxId = 2; + + // Combined encode: capture each table's body bytes exactly as + // flushPendingRows' splitFrameBodyBytes does (position delta per addTable). + int[] combinedBody = new int[tables.length]; + encoder.beginMessage(tables.length, dict, confirmedMaxId, batchMaxId); + int bodyStart = encoder.getBuffer().getPosition(); + for (int i = 0; i < tables.length; i++) { + encoder.addTable(tables[i]); + int bodyEnd = encoder.getBuffer().getPosition(); + combinedBody[i] = bodyEnd - bodyStart; + bodyStart = bodyEnd; + } + + // Solo encode each table under the SAME baseline/batch max, as the + // split publish loop does, and assert the body bytes match the capture. + for (int i = 0; i < tables.length; i++) { + encoder.beginMessage(1, dict, confirmedMaxId, batchMaxId); + int soloBodyStart = encoder.getBuffer().getPosition(); + encoder.addTable(tables[i]); + int soloBody = encoder.getBuffer().getPosition() - soloBodyStart; + Assert.assertEquals("table " + i + " body must encode identically solo vs combined " + + "(splitFrameBodyBytes relies on a context-free body)", combinedBody[i], soloBody); + } + } + }); + } + @Test public void testVersionByteInHeader() throws Exception { assertMemoryLeak(() -> { From 2fa5cd9bb24b219f8f83895545fce77de6441b57 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Tue, 14 Jul 2026 16:24:38 +0100 Subject: [PATCH 53/96] Recover full-dict SF slots without bricking build A store-and-forward slot written in full-dict fallback -- the .symbol-dict could not open when writing, so every frame re-ships the whole dictionary from id 0 -- leaves self-sufficient frames behind but no side-file. On recovery the engine opens a fresh empty .symbol-dict, so the surviving frames out-reach it, and the sender's seed-time guard mistook the empty dictionary for a host-crash tear: it failed Sender.build() for a slot the orphan drainer drains fine. CursorSendEngine now tells the two cases apart with a new maxSymbolDeltaStart walk (MmapSegment, SegmentRing). When the persisted dictionary is a subset of the ids the frames reference but every such frame is self-sufficient (maxSymbolDeltaStart == 0), the engine discards the empty side-file so isDeltaDictEnabled() reports false and the slot recovers in full-dict mode, exactly how it was written. A genuine torn delta dictionary keeps a deltaStart > 0 frame and still fails clean. testFullDictFramesRecoverInFullDictModeInsteadOfBricking covers the new path -- red without the discard, green with it. The torn-delta fail-clean tests are unchanged. Also regroups CursorWebSocketSendLoop's six new mutable delta-dict fields with the other mutable fields (deltaDictEnabled stays final, in its alphabetical slot) and adds the missing thousands separator to a test's 16_384 batch-size literal. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../qwp/client/QwpWebSocketSender.java | 25 +++-- .../client/sf/cursor/CursorSendEngine.java | 30 ++++++ .../sf/cursor/CursorWebSocketSendLoop.java | 93 ++++++++++--------- .../qwp/client/sf/cursor/MmapSegment.java | 56 +++++++++++ .../qwp/client/sf/cursor/SegmentRing.java | 19 ++++ .../qwp/client/DeltaDictRecoveryTest.java | 81 ++++++++++++++++ ...WebSocketSendLoopCatchUpAlignmentTest.java | 2 +- 7 files changed, 252 insertions(+), 54 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index 70a7e877..c9f7203c 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -3900,14 +3900,23 @@ private void seedGlobalDictionaryFromPersisted(PersistedSymbolDict pd) { return; } // Run the torn-dictionary guard BEFORE the empty-dictionary short-circuit - // below: a TOTAL tear (pd.size() == 0) with surviving symbol-bearing frames - // must fail clean too, not slip through. Such a frame starts at deltaStart=0 - // and self-heals the I/O-thread catch-up mirror, so the send loop's replay - // guard (deltaStart > sentDictCount) never fires -- this seed-time guard is - // then the only defense against the producer resuming unseeded and silently - // reusing ids the frames already define. A genuinely empty slot (no - // symbol-bearing frames) has recoveredMaxSymbolId() == -1, so -1 >= 0 is false - // and the pd.size() == 0 return below still fires. + // below: a TOTAL tear (pd.size() == 0) of a DELTA dictionary with surviving + // symbol-bearing frames must fail clean too, not slip through. Such a frame + // starts at deltaStart=0 and self-heals the I/O-thread catch-up mirror, so the + // send loop's replay guard (deltaStart > sentDictCount) never fires -- this + // seed-time guard is then the only defense against the producer resuming + // unseeded and silently reusing ids the frames already define. A genuinely + // empty slot (no symbol-bearing frames) has recoveredMaxSymbolId() == -1, so + // -1 >= 0 is false and the pd.size() == 0 return below still fires. + // + // This guard fires ONLY for a torn DELTA dictionary. The other way the frames + // can out-reach the recovered dictionary -- a slot written in FULL-DICT + // fallback (the dictionary never opened when writing, every frame + // self-sufficient at deltaStart=0), then recovered against a fresh empty one -- + // is caught upstream in CursorSendEngine, which discards the empty side-file so + // isDeltaDictEnabled() is false and this seed never runs. Those frames need no + // dictionary; failing clean here would needlessly brick build() for a slot the + // orphan drainer drains fine. if (cursorEngine != null && cursorEngine.recoveredMaxSymbolId() >= pd.size()) { throw new LineSenderException( "recovered store-and-forward symbol dictionary is a subset of the surviving frames " diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java index 2c9746af..88a59735 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java @@ -338,6 +338,36 @@ private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager man QwpConstants.FLAG_DELTA_SYMBOL_DICT, QwpConstants.HEADER_SIZE, recoveredCommitBoundaryFsn) - 1L; + // Full-dict-fallback recovery. When the persisted .symbol-dict is a + // SUBSET of the ids the surviving frames reference + // (recoveredMaxSymbolId >= its size) YET every such frame is + // self-sufficient (maxSymbolDeltaStart == 0 -- a full-dict frame that + // re-registers its dictionary from id 0), the slot was written in + // full-dict fallback: the dictionary never opened when writing, so no + // side-file exists and this recovery opened a FRESH EMPTY one. Those + // frames replay with no dictionary, so discard the empty side-file and + // recover in full-dict mode -- isDeltaDictEnabled() then reports false + // and the producer + send loop both run full-dict, exactly as the slot + // was written. Without this the sender's seed-time guard would treat the + // empty dictionary as a host-crash tear and brick build(), even though + // the orphan drainer drains the same frames fine. A genuine torn DELTA + // dictionary keeps a frame with deltaStart > 0 (maxSymbolDeltaStart > 0) + // and is NOT discarded here: it still fails clean at seed time, since + // the ids its delta frames reference cannot be rebuilt without the lost + // dictionary. The recoveredMaxSymbolId >= size guard means this never + // fires for a slot whose dictionary is intact, nor for an empty slot + // (recoveredMaxSymbolId == -1). Single-threaded; before the I/O loop. + if (persistedDictInProgress != null + && recoveredMaxSymbolId >= persistedDictInProgress.size() + && recovered.maxSymbolDeltaStart( + QwpConstants.MAGIC_MESSAGE, + QwpConstants.HEADER_OFFSET_FLAGS, + QwpConstants.FLAG_DELTA_SYMBOL_DICT, + QwpConstants.HEADER_SIZE, + recoveredCommitBoundaryFsn) == 0L) { + persistedDictInProgress.close(); + persistedDictInProgress = null; + } } else { // Fresh start with no recovered segments. Any stale // watermark from a prior fully-drained session refers diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java index 1ea87523..0c17614f 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java @@ -168,6 +168,11 @@ public final class CursorWebSocketSendLoop implements QuietCloseable { * Throttle "reconnect attempt N failed" WARN logs to one per 5 s. */ private static final long RECONNECT_LOG_THROTTLE_NANOS = 5_000_000_000L; + // True when this loop delta-encodes symbol dictionaries: it keeps a reconnect + // catch-up mirror (sentDict*) and re-registers the whole dictionary on a fresh + // server before replaying delta frames. See the sentDict* fields in the mutable + // section for the full mechanism. Gates all the delta-dict state. + private final boolean deltaDictEnabled; // Pre-converted to nanos for the comparison in sendDurableAckKeepaliveIfDue. // Zero or negative disables the keepalive entirely. private final long durableAckKeepaliveIntervalNanos; @@ -211,51 +216,6 @@ public final class CursorWebSocketSendLoop implements QuietCloseable { private final long reconnectMaxDurationMillis; private final WebSocketResponse response = new WebSocketResponse(); private final ResponseHandler responseHandler = new ResponseHandler(); - // Delta symbol dictionary catch-up state (see swapClient). Active in memory - // mode, and in disk mode whenever the per-slot persisted dictionary opened -- - // fresh slots included, not just recovered / orphan-drained ones. On a - // recovered / orphan-drained slot the constructor additionally SEEDS sentDict* - // from that persisted dictionary; a fresh slot starts with an empty mirror and - // grows it as frames are sent. - // deltaDictEnabled gates all of it. The loop mirrors, in sentDict*, every - // symbol it has ever sent -- the concatenated [len varint][utf8] bytes in - // global-id order (sentDictBytes*) plus the count (sentDictCount) -- so that - // on reconnect it can re-register the whole dictionary on the fresh server - // (which discards its dictionary on every disconnect) before replaying frames - // whose deltas start above id 0. All of this is touched only by the I/O thread. - // Footprint note: this mirror is a SECOND copy of the dictionary -- the same - // symbols the producer's GlobalSymbolDictionary already holds as Java Strings -- - // kept as native UTF-8 bytes for the reconnect-catch-up capability. So a - // memory-mode connection's steady-state dictionary footprint is ~2x the symbol - // set. It is bounded by distinct-symbol count (not per-row) and never trimmed - // for the connection's lifetime (a reconnect may need the whole dictionary at - // any moment), so it cannot be dropped; it is an intentional cost of the feature. - private final boolean deltaDictEnabled; - // Cap-gap attempts -- catch-ups that reached a node and found an entry too large - // for its batch cap -- since the last SUCCESSFUL catch-up (see - // MAX_CATCHUP_CAP_GAP_ATTEMPTS for the full budget accounting). A successful - // catch-up resets it (sendDictCatchUp); a transient reconnect neither increments - // nor resets it. NOT reset per connection -- it measures the cap-gap episode - // across reconnects so a persistent gap eventually latches. I/O-thread-only. - private int catchUpCapGapAttempts; - // True once a real ring frame (data or commit) has been sent on the CURRENT - // connection, as opposed to only the dictionary catch-up. The catch-up - // consumes wire sequences (nextWireSeq), so nextWireSeq > 0 no longer implies - // "the head frame was sent": onClose's poison-strike gate and - // handleServerRejection's pre-send gate key off THIS instead. Without it, a - // transient outage AFTER the catch-up but BEFORE the first data frame (a - // flapping LB/middlebox that accepts the upgrade + catch-up then closes) would - // be mistaken for a deterministic head-frame rejection and escalate to a - // PROTOCOL_VIOLATION terminal -- breaking the store-and-forward "retry a - // transient outage forever" contract. Reset per connection in - // setWireBaselineWithCatchUp; set in trySendOne after a successful send. - private boolean dataFrameSentThisConnection; - private long sentDictBytesAddr; - private int sentDictBytesCapacity; - private int sentDictBytesLen; - private int sentDictCount; - // End position (native address) written by the last readVarintAt() call. - private long varintEnd; private final CountDownLatch shutdownLatch = new CountDownLatch(1); private final AtomicLong totalAcks = new AtomicLong(); // Counters for observability of the durable-ack path. Both are zero @@ -279,6 +239,49 @@ public final class CursorWebSocketSendLoop implements QuietCloseable { // by category. Includes both retriable and terminal outcomes — i.e. every // server-side rejection observed regardless of how the loop reacted. private final AtomicLong totalServerErrors = new AtomicLong(); + // Delta symbol dictionary catch-up state (see swapClient, deltaDictEnabled). + // Active in memory mode, and in disk mode whenever the per-slot persisted + // dictionary opened -- fresh slots included, not just recovered / orphan-drained + // ones. On a recovered / orphan-drained slot the constructor additionally SEEDS + // sentDict* from that persisted dictionary; a fresh slot starts with an empty + // mirror and grows it as frames are sent. The loop mirrors, in sentDict*, every + // symbol it has ever sent -- the concatenated [len varint][utf8] bytes in + // global-id order (sentDictBytes*) plus the count (sentDictCount) -- so that on + // reconnect it can re-register the whole dictionary on the fresh server (which + // discards its dictionary on every disconnect) before replaying frames whose + // deltas start above id 0. All of this is touched only by the I/O thread. + // Footprint note: this mirror is a SECOND copy of the dictionary -- the same + // symbols the producer's GlobalSymbolDictionary already holds as Java Strings -- + // kept as native UTF-8 bytes for the reconnect-catch-up capability. So a + // memory-mode connection's steady-state dictionary footprint is ~2x the symbol + // set. It is bounded by distinct-symbol count (not per-row) and never trimmed for + // the connection's lifetime (a reconnect may need the whole dictionary at any + // moment), so it cannot be dropped; it is an intentional cost of the feature. + private long sentDictBytesAddr; + private int sentDictBytesCapacity; + private int sentDictBytesLen; + private int sentDictCount; + // Cap-gap attempts -- catch-ups that reached a node and found an entry too large + // for its batch cap -- since the last SUCCESSFUL catch-up (see + // MAX_CATCHUP_CAP_GAP_ATTEMPTS for the full budget accounting). A successful + // catch-up resets it (sendDictCatchUp); a transient reconnect neither increments + // nor resets it. NOT reset per connection -- it measures the cap-gap episode + // across reconnects so a persistent gap eventually latches. I/O-thread-only. + private int catchUpCapGapAttempts; + // True once a real ring frame (data or commit) has been sent on the CURRENT + // connection, as opposed to only the dictionary catch-up. The catch-up consumes + // wire sequences (nextWireSeq), so nextWireSeq > 0 no longer implies "the head + // frame was sent": onClose's poison-strike gate and handleServerRejection's + // pre-send gate key off THIS instead. Without it, a transient outage AFTER the + // catch-up but BEFORE the first data frame (a flapping LB/middlebox that accepts + // the upgrade + catch-up then closes) would be mistaken for a deterministic + // head-frame rejection and escalate to a PROTOCOL_VIOLATION terminal -- breaking + // the store-and-forward "retry a transient outage forever" contract. Reset per + // connection in setWireBaselineWithCatchUp; set in trySendOne after a successful + // send. + private boolean dataFrameSentThisConnection; + // End position (native address) written by the last readVarintAt() call. + private long varintEnd; private WebSocketClient client; // Optional: when non-null, every server-rejection error (retriable and // terminal alike) is offered to the dispatcher for async delivery to the user's diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java index 50fc18ff..602c9ed2 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java @@ -606,6 +606,62 @@ public long maxSymbolDeltaEnd(int headerMagic, int flagsOffset, int flagDeltaMas return maxEnd; } + /** + * Highest {@code deltaStart} any delta-flagged frame at or below + * {@code maxFsnInclusive} carries, or {@code 0} when NO delta-flagged frame is + * present. A result of {@code 0} means every surviving symbol-bearing frame is + * SELF-SUFFICIENT -- it re-registers its dictionary from id 0 (a full-dict + * frame), so the slot replays with no persisted dictionary at all; a result + * {@code > 0} means at least one frame is a non-self-sufficient delta whose ids + * depend on a prior frame's (or the persisted dictionary's) registrations. + * {@link CursorSendEngine} pairs this with {@link #maxSymbolDeltaEnd} at + * recovery: when the persisted {@code .symbol-dict} is a subset of the ids the + * frames reference BUT this returns {@code 0}, the slot was written in full-dict + * fallback (the dictionary never opened when writing) and recovers in full-dict + * mode instead of failing clean; a torn DELTA dictionary has a {@code + * deltaStart > 0} frame and still fails clean. Same read-only frame walk and + * layout as {@link #maxSymbolDeltaEnd}; only frames at or below {@code + * maxFsnInclusive} count (the retired orphan-deferred tail is excluded). + */ + public long maxSymbolDeltaStart(int headerMagic, int flagsOffset, int flagDeltaMask, int qwpHeaderSize, long maxFsnInclusive) { + long maxStart = 0L; + long off = HEADER_SIZE; + long frames = frameCount; + for (long i = 0; i < frames; i++) { + long fsn = baseSeq + i; + int payloadLen = Unsafe.getUnsafe().getInt(mmapAddress + off + 4); + long payload = mmapAddress + off + FRAME_HEADER_SIZE; + // Skip the orphan-deferred tail (fsn > maxFsnInclusive): retired, never + // sent, so its baseline must not count. off still advances below. + if (fsn <= maxFsnInclusive + && payloadLen >= qwpHeaderSize + && payloadLen > flagsOffset + && Unsafe.getUnsafe().getInt(payload) == headerMagic + && (Unsafe.getUnsafe().getByte(payload + flagsOffset) & flagDeltaMask) != 0) { + long p = payload + qwpHeaderSize; + long limit = payload + payloadLen; + long deltaStart = 0L; + int shift = 0; + while (p < limit) { + byte b = Unsafe.getUnsafe().getByte(p++); + deltaStart |= (long) (b & 0x7F) << shift; + if ((b & 0x80) == 0) { + break; + } + shift += 7; + if (shift > 35) { + break; // corrupt run; recovered frames are CRC-valid, so defensive only + } + } + if (deltaStart > maxStart) { + maxStart = deltaStart; + } + } + off += FRAME_HEADER_SIZE + payloadLen; + } + return maxStart; + } + /** * Number of frames written since {@link #create} (or recovered by * {@link #openExisting}). Used by {@code SegmentRing} to compute diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java index 21cfa9d1..f9cd08c6 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java @@ -566,6 +566,25 @@ public synchronized long maxSymbolDeltaEnd(int headerMagic, int flagsOffset, int return Math.max(maxEnd, end); } + /** + * Highest {@code deltaStart} any symbol-dict delta frame at or below + * {@code maxFsnInclusive} carries (0 when none, or when every such frame is + * self-sufficient). See {@link MmapSegment#maxSymbolDeltaStart}; paired with + * {@link #maxSymbolDeltaEnd} at recovery to tell a full-dict-fallback slot + * (recoverable with no dictionary) from a torn delta dictionary (fails clean). + */ + public synchronized long maxSymbolDeltaStart(int headerMagic, int flagsOffset, int flagDeltaMask, int qwpHeaderSize, long maxFsnInclusive) { + long maxStart = 0L; + for (int i = 0, n = sealedSegments.size(); i < n; i++) { + long start = sealedSegments.get(i).maxSymbolDeltaStart(headerMagic, flagsOffset, flagDeltaMask, qwpHeaderSize, maxFsnInclusive); + if (start > maxStart) { + maxStart = start; + } + } + long start = active.maxSymbolDeltaStart(headerMagic, flagsOffset, flagDeltaMask, qwpHeaderSize, maxFsnInclusive); + return Math.max(maxStart, start); + } + public MmapSegment getActive() { return active; } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java index fb355b82..e576902b 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java @@ -897,6 +897,87 @@ public void testTornDictTotalLossFailsCleanOnResume() throws Exception { }); } + @Test + public void testFullDictFramesRecoverInFullDictModeInsteadOfBricking() throws Exception { + // M1 regression (counterpoint to testTornDictTotalLossFailsCleanOnResume): a + // slot written in FULL-DICT fallback -- the .symbol-dict could not open when + // writing, so isDeltaDictEnabled() was false and every frame re-ships the + // whole dictionary from id 0 (deltaStart=0) -- leaves SELF-SUFFICIENT frames + // and NO side-file. On recovery the engine opens a FRESH EMPTY .symbol-dict, + // so the surviving frames out-reach it (recoveredMaxSymbolId >= pd.size()==0). + // Those frames carry their whole dictionary inline and need no side-file, so + // they must RECOVER, not brick build(). CursorSendEngine detects them + // (maxSymbolDeltaStart == 0) and discards the empty side-file, recovering the + // slot in full-dict mode. Before the fix the sender's seed-time guard treated + // the empty dictionary as a host-crash tear and threw from Sender.build(), + // even though the orphan drainer drains the very same frames fine. A torn + // DELTA dictionary (deltaStart > 0) still fails clean -- see that other test. + assertMemoryLeak(() -> { + java.nio.file.Path slot = Paths.get(sfDir, "default"); + java.nio.file.Path dict = slot.resolve(".symbol-dict"); + // Force full-dict fallback in phase 1: plant a non-empty DIRECTORY where + // the dictionary file belongs, so openCleanRW fails and delta encoding + // stays disabled -- the frames are then written self-sufficient. + java.nio.file.Files.createDirectories(dict); + java.nio.file.Path blocker = dict.resolve("blocker"); + java.nio.file.Files.createFile(blocker); + + // Phase 1: silent server (no acks). Sender 1 writes new-symbol rows in + // full-dict mode and close-fast, leaving unacked self-sufficient frames. + try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) { + int port = silent.getPort(); + silent.start(); + Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS)); + String cfg = "ws::addr=localhost:" + port + + ";sf_dir=" + sfDir + + ";close_flush_timeout_millis=0;"; + try (Sender s1 = Sender.fromConfig(cfg)) { + for (int i = 0; i < DISTINCT_SYMBOLS; i++) { + s1.table("m").symbol("s", "sym-" + i).longColumn("v", i).atNow(); + s1.flush(); + } + } + // The planted directory is untouched -- the dictionary never opened, + // so the frames were written full-dict with no side-file. + Assert.assertTrue("full-dict fallback: .symbol-dict must stay a directory", + java.nio.file.Files.isDirectory(dict)); + } + + // Drop the planted directory so recovery opens a FRESH EMPTY .symbol-dict + // where it belongs -- exactly the state a full-dict-fallback slot recovers + // into (frames on disk, no dictionary behind them). + java.nio.file.Files.delete(blocker); + java.nio.file.Files.delete(dict); + + // Phase 2: recover. build() must SUCCEED (not throw the torn-dict guard), + // and the self-sufficient frames replay against a fresh server gap-free. + DictReconstructingHandler handler = new DictReconstructingHandler(); + try (TestWebSocketServer good = new TestWebSocketServer(handler)) { + int port = good.getPort(); + good.start(); + Assert.assertTrue(good.awaitStart(5, TimeUnit.SECONDS)); + String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";"; + try (Sender ignored = Sender.fromConfig(cfg)) { // must NOT throw + long deadline = System.currentTimeMillis() + 5_000; + while (System.currentTimeMillis() < deadline + && handler.maxDictSize() < DISTINCT_SYMBOLS) { + Thread.sleep(20); + } + } + // Full-dict recovery re-ships the whole dictionary inline on every + // frame, so there is NO 0-table catch-up frame (that is the delta-mode + // reconnect path). The reconstructed dictionary is still gap-free. + Assert.assertFalse("full-dict recovery must NOT send a catch-up frame", + handler.sawCatchUpFrame); + List reconstructed = handler.dictSnapshot(); + Assert.assertEquals("reconstructed dictionary size", DISTINCT_SYMBOLS, reconstructed.size()); + for (int i = 0; i < DISTINCT_SYMBOLS; i++) { + Assert.assertEquals("dictionary id " + i, "sym-" + i, reconstructed.get(i)); + } + } + }); + } + private static int globalDictSize(Sender sender) throws Exception { Field f = sender.getClass().getDeclaredField("globalSymbolDictionary"); f.setAccessible(true); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java index c21b7d82..ba81775f 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java @@ -528,7 +528,7 @@ private CursorWebSocketSendLoop newLoop(CursorSendEngine engine, WebSocketClient } private CursorSendEngine newEngine() { - return new CursorSendEngine(tmpDir, 16384); + return new CursorSendEngine(tmpDir, 16_384); } private static long readLong(CursorWebSocketSendLoop loop, String name) throws Exception { From c181e4fa31f915387713ff24076641bd491a8f12 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Tue, 14 Jul 2026 19:49:00 +0100 Subject: [PATCH 54/96] Recover slots whose symbol dictionary is damaged The QWP store-and-forward recovery path treated a damaged .symbol-dict as unrecoverable in two places, stranding slots that are in fact replayable. Both stem from keying on the dictionary FILE rather than on what the surviving frames themselves define. CursorWebSocketSendLoop ran the torn-dictionary guard unconditionally but gated accumulateSentDict, the reconnect catch-up and the mirror seed on engine.isDeltaDictEnabled(). That flag goes false exactly when a disk slot's dictionary fails to open -- fd exhaustion, a read-only remount, ENOSPC -- which is precisely when the frames already on disk are still deltas. With the mirror gated off, sentDictCount froze at 0 while the ungated guard kept comparing against it, so the frame at deltaStart=1 latched a terminal and the background drainer quarantined the slot behind a .failed sentinel. Yet replaying that sequence from id 0 rebuilds the dictionary on the server contiguously and drains perfectly. The loop now always mirrors, always catches up and always guards: the mirror is its model of what the SERVER holds, so it must track every mode. isDeltaDictEnabled() now governs only what the producer emits and persists. seedGlobalDictionaryFromPersisted threw whenever the surviving frames out-reached the persisted dictionary. That bricked Sender.build() for slots the drainer replays fine, and permanently: build()'s catch releases the slot lock, but the sender that would have hosted the orphan drainer is the one that just failed, so the retry recovers the same slot and throws again. Relaxing the guard would have been worse than the bug -- with an empty dictionary the producer hands its next symbol id 0, an id the frames already define, and accumulateSentDict skips it, so the next catch-up replays the stale symbol and misattributes it silently. The producer now seeds from the dictionary's intact prefix AND THEN from the surviving frames' own delta sections, through collectReplaySymbolsAbove (MmapSegment -> SegmentRing -> CursorSendEngine). Those are the same two sources, in the same order, that the send loop's mirror is built from, so the producer's delta baseline and the mirror's coverage land on the same number by construction. A genuine gap -- ids introduced by frames since acked and trimmed away -- still fails clean, and now on exactly the condition the send loop's replay guard trips on, so producer and drainer agree on which slots are recoverable. The two cap-gap settle-budget tests derived their retriable loop bound from MAX_CATCHUP_CAP_GAP_ATTEMPTS itself, so a regression of that constant to 1 -- the very bug they name -- ran the loop zero times and let the test pass green. They now pin the budget against a literal. Tests: an untrimmed unopenable-dictionary slot must replay gap-free; a torn (subset) and a wholly lost dictionary must rebuild from the surviving frames and place a new symbol above the recovered tip. The two watermark-stamped siblings still fail clean. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../qwp/client/QwpWebSocketSender.java | 151 +++++----- .../client/sf/cursor/CursorSendEngine.java | 38 +++ .../sf/cursor/CursorWebSocketSendLoop.java | 159 ++++++----- .../qwp/client/sf/cursor/MmapSegment.java | 117 ++++++++ .../qwp/client/sf/cursor/SegmentRing.java | 35 +++ .../qwp/client/DeltaDictRecoveryTest.java | 263 +++++++++++++----- ...WebSocketSendLoopCatchUpAlignmentTest.java | 20 ++ 7 files changed, 573 insertions(+), 210 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index c9f7203c..bc08948e 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -3847,91 +3847,88 @@ private void resetSymbolDictStateForNewConnection() { } /** - * On recovery, repopulates the producer's {@link GlobalSymbolDictionary} from - * the slot's persisted dictionary (ids assigned in the same ascending order, - * so they match the recovered frames) and resumes the delta baseline at the - * recovered tip, so newly ingested symbols continue above the recovered ids. + * On recovery, repopulates the producer's {@link GlobalSymbolDictionary} so that newly + * ingested symbols continue ABOVE every id the surviving frames already define, and + * resumes the delta baseline at that tip. *

- * Uses {@link GlobalSymbolDictionary#addRecoveredSymbol} (append, NOT de-dup): - * the persisted dictionary, the on-wire delta and the send-loop catch-up mirror - * all key on the entry POSITION (id), so the producer id space must match the - * persisted entry count exactly. {@code getOrAddSymbol} would collapse two - * source strings that decode to the same characters -- only malformed lone - * UTF-16 surrogates, which UTF-8-encode to {@code '?'} -- leaving this - * dictionary shorter than {@code pd.size()} and desyncing - * {@code sentMaxSymbolId} from the mirror's {@code sentDictCount = pd.size()}, - * which silently misattributes later symbols after a reconnect. + * Seeds from TWO sources, in this order: + *

    + *
  1. the slot's persisted {@code .symbol-dict} -- its intact prefix; then
    2. + *
  2. the surviving frames' OWN delta sections, for every id above that prefix + * ({@link CursorSendEngine#collectReplaySymbolsAbove}).
    3. + *
+ * Those are exactly the two sources, in exactly the order, that the send loop's mirror + * is built from: its constructor seeds {@code sentDictCount} from the same dictionary, + * and {@code accumulateSentDict} then extends it from the same frames as they replay. So + * the producer's {@code sentMaxSymbolId + 1} and the loop's {@code sentDictCount} land on + * the same number BY CONSTRUCTION -- the invariant the torn-dictionary guard rests on -- + * rather than by the two happening to agree. *

- * Host-crash tear guard. The persisted dictionary is NOT fsync'd (see - * {@code PersistedSymbolDict}), so a host/power crash can lose its - * most-recently-written (highest-id) entries while the segment frames that - * introduced those ids survive -- and those newest frames, being the least - * likely to be acked, replay on recovery. The send loop's catch-up mirror then - * rebuilds the missing ids from those frames' own delta bytes, but THIS producer - * -- seeded only from the shorter dictionary -- would assign its next new symbol - * an id the surviving frames already define, putting two symbols on one id and - * silently misattributing values. The send loop's replay guard only catches a - * GAP ({@code deltaStart > sentDictCount}); a frame that introduces exactly the - * torn-off id ({@code deltaStart == pd.size()}) slips through and self-heals the - * mirror, leaving only this producer diverged. Detect it here -- the surviving - * frames reference an id at or beyond the recovered dictionary size -- and fail - * clean: the affected data must be resent, matching the design's torn-dict - * "resend required" contract. + * Uses {@link GlobalSymbolDictionary#addRecoveredSymbol} (append, NOT de-dup): the + * persisted dictionary, the on-wire delta and the mirror all key on the entry POSITION + * (id), so the producer's id space must match the recovered entry count exactly. + * {@code getOrAddSymbol} would collapse two source strings that decode to the same + * characters -- only malformed lone UTF-16 surrogates, which UTF-8-encode to {@code '?'} + * -- leaving this dictionary SHORTER than the count and silently misattributing later + * symbols. *

- * The background drainer self-heals the mirror ONLY when a surviving frame - * STRADDLES the tear ({@code deltaStart <= pd.size() < deltaStart + deltaCount}): - * such a frame carries the torn-off ids in its own delta and - * {@code accumulateSentDict} re-registers them, so the drainer drains the slot. - * But when the symbol-introducing frames were already acked and trimmed and only - * a HIGHER-baseline frame survives ({@code deltaStart > pd.size()} -- e.g. a - * commit or a symbol-reusing frame, since {@code beginMessage} always sets the - * delta flag), the drainer's own replay guard ({@code deltaStart > sentDictCount}) - * fires too and quarantines the slot: the recorded bytes are not silently lost, - * but the slot is NOT auto-drained -- it must be resent. That is a deliberate - * CONSERVATIVE over-strand -- the guard keys on {@code deltaStart}, not on the - * frame's actual highest referenced id, to avoid parsing row data at recovery, - * so it may reject a frame whose rows reference only ids the truncated - * dictionary still holds. It fails clean rather than risk a silent id shift. - * Only a host crash reaches this -- a process crash keeps the page cache, so the - * write-ahead ordering keeps the dictionary a superset of the frames. + * Why seeding from the frames matters. The dictionary is not fsync'd (see + * {@code PersistedSymbolDict}), so a host/power crash can tear off its newest entries + * while the segment frames that introduced those ids survive -- and those newest frames, + * being the least likely to be acked, are exactly the ones that replay. Seeded from the + * short dictionary alone, this producer would hand its next new symbol an id those frames + * already define, putting two symbols on one id and silently misattributing values. The + * old code detected that and threw, which was safe but far too blunt: it bricked + * {@code build()} for slots the background drainer replays PERFECTLY, because the frames + * carry the torn-off symbols in their own deltas and {@code accumulateSentDict} rebuilds + * the dictionary from them. This method now rebuilds the producer from the same bytes, + * so a torn -- or entirely lost -- dictionary is recoverable whenever the surviving + * frames define the ids themselves. The next flush's write-ahead persist then re-writes + * those ids (it resumes from {@code pd.size()}), healing the side-file on disk. + *

+ * What still fails clean. A genuine GAP: the ids below a surviving frame's delta + * start were introduced by frames that were acked and TRIMMED away, so they lived only in + * the lost dictionary and nothing can rebuild them. + * {@code collectReplaySymbolsAbove} returns -1 for that and we throw. It is the same + * condition the send loop's replay guard ({@code deltaStart > sentDictCount}) trips on, so + * producer and drainer now agree on exactly which slots are recoverable, instead of the + * producer rejecting slots the drainer drains. */ private void seedGlobalDictionaryFromPersisted(PersistedSymbolDict pd) { - if (pd == null) { + if (cursorEngine == null) { return; } - // Run the torn-dictionary guard BEFORE the empty-dictionary short-circuit - // below: a TOTAL tear (pd.size() == 0) of a DELTA dictionary with surviving - // symbol-bearing frames must fail clean too, not slip through. Such a frame - // starts at deltaStart=0 and self-heals the I/O-thread catch-up mirror, so the - // send loop's replay guard (deltaStart > sentDictCount) never fires -- this - // seed-time guard is then the only defense against the producer resuming - // unseeded and silently reusing ids the frames already define. A genuinely - // empty slot (no symbol-bearing frames) has recoveredMaxSymbolId() == -1, so - // -1 >= 0 is false and the pd.size() == 0 return below still fires. - // - // This guard fires ONLY for a torn DELTA dictionary. The other way the frames - // can out-reach the recovered dictionary -- a slot written in FULL-DICT - // fallback (the dictionary never opened when writing, every frame - // self-sufficient at deltaStart=0), then recovered against a fresh empty one -- - // is caught upstream in CursorSendEngine, which discards the empty side-file so - // isDeltaDictEnabled() is false and this seed never runs. Those frames need no - // dictionary; failing clean here would needlessly brick build() for a slot the - // orphan drainer drains fine. - if (cursorEngine != null && cursorEngine.recoveredMaxSymbolId() >= pd.size()) { + // 1. The dictionary's intact prefix. addRecoveredSymbol appends without de-dup, so + // the producer's size tracks pd.size() exactly -- which is what the send loop's + // mirror also seeds sentDictCount from. + int baseline = 0; + if (pd != null && pd.size() > 0) { + ObjList persisted = pd.readLoadedSymbols(); + for (int i = 0, n = persisted.size(); i < n; i++) { + globalSymbolDictionary.addRecoveredSymbol(persisted.getQuick(i)); + } + baseline = globalSymbolDictionary.size(); + } + // 2. Everything the surviving frames define above that prefix, straight out of their + // own delta sections -- the same bytes, in the same order, accumulateSentDict will + // feed the mirror as those frames go back on the wire. + ObjList fromFrames = new ObjList<>(); + long coverage = cursorEngine.collectReplaySymbolsAbove(baseline, fromFrames); + if (coverage < 0) { throw new LineSenderException( - "recovered store-and-forward symbol dictionary is a subset of the surviving frames " - + "(likely a host crash tore its unsynced tail): frames reference symbol id " - + cursorEngine.recoveredMaxSymbolId() + " but the recovered dictionary holds only " - + pd.size() + " id(s); resuming would reuse ids the frames already define -- " - + "resend the affected data"); - } - if (pd.size() == 0) { - return; - } - ObjList symbols = pd.readLoadedSymbols(); - for (int i = 0, n = symbols.size(); i < n; i++) { - globalSymbolDictionary.addRecoveredSymbol(symbols.getQuick(i)); - } + "recovered store-and-forward symbol dictionary is incomplete and cannot be " + + "rebuilt from the surviving frames (likely a host crash tore its unsynced " + + "tail): the frames reference symbol ids below their own delta start, which " + + "were introduced by frames since acked and trimmed away, so nothing still " + + "holds them; the recovered dictionary holds only " + + (pd == null ? 0 : pd.size()) + " id(s) -- resend the affected data"); + } + for (int i = 0, n = fromFrames.size(); i < n; i++) { + globalSymbolDictionary.addRecoveredSymbol(fromFrames.getQuick(i)); + } + // Producer baseline == the coverage the replay will establish == the mirror's + // sentDictCount once those frames have gone out. The first new frame therefore + // starts its delta exactly at the tip, and the replay guard passes. sentMaxSymbolId = globalSymbolDictionary.size() - 1; } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java index 88a59735..91035a0e 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java @@ -661,6 +661,44 @@ public synchronized void close() { } } + /** + * Rebuilds, from the surviving frames' OWN delta sections, the symbols the upcoming + * replay will register ABOVE {@code baseline}, appending them to {@code out} in + * ascending id order. Returns the coverage the replay establishes (one past the highest + * id it registers), or {@code -1} when the frames have a genuine GAP above the baseline. + *

+ * The producer seeds its dictionary from the persisted {@code .symbol-dict} and THEN + * from this, so it is fed the same entries, in the same order, that the send loop's + * {@code accumulateSentDict} will feed its mirror from as those very frames replay. The + * producer's delta baseline and the loop's mirror coverage therefore land on the same + * number by construction, which is the invariant the torn-dictionary guard rests on. + *

+ * This is what makes a slot whose dictionary was torn -- or lost outright -- still + * recoverable when the surviving frames define the ids themselves (they carry the + * symbols in their own deltas; it is why the orphan drainer can drain such a slot). Only + * a real gap, where the ids were introduced by frames since acked and trimmed away, is + * unrecoverable -- and that is precisely when this returns -1. + *

+ * Bounded above by {@link #recoveredCommitBoundaryFsn} like {@link #recoveredMaxSymbolId}: + * frames past it are the aborted orphan-deferred tail, retired without ever being + * transmitted, so their ids never reach a server and must not inflate the baseline. + */ + public long collectReplaySymbolsAbove(int baseline, ObjList out) { + if (ring == null) { + return baseline; + } + return ring.collectReplaySymbolsAbove( + QwpConstants.MAGIC_MESSAGE, + QwpConstants.HEADER_OFFSET_FLAGS, + QwpConstants.FLAG_DELTA_SYMBOL_DICT, + QwpConstants.HEADER_SIZE, + ackedFsn() + 1L, + recoveredCommitBoundaryFsn, + baseline, + out + ); + } + /** * Pass-through to {@link SegmentRing#findSegmentContaining(long)}. */ diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java index 0c17614f..93b0b137 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java @@ -168,11 +168,6 @@ public final class CursorWebSocketSendLoop implements QuietCloseable { * Throttle "reconnect attempt N failed" WARN logs to one per 5 s. */ private static final long RECONNECT_LOG_THROTTLE_NANOS = 5_000_000_000L; - // True when this loop delta-encodes symbol dictionaries: it keeps a reconnect - // catch-up mirror (sentDict*) and re-registers the whole dictionary on a fresh - // server before replaying delta frames. See the sentDict* fields in the mutable - // section for the full mechanism. Gates all the delta-dict state. - private final boolean deltaDictEnabled; // Pre-converted to nanos for the comparison in sendDurableAckKeepaliveIfDue. // Zero or negative disables the keepalive entirely. private final long durableAckKeepaliveIntervalNanos; @@ -239,17 +234,21 @@ public final class CursorWebSocketSendLoop implements QuietCloseable { // by category. Includes both retriable and terminal outcomes — i.e. every // server-side rejection observed regardless of how the loop reacted. private final AtomicLong totalServerErrors = new AtomicLong(); - // Delta symbol dictionary catch-up state (see swapClient, deltaDictEnabled). - // Active in memory mode, and in disk mode whenever the per-slot persisted - // dictionary opened -- fresh slots included, not just recovered / orphan-drained - // ones. On a recovered / orphan-drained slot the constructor additionally SEEDS - // sentDict* from that persisted dictionary; a fresh slot starts with an empty - // mirror and grows it as frames are sent. The loop mirrors, in sentDict*, every - // symbol it has ever sent -- the concatenated [len varint][utf8] bytes in - // global-id order (sentDictBytes*) plus the count (sentDictCount) -- so that on - // reconnect it can re-register the whole dictionary on the fresh server (which - // discards its dictionary on every disconnect) before replaying frames whose - // deltas start above id 0. All of this is touched only by the I/O thread. + // Delta symbol dictionary catch-up state (see swapClient). + // ALWAYS active -- in memory mode, in disk mode, and (critically) even when the + // per-slot persisted dictionary failed to open. sentDictCount is this loop's model + // of how many ids the CURRENT server has been told about, and the torn-dict guard + // in trySendOne reads it, so it must track the wire in every mode; gating it on + // engine.isDeltaDictEnabled() is what once froze it at 0 and terminal'd a slot + // whose frames replay perfectly from id 0 (see the constructor). On a recovered / + // orphan-drained slot the constructor SEEDS sentDict* from the persisted + // dictionary when there is one; otherwise the mirror starts empty and grows from + // the frames themselves. The loop mirrors, in sentDict*, every symbol it has ever + // sent -- the concatenated [len varint][utf8] bytes in global-id order + // (sentDictBytes*) plus the count (sentDictCount) -- so that on reconnect it can + // re-register the whole dictionary on the fresh server (which discards its + // dictionary on every disconnect) before replaying frames whose deltas start above + // id 0. All of this is touched only by the I/O thread. // Footprint note: this mirror is a SECOND copy of the dictionary -- the same // symbols the producer's GlobalSymbolDictionary already holds as Java Strings -- // kept as native UTF-8 bytes for the reconnect-catch-up capability. So a @@ -571,39 +570,50 @@ public CursorWebSocketSendLoop(WebSocketClient client, CursorSendEngine engine, } this.client = client; this.engine = engine; - this.deltaDictEnabled = engine.isDeltaDictEnabled(); // Recovery / orphan-drain: the loop starts with a fresh in-memory mirror, // so seed it from the slot's persisted dictionary. That way the very first // connection re-registers the whole dictionary (via a catch-up frame) // before replaying the recovered delta frames. - if (deltaDictEnabled) { - PersistedSymbolDict pd = engine.getPersistedSymbolDict(); - if (pd != null && pd.size() > 0) { - int len = pd.loadedEntriesLen(); - if (len > 0) { - // COPY the persisted dictionary's loaded-entries buffer into this - // loop's own mirror rather than taking ownership of it. The engine - // (and its PersistedSymbolDict) OUTLIVES this loop on the orphan - // drainer path: BackgroundDrainer builds a fresh send loop per wire - // session against the same engine on a durable-ack capability-gap - // recycle. A one-shot ownership transfer would leave every loop - // after the first with an EMPTY mirror -- it would then send no - // reconnect catch-up, and the first replayed delta frame - // (deltaStart > 0) would trip the torn-dict guard, falsely - // quarantining a healthy slot. Copying keeps the dictionary's - // loaded entries intact for the engine's lifetime so every - // recycled loop re-seeds; pd.close() (at engine close) frees the - // dictionary's copy, this loop frees its own copy on exit. - sentDictBytesAddr = Unsafe.malloc(len, MemoryTag.NATIVE_DEFAULT); - Unsafe.getUnsafe().copyMemory(pd.loadedEntriesAddr(), sentDictBytesAddr, len); - sentDictBytesCapacity = len; - sentDictBytesLen = len; - // Set the count only alongside the bytes so sentDictCount can - // never claim symbols the mirror does not hold. A recovered slot - // always has loadedEntriesLen > 0 when size > 0, so this is the - // same result -- it just makes the coupling explicit. - sentDictCount = pd.size(); - } + // + // Deliberately NOT gated on engine.isDeltaDictEnabled(). The mirror, the + // catch-up and the replay guard together are this loop's model of what the + // SERVER's dictionary holds, and that model must track every mode. Gating + // them on that flag is what once bricked a recoverable slot: an unopenable + // .symbol-dict reports isDeltaDictEnabled()=false, yet the frames already on + // disk are still DELTA frames (deltaStart 0,1,2,...). With the mirror gated + // off, sentDictCount froze at 0 while the ungated guard kept comparing + // against it, so the frame at deltaStart=1 tripped a terminal -- even though + // replaying the whole sequence from id 0 rebuilds the dictionary on the + // server contiguously and drains perfectly. isDeltaDictEnabled() decides + // what the PRODUCER emits and persists; it must never decide what this loop + // tracks. When the dictionary could not be opened, pd is null, the mirror + // simply starts empty and grows from the frames themselves. + PersistedSymbolDict pd = engine.getPersistedSymbolDict(); + if (pd != null && pd.size() > 0) { + int len = pd.loadedEntriesLen(); + if (len > 0) { + // COPY the persisted dictionary's loaded-entries buffer into this + // loop's own mirror rather than taking ownership of it. The engine + // (and its PersistedSymbolDict) OUTLIVES this loop on the orphan + // drainer path: BackgroundDrainer builds a fresh send loop per wire + // session against the same engine on a durable-ack capability-gap + // recycle. A one-shot ownership transfer would leave every loop + // after the first with an EMPTY mirror -- it would then send no + // reconnect catch-up, and the first replayed delta frame + // (deltaStart > 0) would trip the torn-dict guard, falsely + // quarantining a healthy slot. Copying keeps the dictionary's + // loaded entries intact for the engine's lifetime so every + // recycled loop re-seeds; pd.close() (at engine close) frees the + // dictionary's copy, this loop frees its own copy on exit. + sentDictBytesAddr = Unsafe.malloc(len, MemoryTag.NATIVE_DEFAULT); + Unsafe.getUnsafe().copyMemory(pd.loadedEntriesAddr(), sentDictBytesAddr, len); + sentDictBytesCapacity = len; + sentDictBytesLen = len; + // Set the count only alongside the bytes so sentDictCount can + // never claim symbols the mirror does not hold. A recovered slot + // always has loadedEntriesLen > 0 when size > 0, so this is the + // same result -- it just makes the coupling explicit. + sentDictCount = pd.size(); } } this.fsnAtZero = fsnAtZero; @@ -2055,7 +2065,15 @@ private void setWireBaselineWithCatchUp(long replayStart) { // handleServerRejection can tell "only the catch-up went out" from "the // head data frame went out". dataFrameSentThisConnection = false; - if (client != null && deltaDictEnabled && sentDictCount > 0) { + // Not gated on isDeltaDictEnabled() -- see the constructor. The mirror is + // non-empty exactly when this loop has already put symbols on a wire, and a + // FRESH server holds none of them, so it must be re-registered before any + // frame that back-references an id. In full-dict mode the mirror is non-empty + // too and the catch-up is then redundant (the next frame re-registers from id + // 0 anyway) but harmless: the server overwrites identical ids with identical + // symbols. Redundant-but-uniform beats a mode flag that can go false while the + // frames on disk are still deltas. + if (client != null && sentDictCount > 0) { this.nextWireSeq = 0L; // The catch-up may span several frames when the dictionary exceeds the // server's batch cap; each consumes a wire sequence (0 .. n-1) that maps @@ -2493,24 +2511,23 @@ private boolean trySendOne() { return false; // payload not fully published yet } long frameAddr = base + sendOffset + MmapSegment.FRAME_HEADER_SIZE; - // Torn-dictionary guard. Decode the delta start unconditionally (-1 for a - // non-delta frame); the guard MUST run even when deltaDictEnabled is false. - // A disk slot recovered with its persisted dictionary unavailable - // (PersistedSymbolDict.open() returned null -- fd exhaustion, a read-only - // remount, ENOSPC) reports deltaDictEnabled=false, yet its recorded frames - // are still DELTA frames (deltaStart > 0). Replaying those against a fresh - // empty-dictionary server would null-pad the missing ids and SILENTLY - // corrupt the table -- precisely what this guard exists to prevent -- so it - // cannot be gated on the very flag that goes false in that failure mode. In - // normal operation a delta frame's start id never exceeds the dictionary - // coverage established so far (replayed frames overlap the catch-up dict; - // fresh frames extend it contiguously), so a gap here means the recovered - // dictionary is incomplete (a host/power crash that lost recently-written - // entries, SF being process-crash but not host-crash durable). Fail - // terminally; the unreplayable data must be resent. Full-dict / fallback - // frames carry deltaStart=0 with sentDictCount=0, so 0 > 0 never - // false-positives; only the sent-dictionary mirror below stays gated on - // deltaDictEnabled. + // Torn-dictionary guard. sentDictCount is this loop's model of how many ids + // the CURRENT server has been told about: seeded from the persisted + // dictionary, re-registered by the catch-up, and extended by + // accumulateSentDict as each frame goes out. A frame whose delta starts ABOVE + // that coverage references ids the server was never given; replaying it would + // make the server null-pad the hole (QwpMessageCursor grows the dict with + // nulls to deltaStart+deltaCount) and land rows with SILENTLY NULL symbols. + // Fail terminally instead; the unreplayable data must be resent. + // + // The guard, the mirror and the catch-up are ONE mechanism and are all + // ungated (see the constructor). A frame at deltaStart == sentDictCount is + // contiguous and extends the coverage, so a slot whose frames start at id 0 + // replays cleanly with no persisted dictionary at all -- which is exactly why + // the mirror below must keep advancing even when the dictionary is missing. + // A gap here therefore means genuine loss: a host/power crash tore the + // unsynced .symbol-dict below the ids the surviving frames still reference + // (SF is process-crash but not host-crash durable). int deltaStart = frameDeltaStart(frameAddr, payloadLen); if (deltaStart > sentDictCount) { recordFatal(new LineSenderException( @@ -2530,10 +2547,16 @@ private boolean trySendOne() { // key their poison-strike vs pre-send decision off this, not off nextWireSeq // (which the catch-up advances). dataFrameSentThisConnection = true; - if (deltaDictEnabled && deltaStart >= 0) { - // Mirror the symbols this frame introduced so a later reconnect can - // rebuild the whole dictionary. Idempotent on replay: a frame whose - // delta we already hold advances nothing. + if (deltaStart >= 0) { + // Mirror the symbols this frame just registered on the server, so a later + // reconnect can rebuild the whole dictionary and the guard above keeps an + // accurate view of the server's coverage. Idempotent on replay: a frame + // whose delta we already hold advances nothing. + // + // Ungated (see the constructor): this is the ONLY thing that advances + // sentDictCount from the frames themselves, so gating it while leaving the + // guard ungated froze the coverage at 0 and terminal'd a slot that replays + // perfectly from id 0. accumulateSentDict(frameAddr, payloadLen, deltaStart); } lastFrameOrPingNanos = System.nanoTime(); diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java index 602c9ed2..5ad281e1 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java @@ -28,9 +28,11 @@ import io.questdb.client.std.Files; import io.questdb.client.std.FilesFacade; import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.ObjList; import io.questdb.client.std.Os; import io.questdb.client.std.QuietCloseable; import io.questdb.client.std.Unsafe; +import io.questdb.client.std.str.Utf8s; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -553,6 +555,121 @@ public long findLastFrameFsnWithoutPayloadFlag(int flagsOffset, int flagMask, in * retired orphan-deferred tail -- pass * {@code recoveredCommitBoundaryFsn} */ + /** + * Rebuilds, from the frames' own delta sections, the symbols a replay of + * {@code [minFsnInclusive .. maxFsnInclusive]} would register ABOVE {@code coverage}, + * appending them to {@code out} in ascending id order. Returns the coverage the walk + * establishes (one past the highest id registered), or {@code -1} when a frame's delta + * starts ABOVE the coverage reached so far. + *

+ * This is the recovery-time twin of {@code CursorWebSocketSendLoop.accumulateSentDict}: + * it visits the same frames, in the same FSN order, and extracts exactly the entries + * that method will add to the send loop's mirror as those frames go back on the wire. + * The producer's dictionary is seeded from it so the producer's delta baseline and the + * loop's mirror coverage stay in lockstep BY CONSTRUCTION -- which is what makes a slot + * whose {@code .symbol-dict} was torn (or lost entirely) recoverable, as long as the + * surviving frames define the ids themselves. + *

+ * A {@code -1} return is a genuine GAP: the ids between the coverage and the frame's + * delta start were introduced by frames that were acked and trimmed away, so they lived + * ONLY in the lost dictionary and cannot be rebuilt from anything. That is the one case + * the data really must be resent, and it is exactly the condition the send loop's + * torn-dictionary guard ({@code deltaStart > sentDictCount}) would trip on at replay -- + * so the two agree, rather than the producer failing on slots the drainer drains fine. + *

+ * Same frame layout and FSN bound as {@link #maxSymbolDeltaEnd}. + */ + public long collectReplaySymbolsAbove( + int headerMagic, + int flagsOffset, + int flagDeltaMask, + int qwpHeaderSize, + long minFsnInclusive, + long maxFsnInclusive, + long coverage, + ObjList out + ) { + long off = HEADER_SIZE; + long frames = frameCount; + for (long i = 0; i < frames; i++) { + long fsn = baseSeq + i; + int payloadLen = Unsafe.getUnsafe().getInt(mmapAddress + off + 4); + long payload = mmapAddress + off + FRAME_HEADER_SIZE; + if (fsn >= minFsnInclusive + && fsn <= maxFsnInclusive + && payloadLen >= qwpHeaderSize + && payloadLen > flagsOffset + && Unsafe.getUnsafe().getInt(payload) == headerMagic + && (Unsafe.getUnsafe().getByte(payload + flagsOffset) & flagDeltaMask) != 0) { + long p = payload + qwpHeaderSize; + long limit = payload + payloadLen; + long deltaStart = 0L; + int shift = 0; + while (p < limit) { + byte b = Unsafe.getUnsafe().getByte(p++); + deltaStart |= (long) (b & 0x7F) << shift; + if ((b & 0x80) == 0) { + break; + } + shift += 7; + if (shift > 35) { + break; // corrupt run; recovered frames are CRC-valid, so defensive only + } + } + long deltaCount = 0L; + shift = 0; + while (p < limit) { + byte b = Unsafe.getUnsafe().getByte(p++); + deltaCount |= (long) (b & 0x7F) << shift; + if ((b & 0x80) == 0) { + break; + } + shift += 7; + if (shift > 35) { + break; + } + } + if (deltaStart > coverage) { + return -1L; // gap: ids [coverage, deltaStart) are unrecoverable + } + // Walk this frame's [len varint][utf8] entries. Ids below the coverage we + // already hold are skipped (a replayed frame legitimately overlaps the tip); + // ids at or above it extend the dictionary. + long id = deltaStart; + for (long k = 0; k < deltaCount; k++, id++) { + long len = 0L; + shift = 0; + while (p < limit) { + byte b = Unsafe.getUnsafe().getByte(p++); + len |= (long) (b & 0x7F) << shift; + if ((b & 0x80) == 0) { + break; + } + shift += 7; + if (shift > 35) { + break; + } + } + if (p + len > limit) { + // Malformed entry region. Do NOT guess: report it as unrecoverable + // so the caller fails clean rather than seeding a shifted dictionary. + return -1L; + } + if (id >= coverage) { + out.add(Utf8s.stringFromUtf8Bytes(p, p + len)); + } + p += len; + } + long deltaEnd = deltaStart + deltaCount; + if (deltaEnd > coverage) { + coverage = deltaEnd; + } + } + off += FRAME_HEADER_SIZE + payloadLen; + } + return coverage; + } + public long maxSymbolDeltaEnd(int headerMagic, int flagsOffset, int flagDeltaMask, int qwpHeaderSize, long maxFsnInclusive) { long maxEnd = 0L; long off = HEADER_SIZE; diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java index f9cd08c6..3ec06014 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java @@ -554,6 +554,41 @@ public synchronized long findLastFsnWithoutPayloadFlag(int flagsOffset, int flag * reference. Frames above {@code maxFsnInclusive} are the retired orphan-deferred * tail and are excluded. */ + /** + * Rebuilds the symbols a replay of {@code [minFsnInclusive .. maxFsnInclusive]} would + * register ABOVE {@code baseline}, appending them to {@code out} in ascending id order. + * Returns the coverage the replay establishes, or {@code -1} on a genuine gap. See + * {@link MmapSegment#collectReplaySymbolsAbove}. + *

+ * Walks the segments in ascending FSN order -- {@code sealedSegments} are held in + * baseSeq order and {@code active} is the newest (see {@code recover}) -- threading the + * coverage through, because unlike {@link #maxSymbolDeltaEnd} (a plain max) this walk is + * ORDER-DEPENDENT: each frame may only extend a dictionary the frames before it built. + */ + public synchronized long collectReplaySymbolsAbove( + int headerMagic, + int flagsOffset, + int flagDeltaMask, + int qwpHeaderSize, + long minFsnInclusive, + long maxFsnInclusive, + long baseline, + ObjList out + ) { + long coverage = baseline; + for (int i = 0, n = sealedSegments.size(); i < n; i++) { + coverage = sealedSegments.get(i).collectReplaySymbolsAbove( + headerMagic, flagsOffset, flagDeltaMask, qwpHeaderSize, + minFsnInclusive, maxFsnInclusive, coverage, out); + if (coverage < 0) { + return -1L; + } + } + return active.collectReplaySymbolsAbove( + headerMagic, flagsOffset, flagDeltaMask, qwpHeaderSize, + minFsnInclusive, maxFsnInclusive, coverage, out); + } + public synchronized long maxSymbolDeltaEnd(int headerMagic, int flagsOffset, int flagDeltaMask, int qwpHeaderSize, long maxFsnInclusive) { long maxEnd = 0L; for (int i = 0, n = sealedSegments.size(); i < n; i++) { diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java index e576902b..31153408 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java @@ -204,7 +204,7 @@ public void testTornDictionaryFailsCleanlyInsteadOfCorrupting() throws Exception 0, handler.frames.get()); Assert.assertNotNull("a torn dictionary must surface a terminal error", terminal); Assert.assertTrue(terminal.getMessage(), - terminal.getMessage().contains("subset of the surviving frames") + terminal.getMessage().contains("cannot be rebuilt from the surviving frames") && terminal.getMessage().contains("resend")); } }); @@ -347,6 +347,106 @@ public void testRecoveredSenderContinuesIngestingNewSymbols() throws Exception { }); } + @Test + public void testUnopenablePersistedDictReplaysSelfSufficientFrameSequence() throws Exception { + // Untrimmed counterpart of + // testUnopenablePersistedDictStillGuardsAgainstReplayingDeltaFrames. + // + // Same failure mode -- the persisted dictionary cannot be OPENED, so + // CursorSendEngine.isDeltaDictEnabled() is false -- but NOTHING was acked, so + // replay starts at the FIRST frame. Frames 0..5 carry deltaStart 0..5 and are + // COLLECTIVELY self-sufficient: replayed in order from id 0, the server + // accumulates the dictionary contiguously, needing no dictionary file at all. + // + // Pre-fix, accumulateSentDict was gated on deltaDictEnabled while the + // torn-dict guard was NOT, so sentDictCount froze at 0: frame 0 (deltaStart=0) + // passed the guard, but frame 1 (deltaStart=1 > 0) tripped it and latched a + // terminal -- and for the background drainer that means markFailed + a .failed + // sentinel, permanently quarantining a slot that drains perfectly. A store- + // and-forward contract violation: a TRANSIENT disk condition (fd exhaustion, a + // read-only remount, ENOSPC) stranding recoverable data. + // + // The trimmed sibling pins the other half of the contract: when replay starts + // ABOVE the frames that introduced the ids, the terminal is still correct. + assertMemoryLeak(() -> { + // Phase 1: each row introduces a new symbol => frame i carries deltaStart=i. + // The server never acks, so nothing is trimmed and replay starts at frame 0 + // (the common orphan-drain profile: the server was down the whole time). + try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) { + int port = silent.getPort(); + silent.start(); + Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS)); + String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + + ";close_flush_timeout_millis=0;"; + try (Sender s1 = Sender.fromConfig(cfg)) { + for (int i = 0; i < 6; i++) { + s1.table("m").symbol("s", "sym-" + i).longColumn("v", i).atNow(); + s1.flush(); + } + } + } + + // Make the persisted dictionary UNOPENABLE: a directory of the same name + // defeats both openRW and openCleanRW, so PersistedSymbolDict.open() + // returns null. Deliberately do NOT stamp the ack watermark -- replay must + // start at frame 0, which is what makes the sequence self-sufficient. + java.nio.file.Path slot = Paths.get(sfDir, "default"); + java.nio.file.Path dict = slot.resolve(".symbol-dict"); + java.nio.file.Files.delete(dict); + java.nio.file.Files.createDirectory(dict); + + // Phase 2: recover against a fresh server that reconstructs its per- + // connection dictionary from the wire exactly as the real one does -- + // null-padding any gap, so a gap surfaces as a null rather than passing + // silently. + DictReconstructingHandler handler = new DictReconstructingHandler(); + try (TestWebSocketServer good = new TestWebSocketServer(handler)) { + int port = good.getPort(); + good.start(); + Assert.assertTrue(good.awaitStart(5, TimeUnit.SECONDS)); + + String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";"; + LineSenderException terminal = null; + Sender s2 = Sender.fromConfig(cfg); + try { + long deadline = System.currentTimeMillis() + 10_000; + while (System.currentTimeMillis() < deadline + && handler.maxDictSize() < 6 + && terminal == null) { + try { + s2.flush(); + Thread.sleep(20); + } catch (LineSenderException e) { + terminal = e; + } + } + } finally { + try { + s2.close(); + } catch (LineSenderException e) { + if (terminal == null) { + terminal = e; + } + } + } + + // Pre-fix: the "symbol dictionary is incomplete" terminal fires on frame 1. + Assert.assertNull("a self-sufficient frame sequence must replay without a terminal: " + + (terminal == null ? "" : terminal.getMessage()), + terminal); + // Pre-fix: the server dictionary stops at ["sym-0"] -- frame 1 never shipped. + List serverDict = handler.dictSnapshot(); + Assert.assertEquals("every delta frame must replay [dict=" + serverDict + ']', + 6, serverDict.size()); + for (int i = 0; i < 6; i++) { + Assert.assertEquals("id " + i + " must resolve; a null here is a server-side " + + "null-pad, i.e. a silently NULL symbol column [dict=" + serverDict + ']', + "sym-" + i, serverDict.get(i)); + } + } + }); + } + @Test public void testUnopenablePersistedDictStillGuardsAgainstReplayingDeltaFrames() throws Exception { // C1 regression: when a recovered disk slot's persisted dictionary cannot be @@ -760,23 +860,29 @@ public void testRecoverySeedKeepsUtf8CollidingSymbolsInLockstep() throws Excepti } @Test - public void testTornDictSubsetFailsCleanOnResume() throws Exception { - // C1 regression: the persisted .symbol-dict is NOT fsync'd, so a host crash - // can lose its highest-id tail entries while the segment frames that - // introduced those ids survive (out-of-order page loss). Recovery then seeds - // the producer from the SHORTER dictionary, but the I/O-thread catch-up mirror - // rebuilds the missing ids from those surviving frames -- so a resumed producer - // would assign its next new symbol an id the frames already define, silently - // misattributing symbol values on the wire. The send loop's replay guard only - // catches a GAP (deltaStart > mirror); a frame introducing exactly the - // torn-off id slips through and self-heals the mirror, leaving only this - // producer diverged. seedGlobalDictionaryFromPersisted must detect that the - // surviving frames reference an id at/beyond the recovered dictionary size and - // fail clean. Without the fix the resume succeeds and the reuse corrupts - // silently -- so the build below would NOT throw and the test's fail() fires. + public void testTornDictSubsetRebuildsFromSurvivingFrames() throws Exception { + // The persisted .symbol-dict is NOT fsync'd, so a host crash can lose its + // highest-id tail entries while the segment frames that introduced those ids + // survive (out-of-order page loss). Those frames carry the torn-off symbols in + // their OWN delta sections -- which is exactly why the background drainer replays + // such a slot perfectly: accumulateSentDict rebuilds the dictionary from them. + // + // The producer must do the same. Pre-fix it seeded ONLY from the short dictionary, + // saw that the frames out-reached it, and threw -- permanently bricking + // Sender.build() for a slot the drainer drains fine. And the brick is permanent: + // build()'s catch releases the slot lock, but the sender that would have hosted the + // orphan drainer is the one that just failed, so nothing drains the slot and the + // retry recovers it and throws again. seedGlobalDictionaryFromPersisted now seeds + // from the dictionary's intact prefix AND THEN from the surviving frames' deltas, + // so the producer's baseline lands on exactly the coverage the send loop's mirror + // will reach. + // + // testTornDictionaryFailsCleanlyInsteadOfCorrupting pins the other half of the + // contract: when the symbol-introducing frames were acked and TRIMMED away, the ids + // really are unrecoverable and the resume must still fail clean. assertMemoryLeak(() -> { - // Phase 1: record three delta frames (a@0, b@1, c@2) to disk with nothing - // acked (silent server), so all three survive and replay on recovery. + // Phase 1: three delta frames (a@0, b@1, c@2) with nothing acked, so all three + // survive and replay from frame 0. try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) { int port = silent.getPort(); silent.start(); @@ -796,10 +902,10 @@ public void testTornDictSubsetFailsCleanOnResume() throws Exception { } } - // Simulate the host-crash tear: rewrite the persisted dictionary to drop - // its highest entry (c@2), keeping a@0,b@1 -- while the frame referencing - // c@2 stays on disk. openClean truncates; the two appends leave the exact - // two-entry dictionary a torn tail would recover to. + // Simulate the host-crash tear: rewrite the dictionary to drop its highest + // entry (c@2), keeping a@0,b@1 -- while the frame that introduced c@2 stays on + // disk. openClean truncates; the two appends leave exactly the two-entry + // dictionary a torn tail would recover to. String slotDir = Paths.get(sfDir, "default").toString(); try (PersistedSymbolDict torn = PersistedSymbolDict.openClean(slotDir)) { Assert.assertNotNull(torn); @@ -808,44 +914,58 @@ public void testTornDictSubsetFailsCleanOnResume() throws Exception { Assert.assertEquals(2, torn.size()); } - // Phase 2: a resuming sender must refuse -- the surviving frames reference - // id 2 but the recovered dictionary holds only 2 id(s) [0,1]. - try (TestWebSocketServer good = new TestWebSocketServer(new DictReconstructingHandler())) { + // Phase 2: the resuming sender must REBUILD c@2 from that surviving frame, + // replay everything, and keep ingesting ABOVE the recovered tip. + DictReconstructingHandler handler = new DictReconstructingHandler(); + try (TestWebSocketServer good = new TestWebSocketServer(handler)) { int port = good.getPort(); good.start(); Assert.assertTrue(good.awaitStart(5, TimeUnit.SECONDS)); String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";"; - try { - Sender resumed = Sender.fromConfig(cfg); - resumed.close(); - Assert.fail("resume on a torn (subset) dictionary must fail clean"); - } catch (LineSenderException expected) { - Assert.assertTrue("message must name the torn-dict/resend failure: " + expected.getMessage(), - expected.getMessage().contains("subset of the surviving frames") - && expected.getMessage().contains("resend")); + // Pre-fix this line THREW ("subset of the surviving frames"). + try (Sender resumed = Sender.fromConfig(cfg)) { + long deadline = System.currentTimeMillis() + 5_000; + while (System.currentTimeMillis() < deadline && handler.maxDictSize() < 3) { + Thread.sleep(20); + } + // A NEW symbol must land ABOVE the recovered tip. Seeded short, the + // producer would hand "d" an id the surviving frames already define, + // silently overwriting that symbol on the server -- the corruption the + // old throw existed to prevent, and which this seeding removes at the + // source rather than by refusing to start. + resumed.table("m").symbol("s", "d").longColumn("v", 3).atNow(); + resumed.flush(); + deadline = System.currentTimeMillis() + 5_000; + while (System.currentTimeMillis() < deadline && handler.maxDictSize() < 4) { + Thread.sleep(20); + } } + + List dict = handler.dictSnapshot(); + Assert.assertEquals("dictionary must rebuild gap-free [" + dict + ']', 4, dict.size()); + Assert.assertEquals("a", dict.get(0)); + Assert.assertEquals("b", dict.get(1)); + // Rebuilt from the surviving frame's own delta, not from the torn file. + Assert.assertEquals("c", dict.get(2)); + // The new symbol, placed above the recovered tip -- no id reuse. + Assert.assertEquals("d", dict.get(3)); } }); } @Test - public void testTornDictTotalLossFailsCleanOnResume() throws Exception { - // C1 regression (TOTAL-loss variant of testTornDictSubsetFailsCleanOnResume): - // a host crash can lose the ENTIRE persisted .symbol-dict (size 0) while the - // frames that introduced its ids survive; equivalently, a prior session whose - // openClean failed ran full-dict mode and never wrote the side-file, so this - // session recovers those frames against a FRESH EMPTY dictionary. The recovered - // frames start at deltaStart=0 and self-heal the I/O-thread catch-up mirror, so - // the send loop's replay guard (deltaStart > mirror) never fires -- only - // seedGlobalDictionaryFromPersisted can catch it. Its pd.size()==0 early return - // must NOT skip the torn-dict guard: the surviving frames reference id 2 while - // the recovered dictionary holds 0 id(s), so resuming unseeded would reuse those - // ids and silently misattribute symbol values on the wire. Without the fix the - // build below does NOT throw (the guard is bypassed) and the test's fail() fires. + public void testTornDictTotalLossRebuildsFromSurvivingFrames() throws Exception { + // The total-loss counterpart of testTornDictSubsetRebuildsFromSurvivingFrames: the + // whole dictionary is gone (size 0) but the file still opens, so + // isDeltaDictEnabled() stays true and the engine does NOT discard it (the frames + // carry deltaStart 0,1,2, so maxSymbolDeltaStart != 0 and the full-dict-fallback + // discard correctly stays out of the way). + // + // The surviving frames start at id 0 and are collectively self-sufficient, so the + // producer can rebuild the ENTIRE dictionary from their deltas -- exactly as the + // send loop's mirror does. Pre-fix this threw and bricked build(). assertMemoryLeak(() -> { - // Phase 1: record three delta frames (a@0, b@1, c@2) with nothing acked, so - // all three survive and replay -- from FSN 0 (deltaStart=0), the case the - // I/O-thread guard self-heals rather than rejects. + // Phase 1: a@0, b@1, c@2; nothing acked, so all three replay from frame 0. try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) { int port = silent.getPort(); silent.start(); @@ -865,35 +985,48 @@ public void testTornDictTotalLossFailsCleanOnResume() throws Exception { } } - // Total dictionary loss: openClean truncates .symbol-dict to its bare header - // (size 0) with zero appends, while every frame -- including the ones - // referencing a@0,b@1,c@2 -- stays on disk. This is the fresh-empty-dict - // state a full-dict-mode prior session or a total host-crash tear leaves. + // Total tear: the dictionary recovers EMPTY but still opens. String slotDir = Paths.get(sfDir, "default").toString(); try (PersistedSymbolDict torn = PersistedSymbolDict.openClean(slotDir)) { Assert.assertNotNull(torn); Assert.assertEquals(0, torn.size()); } - // Phase 2: a resuming sender must refuse at build -- the surviving frames - // reference id 2 but the recovered dictionary holds 0 id(s). The I/O-thread - // guard cannot help (deltaStart 0 self-heals the mirror), so the seed-time - // guard is the sole defense. - try (TestWebSocketServer good = new TestWebSocketServer(new DictReconstructingHandler())) { + DictReconstructingHandler handler = new DictReconstructingHandler(); + try (TestWebSocketServer good = new TestWebSocketServer(handler)) { int port = good.getPort(); good.start(); Assert.assertTrue(good.awaitStart(5, TimeUnit.SECONDS)); String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";"; - try { - Sender resumed = Sender.fromConfig(cfg); - resumed.close(); - Assert.fail("resume on a totally lost (size 0) dictionary must fail clean"); - } catch (LineSenderException expected) { - Assert.assertTrue("message must name the torn-dict/resend failure: " + expected.getMessage(), - expected.getMessage().contains("subset of the surviving frames") - && expected.getMessage().contains("resend")); + // Pre-fix this line THREW. + try (Sender resumed = Sender.fromConfig(cfg)) { + long deadline = System.currentTimeMillis() + 5_000; + while (System.currentTimeMillis() < deadline && handler.maxDictSize() < 3) { + Thread.sleep(20); + } + resumed.table("m").symbol("s", "d").longColumn("v", 3).atNow(); + resumed.flush(); + deadline = System.currentTimeMillis() + 5_000; + while (System.currentTimeMillis() < deadline && handler.maxDictSize() < 4) { + Thread.sleep(20); + } } + + List dict = handler.dictSnapshot(); + Assert.assertEquals("the whole dictionary must rebuild from the frames [" + dict + ']', + 4, dict.size()); + Assert.assertEquals("a", dict.get(0)); + Assert.assertEquals("b", dict.get(1)); + Assert.assertEquals("c", dict.get(2)); + Assert.assertEquals("d", dict.get(3)); } + // The side-file is healed in passing -- persistNewSymbolsBeforePublish resumes + // from pd.size() (0 here), so the flush above re-persisted the rebuilt ids + // a,b,c along with d. That is not asserted on disk because it is not + // observable here: the slot drains fully, and a fully-drained close removes + // the dictionary (CursorSendEngine -> PersistedSymbolDict.removeOrphan), so + // re-opening it would just yield a fresh empty file. The persist-resumes-from- + // pd.size() behaviour is pinned by the testFailedPublish* tests. }); } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java index ba81775f..49411275 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java @@ -280,6 +280,19 @@ public void testCatchUpCapGapRetriesUntilBudgetThenLatches() throws Exception { Field maxField = CursorWebSocketSendLoop.class.getDeclaredField("MAX_CATCHUP_CAP_GAP_ATTEMPTS"); maxField.setAccessible(true); int maxAttempts = maxField.getInt(null); + // Pin the budget against a LITERAL before deriving anything from it. The + // retriable loop below is bounded by maxAttempts, so keying this test purely + // off the constant under test makes it TAUTOLOGICAL: a regression of + // MAX_CATCHUP_CAP_GAP_ATTEMPTS to 1 -- which is precisely the pre-fix bug this + // test names, a single cap gap killing the sender -- would run the loop ZERO + // times, the "exhausting" attempt would become the FIRST attempt, and the test + // would still pass green. Requiring > 1 makes that regression fail here, and it + // also guarantees the loop runs at least once, so the first cap gap is genuinely + // asserted retriable rather than vacuously skipped. + assertTrue("the cap-gap settle budget must tolerate MORE THAN ONE gap, else a single " + + "transient failover to a smaller-cap node kills the sender " + + "[MAX_CATCHUP_CAP_GAP_ATTEMPTS=" + maxAttempts + ']', + maxAttempts > 1); // cap 160 => catch-up budget is below a ~216-byte solo frame for a 200-char symbol. CatchUpCapturingClient client = new CatchUpCapturingClient(160); try (CursorSendEngine engine = newEngine()) { @@ -334,6 +347,13 @@ public void testSuccessfulCatchUpResetsCapGapBudget() throws Exception { Field maxField = CursorWebSocketSendLoop.class.getDeclaredField("MAX_CATCHUP_CAP_GAP_ATTEMPTS"); maxField.setAccessible(true); int maxAttempts = maxField.getInt(null); + // Same anti-tautology pin as testCatchUpCapGapRetriesUntilBudgetThenLatches. + // With maxAttempts == 1 the accrual loop below would run ZERO times and the + // "budget accrued to max-1" precondition would degenerate to 0 == 0, so the + // reset-to-0 assertion that is the whole point of this test would prove nothing. + assertTrue("the cap-gap settle budget must tolerate MORE THAN ONE gap " + + "[MAX_CATCHUP_CAP_GAP_ATTEMPTS=" + maxAttempts + ']', + maxAttempts > 1); CatchUpCapturingClient client = new CatchUpCapturingClient(160); // too small for a 200-char symbol try (CursorSendEngine engine = newEngine()) { CursorWebSocketSendLoop loop = newLoop(engine, client); From 017652fae06d4ab17b568682f2955f575590b34d Mon Sep 17 00:00:00 2001 From: glasstiger Date: Tue, 14 Jul 2026 16:05:53 +0100 Subject: [PATCH 55/96] Keep the producer alive through a torn symbol dictionary Four fixes to the QWP delta symbol-dictionary work, all on the store-and-forward recovery and failover paths. Set an unreplayable slot aside instead of bricking the sender. A host crash can tear the unsynced .symbol-dict relative to the frames it serves, and the resulting frames genuinely cannot be replayed. Detecting that is right; throwing out of build() was not. senderId defaults to a stable name, a not-fully-drained slot is retained on close, and so every subsequent build() re-recovered the same slot and threw again -- forever, until an operator removed the directory by hand. The application could not construct a Sender at all, so it could not even buffer new rows: one already-lost batch became an unbounded outage of everything after it. build() now renames the slot aside, marks it .failed for the orphan drainer, and continues on a fresh empty slot. The bytes are kept for resend; not one of its frames ever reaches the wire. Never recreate a dictionary over an existing file. open() fell through to openFresh() -- which is O_TRUNC -- on any read or parse failure, so a single transient EIO, or a rollback to a client that does not know a newer version byte, destroyed the only copy of load-bearing state and made every surviving delta frame permanently unreplayable. It now degrades to null, leaving the file intact: the sender falls back to full self-sufficient frames and a later attempt can still recover the slot in full. Give the catch-up cap-gap terminal a wall-clock dwell. The settle budget was attempt-based, and 16 attempts at the capped reconnect backoff burn through in about two minutes -- less than an ordinary rolling restart of the larger-cap node, which is the very transient the budget exists to ride out. Escalation now needs both the strike count and a minimum dwell, anchored at the first cap gap of the episode, defaulting to five minutes and tunable through catchup_cap_gap_min_escalation_window_millis. Checksum the persisted dictionary per chunk, not per entry. Crc32c is a native call, so a per-entry checksum put one JNI transition, one sub-cache-line copy and one redundant varint decode on the producer thread for every new symbol -- a thousand native calls per flush on the one-new-symbol-per-row batch this feature exists to serve. A chunk is exactly the set of symbols one frame introduces, and every frame's deltaStart is a chunk boundary, so per-entry granularity bought no extra recoverable prefix. Recovery also folds its two passes into one. ensureScratch now throws instead of clamping below the requested size, which would have turned a clean out-of-memory into a native-heap overflow on the dictionary write path. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../main/java/io/questdb/client/Sender.java | 137 +++- .../qwp/client/QwpWebSocketSender.java | 19 +- .../client/sf/cursor/BackgroundDrainer.java | 17 +- .../client/sf/cursor/CursorSendEngine.java | 32 + .../sf/cursor/CursorWebSocketSendLoop.java | 118 +++- .../client/sf/cursor/PersistedSymbolDict.java | 612 ++++++++++-------- .../io/questdb/client/impl/ConfigSchema.java | 1 + .../qwp/client/DeltaDictCatchUpTest.java | 11 +- ...WebSocketSendLoopCatchUpAlignmentTest.java | 124 +++- .../sf/cursor/PersistedSymbolDictTest.java | 260 +++++--- 10 files changed, 974 insertions(+), 357 deletions(-) diff --git a/core/src/main/java/io/questdb/client/Sender.java b/core/src/main/java/io/questdb/client/Sender.java index 4ec70dbd..447d5fad 100644 --- a/core/src/main/java/io/questdb/client/Sender.java +++ b/core/src/main/java/io/questdb/client/Sender.java @@ -38,6 +38,8 @@ import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender; import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine; import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorWebSocketSendLoop; +import io.questdb.client.cutlass.qwp.client.sf.cursor.OrphanScanner; +import io.questdb.client.cutlass.qwp.client.sf.cursor.PersistedSymbolDict; import io.questdb.client.impl.ConfStringParser; import io.questdb.client.impl.ConfigString; import io.questdb.client.impl.ConfigView; @@ -988,6 +990,12 @@ final class LineSenderBuilder { // build() time. 0 or negative is a documented "disable" value, so // a Long.MIN_VALUE sentinel keeps it distinguishable from "unset". private static final long DURABLE_ACK_KEEPALIVE_NOT_SET = Long.MIN_VALUE; + private static final org.slf4j.Logger LOG = org.slf4j.LoggerFactory.getLogger(LineSenderBuilder.class); + // How many quarantined copies of one slot may pile up under sf_dir before build() + // refuses to set aside another. Each is an unreplayable slot a human still has to + // look at; accumulating them without bound would turn a disk-space problem into a + // second incident. + private static final int MAX_QUARANTINE_SLOT_ATTEMPTS = 64; private static final int MIN_BUFFER_SIZE = AuthUtils.CHALLENGE_LEN + 1; // challenge size + 1; // sf-client.md section 4.4: the inbox capacity must accommodate the // distinct error categories in a bursty error stream so that drop-oldest @@ -1003,6 +1011,10 @@ final class LineSenderBuilder { private static final int PROTOCOL_TCP = 0; private static final int PROTOCOL_UDP = 3; private static final int PROTOCOL_WEBSOCKET = 2; + // Suffix for a slot set aside by quarantineTornSlot. Deliberately NOT the + // sender's own slot name, so OrphanScanner sees the quarantined copy and the + // orphan drainer can still deliver it if its frames turn out to be replayable. + private static final String QUARANTINE_SLOT_SUFFIX = ".unreplayable-"; private final ObjList hosts = new ObjList<>(); private final IntList ports = new IntList(); private long authTimeoutMillis = QwpWebSocketSender.DEFAULT_AUTH_TIMEOUT_MS; @@ -1051,6 +1063,7 @@ final class LineSenderBuilder { private int errorInboxCapacity = PARAMETER_NOT_SET_EXPLICITLY; private int maxFrameRejections = PARAMETER_NOT_SET_EXPLICITLY; private long poisonMinEscalationWindowMillis = PARAMETER_NOT_SET_EXPLICITLY; + private long catchUpCapGapMinEscalationWindowMillis = PARAMETER_NOT_SET_EXPLICITLY; private String httpPath; private String httpSettingsPath; private int httpTimeout = PARAMETER_NOT_SET_EXPLICITLY; @@ -1505,6 +1518,10 @@ public Sender build() { long actualPoisonMinEscalationWindowMillis = poisonMinEscalationWindowMillis != PARAMETER_NOT_SET_EXPLICITLY ? poisonMinEscalationWindowMillis : CursorWebSocketSendLoop.DEFAULT_POISON_MIN_ESCALATION_WINDOW_MILLIS; + long actualCatchUpCapGapMinEscalationWindowMillis = + catchUpCapGapMinEscalationWindowMillis != PARAMETER_NOT_SET_EXPLICITLY + ? catchUpCapGapMinEscalationWindowMillis + : CursorWebSocketSendLoop.DEFAULT_CATCHUP_CAP_GAP_MIN_ESCALATION_WINDOW_MILLIS; // sfDir is the parent (group root); the actual slot lives // under sfDir/senderId. This is what the engine sees — the @@ -1543,6 +1560,11 @@ public Sender build() { CursorSendEngine cursorEngine = new CursorSendEngine( slotPath, actualSfMaxBytes, actualSfMaxTotalBytes, actualSfAppendDeadlineNanos); + if (cursorEngine.isRecoveredDictionaryIncomplete()) { + cursorEngine = quarantineTornSlot( + cursorEngine, sfDir, senderId, slotPath, actualSfMaxBytes, + actualSfMaxTotalBytes, actualSfAppendDeadlineNanos); + } int actualErrorInboxCapacity = errorInboxCapacity != PARAMETER_NOT_SET_EXPLICITLY ? errorInboxCapacity : io.questdb.client.cutlass.qwp.client.sf.cursor.SenderErrorDispatcher.DEFAULT_CAPACITY; @@ -1578,7 +1600,8 @@ public Sender build() { connectionListener, actualConnectionListenerInboxCapacity, actualMaxFrameRejections, - actualPoisonMinEscalationWindowMillis + actualPoisonMinEscalationWindowMillis, + actualCatchUpCapGapMinEscalationWindowMillis ); } catch (Throwable t) { // connect() failed before ownership of cursorEngine @@ -1720,6 +1743,37 @@ public Sender build() { *

* WebSocket transport only. */ + /** + * Minimum wall-clock time (millis) a symbol-dictionary catch-up CAP GAP must + * persist before the sender gives up on it and fails. + *

+ * A cap gap means a symbol already accepted by one node is too large to + * re-register on the node the sender just failed over to, because that node + * advertises a smaller maximum batch size. On a homogeneous cluster this cannot + * happen; it takes a heterogeneous or mid-roll cluster, or an operator lowering + * the cap below existing data. + *

+ * The sender retries such a gap across reconnects rather than failing on sight, + * because the larger-cap node may simply be away -- a rolling restart is the most + * likely reason a failover happened at all. It gives up only once the gap has BOTH + * recurred many times AND persisted for this long, so an ordinary cluster + * operation cannot bring down a live producer. Raise it for a cluster whose + * rolling restarts take longer than the 5-minute default; set it to {@code 0} to + * fail as soon as the retry count is exhausted. + *

+ * WebSocket transport only. + */ + public LineSenderBuilder catchUpCapGapMinEscalationWindowMillis(long millis) { + if (protocol != PARAMETER_NOT_SET_EXPLICITLY && protocol != PROTOCOL_WEBSOCKET) { + throw new LineSenderException("catchup_cap_gap_min_escalation_window_millis is only supported for WebSocket transport"); + } + if (millis < 0) { + throw new LineSenderException("catchup_cap_gap_min_escalation_window_millis must be >= 0: ").put(millis); + } + this.catchUpCapGapMinEscalationWindowMillis = millis; + return this; + } + public LineSenderBuilder closeFlushTimeoutMillis(long timeoutMillis) { if (protocol != PARAMETER_NOT_SET_EXPLICITLY && protocol != PROTOCOL_WEBSOCKET) { throw new LineSenderException("close_flush_timeout_millis is only supported for WebSocket transport"); @@ -2916,6 +2970,77 @@ private static long parseSizeValue(@NotNull StringSink value, @NotNull String na } } + /** + * Sets a recovered slot whose symbol dictionary cannot cover its surviving frames + * aside, and returns a fresh engine on an empty slot so the producer can keep + * producing. + *

+ * Such a slot is unreplayable BY THIS PRODUCER: its frames reference symbol ids the + * recovered dictionary lost (a host/power crash tore the unsynced side-file), so a + * producer seeded from the short dictionary would hand those ids to different + * symbols and silently misattribute values. Detecting that is correct and + * load-bearing -- but simply THROWING is not a safe response. {@code senderId} + * defaults to a stable name, so a restarted process re-adopts the same slot; the + * engine's close retains a slot that is not fully drained; and so every subsequent + * {@code build()} would re-recover the same slot and throw again -- forever, until + * an operator deleted the directory by hand. The application could not construct a + * Sender at all, and so could not even BUFFER new rows. That trades a bounded, + * already-lost batch for an unbounded outage of everything after it, which inverts + * the one guarantee store-and-forward exists to give. + *

+ * So: rename the slot aside instead. The bytes are preserved for forensics and for + * the orphan drainer, which reaches the same verdict independently (its send loop's + * replay guard fires and it marks the slot {@code .failed}) -- and which, on a slot + * that turns out to be drainable after all (frames written in full-dictionary + * fallback mode are self-sufficient), simply drains it. The new name is NOT the + * sender's own slot name, so {@code OrphanScanner} will consider it. The producer, + * meanwhile, starts on a clean empty slot and never notices. + *

+ * If the rename fails (a Windows share lock, a read-only mount) there is no way to + * free the slot name without destroying data, so fall back to the old behaviour and + * throw -- loudly, and never silently dropping bytes. + */ + private static CursorSendEngine quarantineTornSlot( + CursorSendEngine torn, String sfDir, String senderId, String slotPath, + long sfMaxBytes, long sfMaxTotalBytes, long sfAppendDeadlineNanos + ) { + long recoveredMaxSymbolId = torn.recoveredMaxSymbolId(); + PersistedSymbolDict dict = torn.getPersistedSymbolDict(); + int coverage = dict != null ? dict.size() : 0; + String detail = "recovered store-and-forward symbol dictionary cannot cover the surviving " + + "frames (likely a host crash tore its unsynced tail): frames reference symbol id " + + recoveredMaxSymbolId + " but the recovered dictionary holds only " + coverage + + " id(s)"; + // Release the slot lock and the dictionary fd before renaming. The slot is not + // fully drained, so close() retains every file. + torn.close(); + + String quarantinePath = null; + for (int i = 0; i < MAX_QUARANTINE_SLOT_ATTEMPTS; i++) { + String candidate = sfDir + "/" + senderId + QUARANTINE_SLOT_SUFFIX + i; + if (!Files.exists(candidate)) { + quarantinePath = candidate; + break; + } + } + if (quarantinePath == null || Files.rename(slotPath, quarantinePath) != 0) { + throw new LineSenderException( + detail + "; the affected data must be resent. The slot could not be set aside " + + "automatically (" + (quarantinePath == null + ? "too many quarantined slots already under " + sfDir + : "rename to " + quarantinePath + " failed") + + "), so this sender cannot start until " + + slotPath + " is moved or removed by hand"); + } + // Mark the quarantined copy so the orphan drainer treats it as a + // human-in-the-loop slot rather than silently retrying it forever. + OrphanScanner.markFailed(quarantinePath, detail); + LOG.error("{} -- the slot has been set aside at {} and the affected data must be resent; " + + "this sender continues on a fresh, empty slot at {}", + detail, quarantinePath, slotPath); + return new CursorSendEngine(slotPath, sfMaxBytes, sfMaxTotalBytes, sfAppendDeadlineNanos); + } + private static int resolveIPv4(String host) { try { byte[] addr = InetAddress.getByName(host).getAddress(); @@ -3413,6 +3538,12 @@ private LineSenderBuilder fromConfig(CharSequence configurationString) { } pos = getValue(configurationString, pos, sink, "poison_min_escalation_window_millis"); poisonMinEscalationWindowMillis(parseLongValue(sink, "poison_min_escalation_window_millis")); + } else if (Chars.equals("catchup_cap_gap_min_escalation_window_millis", sink)) { + if (protocol != PROTOCOL_WEBSOCKET) { + throw new LineSenderException("catchup_cap_gap_min_escalation_window_millis is only supported for WebSocket transport"); + } + pos = getValue(configurationString, pos, sink, "catchup_cap_gap_min_escalation_window_millis"); + catchUpCapGapMinEscalationWindowMillis(parseLongValue(sink, "catchup_cap_gap_min_escalation_window_millis")); } else if (Chars.equals("initial_connect_retry", sink)) { if (protocol != PROTOCOL_WEBSOCKET) { throw new LineSenderException("initial_connect_retry is only supported for WebSocket transport"); @@ -3686,6 +3817,9 @@ private LineSenderBuilder fromConfigWebSocket(CharSequence configurationString) if (view.has("poison_min_escalation_window_millis")) { poisonMinEscalationWindowMillis(wsLong(view, v, "poison_min_escalation_window_millis")); } + if (view.has("catchup_cap_gap_min_escalation_window_millis")) { + catchUpCapGapMinEscalationWindowMillis(wsLong(view, v, "catchup_cap_gap_min_escalation_window_millis")); + } if (view.has("sf_append_deadline_millis")) { sfAppendDeadlineMillis(wsLong(view, v, "sf_append_deadline_millis")); } @@ -3862,6 +3996,7 @@ public java.util.Map wsConfigSnapshotForTest() { m.put("max_background_drainers", maxBackgroundDrainers); m.put("max_frame_rejections", maxFrameRejections); m.put("poison_min_escalation_window_millis", poisonMinEscalationWindowMillis); + m.put("catchup_cap_gap_min_escalation_window_millis", catchUpCapGapMinEscalationWindowMillis); m.put("error_inbox_capacity", errorInboxCapacity); m.put("connection_listener_inbox_capacity", connectionListenerInboxCapacity); m.put("token", httpToken); diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index bc08948e..fb9a7f52 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -313,6 +313,12 @@ public class QwpWebSocketSender implements Sender { // maxFrameRejections (connect-string key poison_min_escalation_window_millis). private long poisonMinEscalationWindowMillis = CursorWebSocketSendLoop.DEFAULT_POISON_MIN_ESCALATION_WINDOW_MILLIS; + // Minimum wall-clock dwell a symbol-dict catch-up cap gap must persist before the + // send loop latches a terminal (connect-string key + // catchup_cap_gap_min_escalation_window_millis). See + // CursorWebSocketSendLoop.DEFAULT_CATCHUP_CAP_GAP_MIN_ESCALATION_WINDOW_MILLIS. + private long catchUpCapGapMinEscalationWindowMillis = + CursorWebSocketSendLoop.DEFAULT_CATCHUP_CAP_GAP_MIN_ESCALATION_WINDOW_MILLIS; private long reconnectInitialBackoffMillis = CursorWebSocketSendLoop.DEFAULT_RECONNECT_INITIAL_BACKOFF_MILLIS; private long reconnectMaxBackoffMillis = @@ -691,7 +697,8 @@ public static QwpWebSocketSender connect( durableAckKeepaliveIntervalMillis, authTimeoutMs, connectTimeoutMs, connectionListener, connectionListenerInboxCapacity, CursorWebSocketSendLoop.DEFAULT_MAX_HEAD_FRAME_REJECTIONS, - CursorWebSocketSendLoop.DEFAULT_POISON_MIN_ESCALATION_WINDOW_MILLIS); + CursorWebSocketSendLoop.DEFAULT_POISON_MIN_ESCALATION_WINDOW_MILLIS, + CursorWebSocketSendLoop.DEFAULT_CATCHUP_CAP_GAP_MIN_ESCALATION_WINDOW_MILLIS); } /** @@ -722,7 +729,8 @@ public static QwpWebSocketSender connect( SenderConnectionListener connectionListener, int connectionListenerInboxCapacity, int maxFrameRejections, - long poisonMinEscalationWindowMillis + long poisonMinEscalationWindowMillis, + long catchUpCapGapMinEscalationWindowMillis ) { QwpWebSocketSender sender = new QwpWebSocketSender( endpoints, tlsConfig, @@ -740,6 +748,7 @@ public static QwpWebSocketSender connect( sender.durableAckKeepaliveIntervalMillis = durableAckKeepaliveIntervalMillis; sender.maxFrameRejections = maxFrameRejections; sender.poisonMinEscalationWindowMillis = poisonMinEscalationWindowMillis; + sender.catchUpCapGapMinEscalationWindowMillis = catchUpCapGapMinEscalationWindowMillis; sender.initialConnectMode = initialConnectMode == null ? Sender.InitialConnectMode.OFF : initialConnectMode; @@ -2414,7 +2423,8 @@ public synchronized void startOrphanDrainers( requestDurableAck, durableAckKeepaliveIntervalMillis, maxFrameRejections, - poisonMinEscalationWindowMillis); + poisonMinEscalationWindowMillis, + catchUpCapGapMinEscalationWindowMillis); ref[0] = drainer; drainerPool.submit(drainer); } @@ -3342,7 +3352,8 @@ private void ensureConnected() { requestDurableAck, durableAckKeepaliveIntervalMillis, maxFrameRejections, - poisonMinEscalationWindowMillis); + poisonMinEscalationWindowMillis, + catchUpCapGapMinEscalationWindowMillis); // Plug the async-delivery sink before start() so the I/O thread // never observes a null dispatcher between recordFatal and // notification — the test for null in dispatchError handles diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/BackgroundDrainer.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/BackgroundDrainer.java index d79080d4..a88429ce 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/BackgroundDrainer.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/BackgroundDrainer.java @@ -132,6 +132,11 @@ public final class BackgroundDrainer implements Runnable { // Minimum wall-clock dwell before poison escalation, forwarded to every // drain loop; mirrors the owner sender's poison_min_escalation_window_millis. private final long poisonMinEscalationWindowMillis; + // Minimum wall-clock dwell a symbol-dict catch-up cap gap must persist before this + // drainer's send loop latches a terminal. Same knob the foreground sender uses; the + // orphan drainer honours it too so a rolling restart cannot quarantine an otherwise + // drainable slot on a strike count alone. + private final long catchUpCapGapMinEscalationWindowMillis; public BackgroundDrainer( String slotPath, @@ -149,7 +154,8 @@ public BackgroundDrainer( reconnectMaxBackoffMillis, requestDurableAck, durableAckKeepaliveIntervalMillis, CursorWebSocketSendLoop.DEFAULT_MAX_HEAD_FRAME_REJECTIONS, - CursorWebSocketSendLoop.DEFAULT_POISON_MIN_ESCALATION_WINDOW_MILLIS); + CursorWebSocketSendLoop.DEFAULT_POISON_MIN_ESCALATION_WINDOW_MILLIS, + CursorWebSocketSendLoop.DEFAULT_CATCHUP_CAP_GAP_MIN_ESCALATION_WINDOW_MILLIS); } /** @@ -169,7 +175,8 @@ public BackgroundDrainer( boolean requestDurableAck, long durableAckKeepaliveIntervalMillis, int maxHeadFrameRejections, - long poisonMinEscalationWindowMillis + long poisonMinEscalationWindowMillis, + long catchUpCapGapMinEscalationWindowMillis ) { this.slotPath = slotPath; this.segmentSizeBytes = segmentSizeBytes; @@ -182,6 +189,7 @@ public BackgroundDrainer( this.durableAckKeepaliveIntervalMillis = durableAckKeepaliveIntervalMillis; this.maxHeadFrameRejections = maxHeadFrameRejections; this.poisonMinEscalationWindowMillis = poisonMinEscalationWindowMillis; + this.catchUpCapGapMinEscalationWindowMillis = catchUpCapGapMinEscalationWindowMillis; } /** @@ -194,7 +202,7 @@ public BackgroundDrainer( @TestOnly public BackgroundDrainer() { this(null, 0L, 0L, null, 0L, 0L, 0L, false, 0L, - CursorWebSocketSendLoop.DEFAULT_MAX_HEAD_FRAME_REJECTIONS, 0L); + CursorWebSocketSendLoop.DEFAULT_MAX_HEAD_FRAME_REJECTIONS, 0L, 0L); } /** @@ -575,7 +583,8 @@ public void run() { requestDurableAck, durableAckKeepaliveIntervalMillis, maxHeadFrameRejections, - poisonMinEscalationWindowMillis); + poisonMinEscalationWindowMillis, + catchUpCapGapMinEscalationWindowMillis); loop.start(); while (!stopRequestedOrInterrupted()) { diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java index 91035a0e..a2e20d35 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java @@ -744,6 +744,38 @@ public boolean isDeltaDictEnabled() { return sfDir == null || persistedSymbolDict != null; } + /** + * Whether this RECOVERED slot's surviving frames reference symbol ids the + * recovered dictionary cannot cover -- i.e. the slot cannot be drained by a + * producer that shares its id space, and a producer seeded from the short + * dictionary would reuse ids those frames already define. + *

+ * True in two situations, both of which mean the same thing to the caller: + *

    + *
  • the dictionary opened but is SHORT of the frames (a host/power crash + * tore its unsynced tail -- see {@code PersistedSymbolDict}'s durability + * note); or
    • + *
  • the dictionary did not open at all ({@code persistedSymbolDict == null} + * -- an unreadable or torn side-file, fd exhaustion, a read-only remount) + * while the surviving frames are DELTA frames that need one. Coverage is + * then zero, so any frame referencing an id at all is unreplayable.
    • + *
+ * Always false for a fresh (non-recovered) slot: {@code recoveredMaxSymbolId} is + * -1 there, and -1 is below every coverage. + *

+ * The caller must NOT drain such a slot with a live producer attached. The + * foreground sender quarantines it and starts on a fresh slot + * ({@code Sender.build()}); the orphan drainer reaches the same verdict + * independently via the send loop's replay guard and marks the slot failed. + * Either way the recorded bytes stay on disk and the affected data must be + * resent -- but the producer keeps producing, which is the whole point of + * store-and-forward. + */ + public boolean isRecoveredDictionaryIncomplete() { + long coverage = persistedSymbolDict != null ? persistedSymbolDict.size() : 0; + return recoveredMaxSymbolId >= coverage; + } + /** * Pass-through to {@link SegmentRing#nextSealedAfter(MmapSegment)}. */ diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java index 93b0b137..8e34011d 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java @@ -132,6 +132,33 @@ public final class CursorWebSocketSendLoop implements QuietCloseable { * {@code LineSenderBuilder.poisonMinEscalationWindowMillis(long)}. */ public static final long DEFAULT_POISON_MIN_ESCALATION_WINDOW_MILLIS = 5_000L; + /** + * Default minimum wall-clock dwell (millis) a symbol-dict catch-up CAP GAP must + * persist before the loop latches a terminal, even once + * {@link #MAX_CATCHUP_CAP_GAP_ATTEMPTS} cap gaps have accrued. Same idea as + * {@link #DEFAULT_POISON_MIN_ESCALATION_WINDOW_MILLIS}, for the same reason: a + * strike count measures "how many times did we look", not "how long has this been + * true", and only the latter distinguishes a permanent cluster capability gap from + * a node that is briefly away. + *

+ * It matters here because the cap gap's trigger IS a failover. With the reconnect + * backoff capped at {@link #DEFAULT_RECONNECT_MAX_BACKOFF_MILLIS} (5 s), 16 cap + * gaps accrue in roughly two minutes -- comfortably less than an ordinary rolling + * restart of the larger-cap node. A count-only budget would therefore hard-fail a + * live store-and-forward producer during a routine cluster operation, which is + * exactly the transient the budget exists to ride out. Requiring the dwell as well + * means the terminal needs BOTH "we have looked many times" AND "it has stayed true + * for a long time"; a node that returns inside the window ends the episode and the + * producer never notices. + *

+ * 5 minutes -- the same figure as {@link #DEFAULT_RECONNECT_MAX_DURATION_MILLIS}, + * this codebase's existing notion of how long a transient outage may plausibly + * last. Configurable per sender via the + * {@code catchup_cap_gap_min_escalation_window_millis} connect-string key or + * {@code LineSenderBuilder.catchUpCapGapMinEscalationWindowMillis(long)}; {@code 0} + * restores count-only escalation. + */ + public static final long DEFAULT_CATCHUP_CAP_GAP_MIN_ESCALATION_WINDOW_MILLIS = 300_000L; private static final Logger LOG = LoggerFactory.getLogger(CursorWebSocketSendLoop.class); // Settle budget for the symbol-dict catch-up cap gap: how many cap-gap attempts // -- catch-ups that reached a fresh server and found a single dictionary entry @@ -185,6 +212,11 @@ public final class CursorWebSocketSendLoop implements QuietCloseable { // by the server -- holding stale watermarks across the wire boundary // would falsely advance trim before re-confirmation. private final CharSequenceLongHashMap durableTableWatermarks = new CharSequenceLongHashMap(); + // Pre-converted to nanos. Zero disables the dwell entirely (count-only escalation + // at MAX_CATCHUP_CAP_GAP_ATTEMPTS) -- the raw-constructor default; the user-facing + // 5-minute default is applied at the config layer, mirroring + // poisonMinEscalationWindowNanos. + private final long catchUpCapGapMinEscalationWindowNanos; private final CursorSendEngine engine; private final long parkNanos; // FIFO of OK-acked batches awaiting durable-upload confirmation. Used only @@ -267,6 +299,16 @@ public final class CursorWebSocketSendLoop implements QuietCloseable { // nor resets it. NOT reset per connection -- it measures the cap-gap episode // across reconnects so a persistent gap eventually latches. I/O-thread-only. private int catchUpCapGapAttempts; + // System.nanoTime() of the FIRST cap gap of the current episode, or -1 when no + // episode is open. Anchors the escalation dwell. Reset together with + // catchUpCapGapAttempts on a successful catch-up, so a node that accepts the + // dictionary ends the episode outright. Anchoring at the FIRST gap (not at loop + // entry, and not refreshed per attempt) is what keeps a TRANSIENT from burning the + // terminal budget: a transient never reaches the catch-up, so it neither increments + // the counter nor moves the anchor -- it can only lengthen the wall clock, which + // alone can never latch the terminal because the strike count still has to be + // satisfied. I/O-thread-only, like catchUpCapGapAttempts. + private long catchUpCapGapFirstNanos = -1L; // True once a real ring frame (data or commit) has been sent on the CURRENT // connection, as opposed to only the dictionary catch-up. The catch-up consumes // wire sequences (nextWireSeq), so nextWireSeq > 0 no longer implies "the head @@ -534,14 +576,47 @@ public CursorWebSocketSendLoop(WebSocketClient client, CursorSendEngine engine, durableAckKeepaliveIntervalMillis, maxHeadFrameRejections, 0L); } + /** + * Twelve-arg overload — omits the symbol-dict cap-gap escalation dwell, which + * defaults to {@code 0} (legacy: escalate as soon as + * {@link #MAX_CATCHUP_CAP_GAP_ATTEMPTS} cap gaps accrue, with no minimum + * wall-clock). Like the poison dwell, the user-facing default + * ({@link #DEFAULT_CATCHUP_CAP_GAP_MIN_ESCALATION_WINDOW_MILLIS}) is applied at the + * config layer (Sender / QwpWebSocketSender), so this raw constructor stays + * unopinionated for tests and internal callers that want deterministic strike-count + * escalation. + */ + public CursorWebSocketSendLoop(WebSocketClient client, CursorSendEngine engine, + long fsnAtZero, long parkNanos, + ReconnectFactory reconnectFactory, + long reconnectMaxDurationMillis, + long reconnectInitialBackoffMillis, + long reconnectMaxBackoffMillis, + boolean durableAckMode, + long durableAckKeepaliveIntervalMillis, + int maxHeadFrameRejections, + long poisonMinEscalationWindowMillis) { + this(client, engine, fsnAtZero, parkNanos, reconnectFactory, + reconnectMaxDurationMillis, reconnectInitialBackoffMillis, + reconnectMaxBackoffMillis, durableAckMode, + durableAckKeepaliveIntervalMillis, maxHeadFrameRejections, + poisonMinEscalationWindowMillis, 0L); + } + /** * Master constructor — also accepts the poison-frame detector threshold * ({@code max_frame_rejections}): consecutive server-active rejections of * the same head-of-line frame, with no ack progress in between, before the - * loop escalates to a typed terminal. Must be {@code >= 1}. The final - * argument is the minimum wall-clock dwell (millis) the suspect must stay - * poisoned before escalation ({@code poison_min_escalation_window_millis}; - * {@code >= 0}, where 0 = legacy immediate escalation at the threshold). + * loop escalates to a typed terminal. Must be {@code >= 1}. Then the minimum + * wall-clock dwell (millis) the suspect must stay poisoned before escalation + * ({@code poison_min_escalation_window_millis}; {@code >= 0}, where 0 = legacy + * immediate escalation at the threshold). + *

+ * The final argument is the analogous dwell for the symbol-dict catch-up cap gap + * ({@code catchup_cap_gap_min_escalation_window_millis}; {@code >= 0}, where 0 = + * escalate as soon as {@link #MAX_CATCHUP_CAP_GAP_ATTEMPTS} cap gaps accrue). See + * {@link #DEFAULT_CATCHUP_CAP_GAP_MIN_ESCALATION_WINDOW_MILLIS} for why a strike + * count alone is not a safe escalation signal on a live producer. */ public CursorWebSocketSendLoop(WebSocketClient client, CursorSendEngine engine, long fsnAtZero, long parkNanos, @@ -552,7 +627,8 @@ public CursorWebSocketSendLoop(WebSocketClient client, CursorSendEngine engine, boolean durableAckMode, long durableAckKeepaliveIntervalMillis, int maxHeadFrameRejections, - long poisonMinEscalationWindowMillis) { + long poisonMinEscalationWindowMillis, + long catchUpCapGapMinEscalationWindowMillis) { if (maxHeadFrameRejections < 1) { throw new IllegalArgumentException( "maxHeadFrameRejections must be >= 1: " + maxHeadFrameRejections); @@ -561,6 +637,13 @@ public CursorWebSocketSendLoop(WebSocketClient client, CursorSendEngine engine, throw new IllegalArgumentException( "poisonMinEscalationWindowMillis must be >= 0: " + poisonMinEscalationWindowMillis); } + if (catchUpCapGapMinEscalationWindowMillis < 0) { + throw new IllegalArgumentException( + "catchUpCapGapMinEscalationWindowMillis must be >= 0: " + + catchUpCapGapMinEscalationWindowMillis); + } + this.catchUpCapGapMinEscalationWindowNanos = + catchUpCapGapMinEscalationWindowMillis * 1_000_000L; if (engine == null) { throw new IllegalArgumentException("engine must be non-null"); } @@ -2314,12 +2397,27 @@ private int sendDictCatchUp() { // recordFatal, NOT fail() -- failing from inside the catch-up would // re-enter connectLoop (see CatchUpSendException); the data must be // resent after the cap is raised. + // Escalation needs BOTH the strike count AND a minimum wall-clock dwell + // (see DEFAULT_CATCHUP_CAP_GAP_MIN_ESCALATION_WINDOW_MILLIS). A count + // alone measures how often we looked, not how long the gap has held -- + // and 16 attempts at the capped backoff take only ~2 minutes, less than + // an ordinary rolling restart of the larger-cap node. Escalating on the + // count alone would therefore hard-fail a live store-and-forward + // producer during a routine cluster operation. + long nowNanos = System.nanoTime(); + if (catchUpCapGapFirstNanos < 0) { + catchUpCapGapFirstNanos = nowNanos; + } catchUpCapGapAttempts++; - boolean exhausted = catchUpCapGapAttempts >= MAX_CATCHUP_CAP_GAP_ATTEMPTS; + long episodeNanos = nowNanos - catchUpCapGapFirstNanos; + boolean exhausted = catchUpCapGapAttempts >= MAX_CATCHUP_CAP_GAP_ATTEMPTS + && episodeNanos >= catchUpCapGapMinEscalationWindowNanos; LineSenderException err = new LineSenderException( "symbol dictionary entry too large for the server batch cap during catch-up [" + "frameLen=" + soloFrameLen + ", cap=" + cap + ", attempt=" - + catchUpCapGapAttempts + '/' + MAX_CATCHUP_CAP_GAP_ATTEMPTS + ']' + + catchUpCapGapAttempts + '/' + MAX_CATCHUP_CAP_GAP_ATTEMPTS + + ", episodeMillis=" + (episodeNanos / 1_000_000L) + + '/' + (catchUpCapGapMinEscalationWindowNanos / 1_000_000L) + ']' + (exhausted ? "; the data must be resent after the cap is raised" : "; retrying -- a larger-cap node may return")); @@ -2345,8 +2443,12 @@ private int sendDictCatchUp() { framesSent++; } // The whole dictionary re-registered without a cap gap: this node accepts - // every entry, so the cap-gap episode (if any) is over -- reset the budget. + // every entry, so the cap-gap episode (if any) is over -- reset BOTH the strike + // count and the wall-clock anchor. Resetting only the count would leave a stale + // anchor from an old episode, so the very first strike of a LATER episode would + // already satisfy the dwell. catchUpCapGapAttempts = 0; + catchUpCapGapFirstNanos = -1L; return framesSent; } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java index 83371201..411346d0 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java @@ -50,20 +50,37 @@ * {@link AckWatermark} -- a discardable optimization protected by a * {@code max()} clamp -- this file is load-bearing: a surviving frame * that references an id missing from it is unrecoverable. It is therefore held - * to a stronger durability contract. + * to a stronger durability contract, and {@link #open} never destroys it (see + * "Never recreate over an existing file" below). *

* Layout (little-endian): * 

  *   offset 0: u32 magic = 'SYD1'
- *   offset 4: u8  version = 2
+ *   offset 4: u8  version = 3
  *   offset 5: 3 bytes reserved (zero)
- *   offset 8: entries, each [len: varint][utf8 bytes][crc32c: u32], in ascending global-id order
+ *   offset 8: chunks, each
+ *             [entryCount: varint][entryBytes: varint][entries][crc32c: u32]
+ *             where entries = [len: varint][utf8] repeated entryCount times,
+ *             occupying exactly entryBytes bytes, and the CRC-32C covers the
+ *             two header varints AND the entry region.
  *
- * Symbol id {@code i} is the {@code i}-th entry (ids are dense and assigned - * sequentially from 0), so no id needs to be stored. Each entry carries a - * CRC-32C over its {@code [len][utf8]} bytes (the same checksum the SF segment - * frames use), so a torn or stale entry is detected on recovery instead of - * being silently mis-parsed. + * A chunk is one append -- i.e. exactly the set of symbols one frame + * introduces, since the producer persists a frame's new symbols in a single call + * before publishing it. Symbol id {@code i} is the {@code i}-th entry across all + * chunks (ids are dense and assigned sequentially from 0), so no id is stored. + *

+ * Why the checksum is per chunk, not per entry. The only consumer of the + * recovered prefix is the send loop's replay guard, which compares a surviving + * frame's {@code deltaStart} against the recovered dictionary size -- and every + * {@code deltaStart} is a chunk boundary, because chunks and frame deltas are + * written one-for-one. A tear inside a chunk therefore invalidates exactly the + * frames a per-entry checksum would have invalidated anyway: per-entry + * granularity buys no extra recoverable prefix. It costs a great deal, though. + * {@link Crc32c#update} is a native call, so checksumming per entry put one JNI + * transition -- plus one sub-cache-line copy and one redundant varint decode -- + * on the producer thread for every new symbol. On the high-cardinality batch this + * feature exists to serve (one new symbol per row), that is a thousand native + * calls per flush where the chunk needs one. *

* Durability / write-ahead ordering: the producer appends the symbols a * frame introduces BEFORE that frame is published to the ring, but does NOT @@ -76,14 +93,12 @@ * exactly as the segment frames themselves may be lost on a host crash. Two * layers keep a host-crash tear from silently corrupting data: *

    - *
  • The per-entry CRC-32C: {@link #open} verifies every entry and stops at + *
  • The per-chunk CRC-32C: {@link #open} verifies every chunk and stops at * the first one whose checksum fails, so an interior page lost out of - * order (reading back as zeroes) or a stale entry left past the end by a + * order (reading back as zeroes) or a stale chunk left past the end by a * failed truncate is DETECTED and the trusted region ends before it -- - * recovery never mis-parses a corrupt entry as a real symbol nor shifts - * the dense id->symbol map. The truncate that drops a torn/stale tail is - * now failure-checked (see {@link #open}): a file that cannot be trimmed - * is untrusted and recreated empty rather than left exposing stale bytes.
    • + * recovery never mis-parses a corrupt chunk as real symbols nor shifts the + * dense id->symbol map. *
  • The send loop's replay guard: once recovery trusts only the intact * prefix, a surviving frame whose delta start id exceeds that prefix * fails loudly (the unreplayable data must be resent) rather than sending @@ -93,10 +108,22 @@ * "resend required" instead of a silent symbol misattribution -- the same * CRC-32C protection the segment frames carry. A tear that happened to leave a * byte run whose CRC still matches is not distinguished, but that is a 1-in-2^32 - * collision per corrupted entry, no weaker than the frames' own checksum. + * collision per corrupted chunk, no weaker than the frames' own checksum. + *

    + * A torn trailing chunk from a crash mid-append is self-healing: {@link #open} + * stops parsing at the first incomplete chunk and the next append overwrites it. *

    - * A torn trailing entry from a crash mid-append is self-healing: {@link #open} - * stops parsing at the first incomplete entry and the next append overwrites it. + * Never recreate over an existing file. {@link #open} -- the RECOVERY + * entry point -- returns {@code null} when an existing file cannot be read or + * parsed, and NEVER falls back to recreating it empty. Recreating would mean + * {@code O_TRUNC} over the only copy of load-bearing state, so a single transient + * read error (an EIO on a flaky disk, a short read) would permanently destroy the + * dictionary the surviving delta frames reference -- turning a recoverable outage + * into unrecoverable data. A {@code null} instead degrades the sender to full + * self-sufficient frames and leaves every byte on disk, so a later attempt, once + * the transient clears, can still recover the slot in full. Only + * {@link #openClean} -- the FRESH-slot path, where discarding is the whole point + * -- truncates. *

    * Lifecycle: single-writer (the producer / user thread) for appends. Read * once at {@link #open} to seed in-memory state on recovery or orphan-drain. The @@ -115,10 +142,25 @@ public final class PersistedSymbolDict implements QuietCloseable { * OrphanScanner, trim) skip it automatically. */ public static final String FILE_NAME = ".symbol-dict"; - static final int CRC_SIZE = 4; // u32 CRC-32C trailing every entry + static final int CRC_SIZE = 4; // u32 CRC-32C trailing every chunk static final int FILE_MAGIC = 0x31445953; // 'SYD1' little-endian static final int HEADER_SIZE = 8; - static final byte VERSION = 2; // v2 appended the per-entry CRC-32C + /** + * Upper bound on a chunk's two header varints ({@code entryCount} and + * {@code entryBytes}): each is at most 5 bytes for a 32-bit value. The + * encoders reserve this much in front of the entry region so the header can + * be back-filled once the region's exact size is known, keeping header, + * entries and CRC one contiguous run. + */ + static final int MAX_CHUNK_HEADER_SIZE = 10; + /** + * Ceiling for the append scratch buffer, mirroring + * {@code CursorWebSocketSendLoop.MAX_SENT_DICT_BYTES}: the capacity math is + * int-typed, so a larger buffer cannot be addressed. Exceeding it throws -- + * {@link #ensureScratch} never silently under-allocates. + */ + static final int MAX_SCRATCH_BYTES = Integer.MAX_VALUE - 8; + static final byte VERSION = 3; // v3 moved the CRC-32C from per-entry to per-chunk private static final Logger LOG = LoggerFactory.getLogger(PersistedSymbolDict.class); private final int fd; // Filesystem seam. Production is FilesFacade.INSTANCE (straight to Files); @@ -127,14 +169,14 @@ public final class PersistedSymbolDict implements QuietCloseable { private final FilesFacade ff; private long appendOffset; private boolean closed; - // In-memory copy of the entry region [len][utf8]... exactly as on disk, - // populated only when open() recovered existing entries (recovery / - // orphan-drain). Zero/empty for a freshly created file. READ (not consumed) to - // seed the producer's id map (readLoadedSymbols) and to seed the send loop's - // catch-up mirror, which COPIES it. This file retains ownership for the engine's - // lifetime -- the orphan drainer builds a fresh send loop per wire session - // against the same engine, and each must re-seed its mirror -- and frees this - // buffer in close(). + // In-memory copy of the WIRE entry region [len][utf8]... -- chunk headers and + // CRCs stripped -- populated only when open() recovered existing chunks + // (recovery / orphan-drain). Zero/empty for a freshly created file. READ (not + // consumed) to seed the producer's id map (readLoadedSymbols) and to seed the + // send loop's catch-up mirror, which COPIES it. This file retains ownership for + // the engine's lifetime -- the orphan drainer builds a fresh send loop per wire + // session against the same engine, and each must re-seed its mirror -- and frees + // this buffer in close(). private long loadedEntriesAddr; private int loadedEntriesLen; private long scratchAddr; @@ -151,12 +193,16 @@ private PersistedSymbolDict(FilesFacade ff, int fd, long appendOffset, int size, } /** - * Opens (creating if absent) the dictionary file in {@code slotDir}. An - * existing file is parsed and its complete entries are loaded into memory - * (see {@link #loadedEntriesAddr()}); a missing or invalid file is (re)created - * with a fresh header. Returns {@code null} on any I/O failure -- the caller - * then falls back to full-dictionary (self-sufficient) frames for this slot, - * so a broken side-file degrades gracefully rather than aborting the sender. + * Opens the dictionary file in {@code slotDir} for RECOVERY, creating it only + * when it does not already exist. An existing file is parsed and its complete, + * CRC-valid chunks are loaded into memory (see {@link #loadedEntriesAddr()}). + *

    + * Returns {@code null} on any I/O or parse failure -- including an existing file + * that cannot be read, carries an unknown version, or fails its checksums. The + * caller then falls back to full-dictionary (self-sufficient) frames for this + * slot, so a broken side-file degrades gracefully rather than aborting the + * sender. Crucially, a {@code null} return NEVER destroys the file: see the + * class-level "Never recreate over an existing file" note. */ public static PersistedSymbolDict open(String slotDir) { return open(FilesFacade.INSTANCE, slotDir); @@ -170,30 +216,27 @@ public static PersistedSymbolDict open(String slotDir) { public static PersistedSymbolDict open(FilesFacade ff, String slotDir) { String filePath = slotDir + "/" + FILE_NAME; long existing = ff.exists(filePath) ? ff.length(filePath) : -1L; - // A dictionary that grew to or past Integer.MAX_VALUE cannot be reopened: - // openExisting reads it into ONE int-sized native buffer. PAST 2GiB the - // (int) cast is either negative (malloc rejects it), exactly zero (getInt - // then reads 4 bytes past a zero-size allocation), or a small positive prefix - // (whose validLen < len branch would then DESTRUCTIVELY truncate the multi-GB - // file); AT exactly Integer.MAX_VALUE the cast is exact but the ~2GB malloc is - // doomed to OutOfMemoryError. The >= guard short-circuits every case to a - // clean re-create instead of the doomed allocation -- fail-clean, exactly like - // every other unreadable-file case here, so the sender falls back to full - // self-sufficient frames. Reaching this needs ~100M+ distinct symbols on one - // slot (far past realistic symbol cardinality); the guard keeps the read/write - // size boundary safe anyway. - if (existing >= Integer.MAX_VALUE) { - LOG.warn("symbol dict {} too large ({} bytes) to reopen; recreating empty", filePath, existing); - return openFresh(ff, filePath); - } if (existing >= HEADER_SIZE) { - PersistedSymbolDict d = openExisting(ff, filePath, existing); - if (d != null) { - return d; + // A dictionary at or past Integer.MAX_VALUE cannot be read back: + // openExisting reads it into ONE int-sized native buffer, and the (int) + // cast is either negative (malloc rejects it) or a small positive prefix + // (whose parse would then trim the real multi-GB file). Degrade to full + // self-sufficient frames and leave the file alone. Reaching this needs + // ~100M+ distinct symbols on one slot, far past realistic cardinality; + // the guard keeps the read/write size boundary safe anyway. + if (existing >= Integer.MAX_VALUE) { + LOG.warn("symbol dict {} too large ({} bytes) to reopen; " + + "falling back to full-dictionary frames (file left intact)", filePath, existing); + return null; } - // Fall through to a clean re-create: a header/parse failure on an - // existing file means it cannot be trusted for delta replay. + // NEVER recreate over an existing file on the recovery path: openFresh + // truncates, and these bytes are the only copy of state the surviving + // delta frames reference. A null degrades this slot to full + // self-sufficient frames and preserves the file for a later attempt. + return openExisting(ff, filePath, existing); } + // Absent, or a sub-header stub left by a crash inside openFresh: no + // load-bearing content to lose, so create it. return openFresh(ff, filePath); } @@ -204,14 +247,14 @@ public static PersistedSymbolDict open(FilesFacade ff, String slotDir) { * dictionary left by a prior lifecycle -- a fully-drained slot whose * best-effort delete failed, or a crash in the close window -- must NOT be * inherited. Unlike {@link #open}, which parses and TRUSTS an existing file for - * recovery/orphan-drain replay, this truncates it: the fresh-start producer is - * not seeded from the dictionary, so trusting a survivor would leave the - * producer's ids diverged from the dictionary the send loop replays and - * silently misattribute symbols on the next reconnect. Truncating (rather than - * relying on an unlink succeeding first) closes the gap even when the delete is - * refused -- e.g. a Windows share lock. Returns {@code null} on I/O failure, so - * the caller falls back to full self-sufficient frames exactly as {@link #open} - * does. + * recovery/orphan-drain replay and never destroys it, this truncates: the + * fresh-start producer is not seeded from the dictionary, so trusting a survivor + * would leave the producer's ids diverged from the dictionary the send loop + * replays and silently misattribute symbols on the next reconnect. Truncating + * (rather than relying on an unlink succeeding first) closes the gap even when + * the delete is refused -- e.g. a Windows share lock. Returns {@code null} on + * I/O failure, so the caller falls back to full self-sufficient frames exactly + * as {@link #open} does. */ public static PersistedSymbolDict openClean(String slotDir) { return openClean(FilesFacade.INSTANCE, slotDir); @@ -244,28 +287,33 @@ public static void removeOrphan(FilesFacade ff, String slotDir) { /** * Appends {@code count} wire entries -- {@code [len varint][utf8]...}, the - * symbol-dict delta section the frame encoder just wrote -- to the on-disk - * dictionary, computing and appending a per-entry CRC-32C as it copies so the - * producer does not re-encode the symbols. The on-disk layout is - * {@code [len varint][utf8][crc32c]} per entry (see the class layout note); the - * {@code addr}/{@code len} bytes carry no CRC, so this walks the {@code count} - * entries to insert one. Advances {@code size} by {@code count}. Same - * durability/idempotency contract as {@link #appendSymbols}: no fsync, and a - * short write throws WITHOUT advancing {@code size}/{@code appendOffset}, so a - * retry keyed off {@link #size()} re-persists the same range at the same - * offset. No-op when the range is empty or the dictionary is closed. + * symbol-dict delta section the frame encoder just wrote -- as ONE chunk. + *

    + * The consistency walk below decodes each entry's length varint, but the bytes + * themselves are copied in a SINGLE {@code copyMemory} and checksummed by a + * SINGLE {@link Crc32c#update} covering the whole chunk. A per-entry checksum + * would put one JNI transition, one sub-cache-line copy and one redundant varint + * decode on the producer thread per new symbol; the chunk needs one of each. + *

    + * Advances {@code size} by {@code count}. Same durability/idempotency contract + * as {@link #appendSymbols}: no fsync, and a short write throws WITHOUT + * advancing {@code size}/{@code appendOffset}, so a retry keyed off + * {@link #size()} re-persists the same range at the same offset. No-op when the + * range is empty or the dictionary is closed. */ public synchronized void appendRawEntries(long addr, int len, int count) { if (closed || count <= 0 || len <= 0) { return; } - int outLen = len + count * CRC_SIZE; - ensureScratch(outLen); + // Validate the (addr, len, count) triple BEFORE writing anything: an + // inconsistent triple would record a chunk whose stored entryCount disagreed + // with the entries it holds, shifting the dense id->symbol map on recovery. + // The sole caller derives count and len from one beginMessage, so this cannot + // fire today -- but the file this guards is the one the "resend required" + // contract rests on, so it stays. long src = addr; long srcLimit = addr + len; - long dst = scratchAddr; for (int i = 0; i < count; i++) { - long entryStart = src; long symLen = 0; int shift = 0; while (src < srcLimit) { @@ -277,51 +325,37 @@ public synchronized void appendRawEntries(long addr, int len, int count) { shift += 7; if (shift > 35) { // A canonical entry-length varint is <= 5 bytes; a longer - // continuation run is corrupt. The downstream entryEnd > srcLimit - // check then rejects it. Matches decodeVarint / readVarintAt. + // continuation run is corrupt. The bound check below rejects it. break; } } - long entryEnd = src + symLen; // src is just past the len varint - if (entryEnd > srcLimit) { + src += symLen; // src was just past the len varint + if (src > srcLimit) { throw new IllegalStateException("malformed raw symbol-dict entries to " + FILE_NAME + " [entry=" + i + ", count=" + count + ']'); } - int wireSpan = (int) (entryEnd - entryStart); // [len][utf8] - Unsafe.getUnsafe().copyMemory(entryStart, dst, wireSpan); - Unsafe.getUnsafe().putInt(dst + wireSpan, Crc32c.update(Crc32c.INIT, entryStart, wireSpan)); - dst += wireSpan + CRC_SIZE; - src = entryEnd; } if (src != srcLimit) { - // The count entries did not consume exactly len bytes -- a caller passed an - // inconsistent (addr, len, count) triple. Writing outLen would flush an - // uninitialised scratch tail and mis-advance size, so fail loudly. The sole - // caller derives count and len from one beginMessage, so this cannot fire - // today. throw new IllegalStateException("raw symbol-dict entries under-filled the buffer to " + FILE_NAME + " [count=" + count + ", len=" + len + ", consumed=" + (int) (src - addr) + ']'); } - long written = ff.write(fd, scratchAddr, outLen, appendOffset); - if (written != outLen) { - throw new IllegalStateException("short write to " + FILE_NAME - + " [expected=" + outLen + ", actual=" + written + ']'); - } - appendOffset += outLen; - size += count; + int hdrLen = NativeBufferWriter.varintSize(count) + NativeBufferWriter.varintSize(len); + ensureScratch((long) hdrLen + len + CRC_SIZE); + long p = NativeBufferWriter.writeVarint(scratchAddr, count); + NativeBufferWriter.writeVarint(p, len); + Unsafe.getUnsafe().copyMemory(addr, scratchAddr + hdrLen, len); + flushChunk(scratchAddr, hdrLen, len, count); } /** - * Appends one symbol, extending the on-disk dictionary. The caller appends a - * frame's new symbols BEFORE publishing that frame, so the write ordering - * (dictionary entry before referencing frame) holds; no fsync is performed - * (see the class-level durability note). Assigns the next dense id implicitly - * (the entry's position). Writes {@code [len varint][utf8][crc32c]}, the CRC - * covering the {@code [len][utf8]} bytes so a torn/stale entry is detected on - * recovery. + * Appends one symbol as a single-entry chunk, extending the on-disk dictionary. + * The caller appends a frame's new symbols BEFORE publishing that frame, so the + * write ordering (dictionary entry before referencing frame) holds; no fsync is + * performed (see the class-level durability note). Assigns the next dense id + * implicitly (the entry's position). *

    - * Test-only: production persists a frame's whole new-symbol range in one write + * Test-only: production persists a frame's whole new-symbol range in one chunk * via {@link #appendSymbols} / {@link #appendRawEntries}. Tests use this to * build a dictionary one entry at a time. */ @@ -331,65 +365,49 @@ public synchronized void appendSymbol(CharSequence symbol) { return; } int utf8Len = Utf8s.utf8Bytes(symbol); - int varLen = NativeBufferWriter.varintSize(utf8Len); - int wireLen = varLen + utf8Len; // [len][utf8] - int recLen = wireLen + CRC_SIZE; // + trailing crc - ensureScratch(recLen); - long p = NativeBufferWriter.writeVarint(scratchAddr, utf8Len); + int wireLen = NativeBufferWriter.varintSize(utf8Len) + utf8Len; // [len][utf8] + ensureScratch((long) MAX_CHUNK_HEADER_SIZE + wireLen + CRC_SIZE); + long entryStart = scratchAddr + MAX_CHUNK_HEADER_SIZE; + long p = NativeBufferWriter.writeVarint(entryStart, utf8Len); if (utf8Len > 0) { Utf8s.strCpyUtf8(symbol, p, utf8Len); } - Unsafe.getUnsafe().putInt(scratchAddr + wireLen, Crc32c.update(Crc32c.INIT, scratchAddr, wireLen)); - long written = ff.write(fd, scratchAddr, recLen, appendOffset); - if (written != recLen) { - throw new IllegalStateException("short write to " + FILE_NAME - + " [expected=" + recLen + ", actual=" + written + ']'); - } - appendOffset += recLen; - size++; + writeChunkFromScratch(wireLen, 1); } /** - * Appends the dense id range {@code [from .. to]} in a SINGLE write. Encodes - * the whole {@code [len varint][utf8][crc32c]...} region into scratch first, - * then issues one positioned write -- versus one {@code pwrite(2)} per symbol - * via {@link #appendSymbol}. That per-symbol syscall count is the hot-path - * cost on a high-cardinality batch (one new symbol per row), which is exactly - * the store-and-forward workload delta encoding targets. Each entry carries a - * trailing CRC-32C over its {@code [len][utf8]} bytes. Callers pass the - * dictionary and the range so the ids resolve to their symbol strings. + * Appends the dense id range {@code [from .. to]} as ONE chunk, in a SINGLE + * write with a SINGLE checksum. Encodes the whole entry region into scratch + * first -- versus one {@code pwrite(2)} and one native CRC call per symbol. + * That per-symbol syscall and JNI count is the hot-path cost on a + * high-cardinality batch (one new symbol per row), which is exactly the + * store-and-forward workload delta encoding targets. Callers pass the dictionary + * and the range so the ids resolve to their symbol strings. *

    - * Same durability and idempotency contract as {@link #appendSymbol}: no - * fsync, and a short write throws WITHOUT advancing {@code size}/{@code - * appendOffset}, so a retry keyed off {@link #size()} re-encodes the same - * range and overwrites at the same offset. No-op when the range is empty or - * the dictionary is closed. + * Same durability and idempotency contract as {@link #appendSymbol}: no fsync, + * and a short write throws WITHOUT advancing {@code size}/{@code appendOffset}, + * so a retry keyed off {@link #size()} re-encodes the same range and overwrites + * at the same offset. No-op when the range is empty or the dictionary is closed. */ public synchronized void appendSymbols(GlobalSymbolDictionary dict, int from, int to) { if (closed || to < from) { return; } - int len = 0; + int count = to - from + 1; + int entriesLen = 0; for (int id = from; id <= to; id++) { CharSequence symbol = dict.getSymbol(id); int utf8Len = Utf8s.utf8Bytes(symbol); int wireLen = NativeBufferWriter.varintSize(utf8Len) + utf8Len; // [len][utf8] - ensureScratch(len + wireLen + CRC_SIZE); - long entryStart = scratchAddr + len; + ensureScratch((long) MAX_CHUNK_HEADER_SIZE + entriesLen + wireLen + CRC_SIZE); + long entryStart = scratchAddr + MAX_CHUNK_HEADER_SIZE + entriesLen; long p = NativeBufferWriter.writeVarint(entryStart, utf8Len); if (utf8Len > 0) { Utf8s.strCpyUtf8(symbol, p, utf8Len); } - Unsafe.getUnsafe().putInt(entryStart + wireLen, Crc32c.update(Crc32c.INIT, entryStart, wireLen)); - len += wireLen + CRC_SIZE; + entriesLen += wireLen; } - long written = ff.write(fd, scratchAddr, len, appendOffset); - if (written != len) { - throw new IllegalStateException("short write to " + FILE_NAME - + " [expected=" + len + ", actual=" + written + ']'); - } - appendOffset += len; - size += to - from + 1; + writeChunkFromScratch(entriesLen, count); } @Override @@ -418,8 +436,9 @@ public synchronized void close() { /** * Base address of the loaded entry region -- the concatenated - * {@code [len][utf8]} bytes of every recovered symbol in id order, exactly - * as a delta section carries them. Zero when nothing was recovered. + * {@code [len][utf8]} bytes of every recovered symbol in id order, exactly as a + * delta section carries them (chunk headers and CRCs stripped). Zero when + * nothing was recovered. *

    * Construction-phase only. This hands out a raw pointer into native * memory that {@link #close()} frees and nulls, with no closed-guard and no @@ -466,10 +485,10 @@ public ObjList readLoadedSymbols() { } shift += 7; if (shift > 35) { - // Bound the varint like decodeVarint / appendRawEntries / - // readVarintAt: a canonical length is <= 5 bytes. open() already - // CRC-validated these bytes, so this is defensive only; the - // p + len > limit check below then rejects the over-long run. + // Bound the varint like the other decoders: a canonical length is + // <= 5 bytes. open() already CRC-validated these bytes, so this is + // defensive only; the p + len > limit check below rejects the + // over-long run. break; } } @@ -489,47 +508,25 @@ public int size() { return size; } - /** - * Decodes an unsigned LEB128 varint from {@code buf[pos..limit)}. Returns - * {@code [value, newPos]} or {@code null} if the varint is truncated - * (torn tail). - */ - private static long[] decodeVarint(long buf, int pos, int limit) { - long value = 0; - int shift = 0; - int cur = pos; - while (cur < limit) { - byte b = Unsafe.getUnsafe().getByte(buf + cur); - cur++; - value |= (long) (b & 0x7F) << shift; - if ((b & 0x80) == 0) { - return new long[]{value, cur}; - } - shift += 7; - if (shift > 35) { - return null; // implausible for an entry length; treat as torn - } - } - return null; - } - private static PersistedSymbolDict openExisting(FilesFacade ff, String filePath, long fileLen) { int fd = ff.openRW(filePath); if (fd < 0) { - LOG.warn("symbol dict {} could not be opened (rc={}); recreating", filePath, fd); + LOG.warn("symbol dict {} could not be opened (rc={}); " + + "falling back to full-dictionary frames (file left intact)", filePath, fd); return null; } long buf = 0L; long entriesAddr = 0L; - int entriesLen = 0; + int entriesCap = 0; try { - int len = (int) fileLen; + int len = (int) fileLen; // open() bounds fileLen to [HEADER_SIZE, Integer.MAX_VALUE) buf = Unsafe.malloc(len, MemoryTag.NATIVE_DEFAULT); long read = ff.read(fd, buf, len, 0); if (read != len || Unsafe.getUnsafe().getInt(buf) != FILE_MAGIC || Unsafe.getUnsafe().getByte(buf + 4) != VERSION) { - LOG.warn("symbol dict {} unreadable, bad magic or unknown version; recreating", filePath); + LOG.warn("symbol dict {} unreadable, bad magic or unknown version; " + + "falling back to full-dictionary frames (file left intact)", filePath); Unsafe.free(buf, len, MemoryTag.NATIVE_DEFAULT); buf = 0L; // null after free so the catch below cannot double-free if ff.close throws int fdToClose = fd; @@ -537,92 +534,89 @@ private static PersistedSymbolDict openExisting(FilesFacade ff, String filePath, ff.close(fdToClose); return null; } - // Parse complete, CRC-valid entries after the header; stop at the first - // torn/incomplete OR crc-mismatched entry. The CRC turns an interior - // tear (a lost page reading back as zeroes) or a stale entry left past - // the end by a failed truncate into a clean stop point, so recovery - // trusts only the intact prefix instead of silently mis-parsing a - // corrupt entry and shifting the dense id->symbol map. - int diskPos = HEADER_SIZE; // walks the on-disk [len][utf8][crc] entries + // Parse the chunks after the header, copying each chunk's entry region + // into entriesAddr AS WE VALIDATE IT -- one pass over the file, not two. + // Every chunk sheds its two header varints and its CRC, so the entry + // region is strictly smaller than the file and len - HEADER_SIZE is a safe + // upper bound to allocate up front; we shrink to the exact size below. + // + // Stop at the first torn/incomplete OR crc-mismatched chunk. The CRC turns + // an interior tear (a lost page reading back as zeroes) or a stale chunk + // left past the end by a failed truncate into a clean stop point, so + // recovery trusts only the intact prefix instead of silently mis-parsing a + // corrupt chunk and shifting the dense id->symbol map. + entriesCap = len - HEADER_SIZE; + long dst = 0L; + if (entriesCap > 0) { + entriesAddr = Unsafe.malloc(entriesCap, MemoryTag.NATIVE_DEFAULT); + dst = entriesAddr; + } + Varint v = new Varint(); + int diskPos = HEADER_SIZE; int count = 0; - int wireLen = 0; // running size of the crc-stripped copy while (diskPos < len) { - long[] vr = decodeVarint(buf, diskPos, len); - if (vr == null) { - break; // torn length varint + if (!v.decode(buf, diskPos, len)) { + break; // torn entryCount varint + } + long entryCount = v.value; + if (!v.decode(buf, v.end, len)) { + break; // torn entryBytes varint } - long symLen = vr[0]; - int afterVar = (int) vr[1]; - // [len varint][utf8] then a u32 CRC. symLen stays a long so a - // corrupt multi-gigabyte length cannot wrap an int back under the - // bound check. No fixed per-entry ceiling -- the write path applies - // none, so a legitimately large symbol must recover here. - long wireEnd = (long) afterVar + symLen; // end of [len][utf8] - if (wireEnd + CRC_SIZE > len) { - break; // torn/incomplete trailing entry (its CRC doesn't fit) + long entryBytes = v.value; + int entriesStart = v.end; + // entryCount/entryBytes stay long so a corrupt multi-gigabyte value + // cannot wrap an int back under the bound checks. + long chunkEnd = (long) entriesStart + entryBytes; // end of the entry region + if (chunkEnd + CRC_SIZE > len) { + break; // torn/incomplete trailing chunk (its CRC doesn't fit) } - int wireEndI = (int) wireEnd; - int wireSpan = wireEndI - diskPos; - int crcStored = Unsafe.getUnsafe().getInt(buf + wireEndI); - int crcCalc = Crc32c.update(Crc32c.INIT, buf + diskPos, wireSpan); + int chunkEndI = (int) chunkEnd; + int crcStored = Unsafe.getUnsafe().getInt(buf + chunkEndI); + int crcCalc = Crc32c.update(Crc32c.INIT, buf + diskPos, chunkEndI - diskPos); if (crcCalc != crcStored) { - break; // corrupt/stale entry -- stop before it (fail-clean) + break; // corrupt/stale chunk -- stop before it (fail-clean) } - diskPos = wireEndI + CRC_SIZE; - wireLen += wireSpan; - count++; - } - int diskConsumed = diskPos - HEADER_SIZE; // valid entries incl. their CRCs - // Materialise the trusted entries as WIRE bytes ([len][utf8]..., no - // CRC) so loadedEntries*/readLoadedSymbols and the send-loop catch-up - // mirror stay wire-shaped -- the on-disk CRC is stripped here, once, at - // open. A second no-alloc walk over the already-validated region. - if (wireLen > 0) { - entriesAddr = Unsafe.malloc(wireLen, MemoryTag.NATIVE_DEFAULT); - // Record the length alongside the malloc so the catch below frees the - // right size (not 0) if this copy walk ever throws. - entriesLen = wireLen; - long dst = entriesAddr; - int p = HEADER_SIZE; - for (int i = 0; i < count; i++) { - int vp = p; - long symLen = 0; - int shift = 0; - while (true) { - byte b = Unsafe.getUnsafe().getByte(buf + vp++); - symLen |= (long) (b & 0x7F) << shift; - if ((b & 0x80) == 0) { - break; - } - shift += 7; - if (shift > 35) { - break; // corrupt run; these entries were CRC-validated above - } - } - int wireSpan = (vp - p) + (int) symLen; // [len][utf8], no CRC - Unsafe.getUnsafe().copyMemory(buf + p, dst, wireSpan); - dst += wireSpan; - p += wireSpan + CRC_SIZE; // skip the entry's CRC + // A chunk carries at least one entry, and the ids are int-dense. A + // CRC-valid chunk cannot violate either at realistic cardinality, but + // keep the int narrowing honest rather than wrapping the id space. + if (entryCount <= 0 || (long) count + entryCount > Integer.MAX_VALUE) { + break; } + Unsafe.getUnsafe().copyMemory(buf + entriesStart, dst, entryBytes); + dst += entryBytes; + diskPos = chunkEndI + CRC_SIZE; + count += (int) entryCount; + } + int entriesLen = entriesAddr != 0L ? (int) (dst - entriesAddr) : 0; + int diskConsumed = diskPos - HEADER_SIZE; // valid chunks incl. headers and CRCs + if (entriesAddr != 0L && entriesLen == 0) { + Unsafe.free(entriesAddr, entriesCap, MemoryTag.NATIVE_DEFAULT); + entriesAddr = 0L; + entriesCap = 0; + } else if (entriesAddr != 0L && entriesLen < entriesCap) { + // Shrink the upper-bound allocation to what the trusted prefix used. + entriesAddr = Unsafe.realloc(entriesAddr, entriesCap, entriesLen, MemoryTag.NATIVE_DEFAULT); + entriesCap = entriesLen; } Unsafe.free(buf, len, MemoryTag.NATIVE_DEFAULT); buf = 0L; // Drop any torn/stale trailing bytes so a LATER, shorter append cannot - // leave residue past its own end. Unlike before, the truncate result IS - // checked: a file we cannot trim could still expose stale post-end bytes - // whose (self-consistent) per-entry CRC the parse would accept at a - // shifted position, so a failed truncate makes the file untrusted -- - // open() then recreates it empty (fail-clean) rather than risk a silent - // misattribution. + // leave residue past its own end. The truncate result IS checked: a file + // we cannot trim could still expose stale post-end bytes whose + // (self-consistent) chunk CRC the parse would accept at a shifted + // position, so a failed truncate makes the file untrusted -- return null + // (the sender falls back to full self-sufficient frames) and, per the + // never-destroy contract, leave every byte on disk. long validLen = HEADER_SIZE + diskConsumed; if (validLen < len && !ff.truncate(fd, validLen)) { - LOG.warn("symbol dict {} could not drop its torn/stale tail (truncate failed); recreating", filePath); + LOG.warn("symbol dict {} could not drop its torn/stale tail (truncate failed); " + + "falling back to full-dictionary frames (file left intact)", filePath); if (entriesAddr != 0L) { - Unsafe.free(entriesAddr, entriesLen, MemoryTag.NATIVE_DEFAULT); + Unsafe.free(entriesAddr, entriesCap, MemoryTag.NATIVE_DEFAULT); // Null after freeing (like buf above) so the catch below cannot // double-free entriesAddr if the following ff.close throws. entriesAddr = 0L; - entriesLen = 0; + entriesCap = 0; } int fdToClose = fd; fd = -1; // relinquish before close so the catch cannot double-close if close throws @@ -640,12 +634,16 @@ private static PersistedSymbolDict openExisting(FilesFacade ff, String filePath, // double-free. Keeps the error path leak-free on any throw between its // malloc and the return. if (entriesAddr != 0L) { - Unsafe.free(entriesAddr, entriesLen, MemoryTag.NATIVE_DEFAULT); + Unsafe.free(entriesAddr, entriesCap, MemoryTag.NATIVE_DEFAULT); } if (fd >= 0) { // a branch that already closed fd relinquished it to -1 ff.close(fd); } - LOG.warn("symbol dict {} recovery failed ({}); recreating", filePath, String.valueOf(t)); + // Pass the throwable as a trailing argument with no matching placeholder so + // slf4j prints the stack trace: this WARN is the only forensic record of why + // a load-bearing dictionary was abandoned. + LOG.warn("symbol dict {} recovery failed; falling back to full-dictionary frames " + + "(file left intact)", filePath, t); return null; } } @@ -669,19 +667,23 @@ private static PersistedSymbolDict openFresh(FilesFacade ff, String filePath) { int fdToClose = fd; fd = -1; // relinquish before close so the catch cannot double-close if close throws ff.close(fdToClose); - ff.remove(filePath); + ff.remove(filePath); // drop the headerless stub rather than litter LOG.warn("symbol dict {} header write failed; proceeding without it", filePath); return null; } } catch (Throwable t) { - // Unreachable today (Files.write is native and returns -1 rather than - // throwing; the Unsafe puts target a valid 8-byte buffer and an 8-byte - // malloc cannot realistically OOM), but close the fd against a future - // edit so it cannot leak -- mirroring openExisting's error handling. + // Unreachable with FilesFacade.INSTANCE (Files.write is native and returns + // -1 rather than throwing; the Unsafe puts target a valid 8-byte buffer and + // an 8-byte malloc cannot realistically OOM), but the ff seam exists so + // tests CAN inject a throwing facade -- close the fd and drop the stub so + // neither leaks. if (fd >= 0) { // the header-write branch relinquished fd to -1 before closing - ff.close(fd); + int fdToClose = fd; + fd = -1; + ff.close(fdToClose); + ff.remove(filePath); } - LOG.warn("symbol dict {} creation failed ({}); proceeding without it", filePath, String.valueOf(t)); + LOG.warn("symbol dict {} creation failed; proceeding without it", filePath, t); return null; } finally { if (hdr != 0L) { @@ -691,19 +693,107 @@ private static PersistedSymbolDict openFresh(FilesFacade ff, String filePath) { return new PersistedSymbolDict(ff, fd, HEADER_SIZE, 0, 0L, 0); } - private void ensureScratch(int required) { + /** + * Grows the append scratch buffer to hold at least {@code required} bytes. + *

    + * Throws when {@code required} exceeds {@link #MAX_SCRATCH_BYTES} rather than + * clamping to it: a clamp would hand back a buffer SMALLER than the caller asked + * for and return normally, and every caller then writes {@code required} bytes + * into it -- turning a clean out-of-memory into a silent native-heap overflow, on + * the very write path the dictionary's integrity rests on. This is the same + * loud-failure shape {@code CursorWebSocketSendLoop.ensureSentDictCapacity} uses + * on the same bound. Unreachable at any realistic cardinality (it needs a ~2 GiB + * dictionary section in a single frame, itself bounded by the server's batch + * cap), but a size guard on a data-integrity write path must never + * under-allocate. + */ + private void ensureScratch(long required) { if (scratchCap >= required) { return; } + if (required > MAX_SCRATCH_BYTES) { + throw new IllegalStateException("symbol dict scratch buffer exceeds the maximum size to " + + FILE_NAME + " [required=" + required + ", max=" + MAX_SCRATCH_BYTES + ']'); + } // Double in long: scratchCap * 2 as an int overflows negative past ~1 GB and - // would make the realloc size negative. required is bounded by one frame's - // entries (the server batch cap), so this never actually caps -- it mirrors the - // long-math growth in CursorWebSocketSendLoop.ensureSentDictCapacity. + // would make the realloc size negative. long newCap = Math.max(required, Math.max(256L, (long) scratchCap * 2)); - if (newCap > Integer.MAX_VALUE - 8) { - newCap = Integer.MAX_VALUE - 8; + if (newCap > MAX_SCRATCH_BYTES) { + newCap = MAX_SCRATCH_BYTES; } scratchAddr = Unsafe.realloc(scratchAddr, scratchCap, (int) newCap, MemoryTag.NATIVE_DEFAULT); scratchCap = (int) newCap; } + + /** + * Checksums {@code [recStart, recStart + hdrLen + entriesLen)} in ONE native + * call, appends the CRC, and issues ONE positioned write. Advances + * {@code size}/{@code appendOffset} only on a complete write, so a short write + * throws and a retry keyed off {@link #size()} re-persists the same range at the + * same offset. + */ + private void flushChunk(long recStart, int hdrLen, int entriesLen, int count) { + int bodyLen = hdrLen + entriesLen; + int recLen = bodyLen + CRC_SIZE; + Unsafe.getUnsafe().putInt(recStart + bodyLen, Crc32c.update(Crc32c.INIT, recStart, bodyLen)); + long written = ff.write(fd, recStart, recLen, appendOffset); + if (written != recLen) { + throw new IllegalStateException("short write to " + FILE_NAME + + " [expected=" + recLen + ", actual=" + written + ']'); + } + appendOffset += recLen; + size += count; + } + + /** + * Writes one chunk whose entry region is ALREADY encoded in scratch at offset + * {@link #MAX_CHUNK_HEADER_SIZE}. Back-fills the header immediately in front of + * the entries -- the header is at most {@code MAX_CHUNK_HEADER_SIZE} bytes, so it + * always fits the reserve -- leaving header, entries and CRC one contiguous run + * for a single checksum and a single write. + */ + private void writeChunkFromScratch(int entriesLen, int count) { + int hdrLen = NativeBufferWriter.varintSize(count) + NativeBufferWriter.varintSize(entriesLen); + long recStart = scratchAddr + MAX_CHUNK_HEADER_SIZE - hdrLen; + long p = NativeBufferWriter.writeVarint(recStart, count); + NativeBufferWriter.writeVarint(p, entriesLen); + flushChunk(recStart, hdrLen, entriesLen, count); + } + + /** + * Zero-allocation LEB128 decoder, one instance per {@link #openExisting} call -- + * not one per chunk. The previous {@code long[]}-returning decoder allocated once + * per ENTRY, so a million-symbol recovery churned a million throwaway arrays in a + * class that is otherwise strictly allocation-free. + */ + private static final class Varint { + int end; + long value; + + /** + * Decodes the varint at {@code buf[pos..limit)}. Returns false -- leaving + * {@code value}/{@code end} undefined -- when the varint is truncated (a torn + * tail) or runs longer than a canonical 5-byte length. + */ + boolean decode(long buf, int pos, int limit) { + long v = 0; + int shift = 0; + int cur = pos; + while (cur < limit) { + byte b = Unsafe.getUnsafe().getByte(buf + cur); + cur++; + v |= (long) (b & 0x7F) << shift; + if ((b & 0x80) == 0) { + value = v; + end = cur; + return true; + } + shift += 7; + if (shift > 35) { + return false; // implausible for a chunk header; treat as torn + } + } + return false; + } + } } diff --git a/core/src/main/java/io/questdb/client/impl/ConfigSchema.java b/core/src/main/java/io/questdb/client/impl/ConfigSchema.java index a255e638..7097f3cd 100644 --- a/core/src/main/java/io/questdb/client/impl/ConfigSchema.java +++ b/core/src/main/java/io/questdb/client/impl/ConfigSchema.java @@ -77,6 +77,7 @@ public final class ConfigSchema { // (one vocabulary, side-owned application). str("max_frame_rejections", Side.INGRESS); str("poison_min_escalation_window_millis", Side.INGRESS); + str("catchup_cap_gap_min_escalation_window_millis", Side.INGRESS); str("max_name_len", Side.INGRESS); str("reconnect_initial_backoff_millis", Side.INGRESS); str("reconnect_max_backoff_millis", Side.INGRESS); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictCatchUpTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictCatchUpTest.java index 132f2c06..db7603f2 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictCatchUpTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictCatchUpTest.java @@ -238,8 +238,17 @@ public void testCatchUpEntryTooLargeForCapFailsTerminally() throws Exception { String bigSymbol = TestUtils.repeat("x", 200); // ~202-byte dict entry LineSenderException terminal = null; + // catchup_cap_gap_min_escalation_window_millis=0 restores count-only + // escalation, so the terminal latches as soon as the settle budget's + // strikes are exhausted. The DEFAULT is a 5-minute wall-clock dwell on + // top of the strike count, precisely so a cap gap that lasts only as + // long as a rolling restart cannot hard-fail a live producer -- and a + // test cannot wait that out. Zeroing it here is what makes the terminal + // observable; the dwell itself is covered by + // CursorWebSocketSendLoopCatchUpAlignmentTest. Sender sender = Sender.fromConfig("ws::addr=localhost:" + port - + ";reconnect_initial_backoff_millis=10;reconnect_max_backoff_millis=50;"); + + ";reconnect_initial_backoff_millis=10;reconnect_max_backoff_millis=50" + + ";catchup_cap_gap_min_escalation_window_millis=0;"); try { sender.table("t").symbol("s", bigSymbol).longColumn("v", 1L).atNow(); sender.flush(); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java index 49411275..707ef5a8 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java @@ -40,6 +40,7 @@ import java.lang.reflect.Field; import java.lang.reflect.InvocationTargetException; +import java.util.concurrent.TimeUnit; import java.lang.reflect.Method; import java.nio.charset.StandardCharsets; import java.util.ArrayList; @@ -266,6 +267,112 @@ public void testCatchUpChunkFrameSizeOverflowFailsLoud() throws Exception { }); } + @Test + public void testCatchUpCapGapStrikesAloneDoNotLatchWithinTheEscalationWindow() throws Exception { + // The strike count alone must NOT latch a terminal: escalation also requires the + // cap-gap episode to have persisted for catchUpCapGapMinEscalationWindowMillis. + // + // This is what keeps a routine rolling restart from killing a live producer. + // MAX_CATCHUP_CAP_GAP_ATTEMPTS strikes accrue in ~2 minutes at the capped + // reconnect backoff -- less than the time the larger-cap node is away -- so a + // count-only budget would hard-fail the sender on the very transient the budget + // exists to ride out. Here we drive far MORE than the budget's strikes inside a + // (deliberately huge) window and assert the sender stays alive. + TestUtils.assertMemoryLeak(() -> { + Field maxField = CursorWebSocketSendLoop.class.getDeclaredField("MAX_CATCHUP_CAP_GAP_ATTEMPTS"); + maxField.setAccessible(true); + int maxAttempts = maxField.getInt(null); + CatchUpCapturingClient client = new CatchUpCapturingClient(160); + try (CursorSendEngine engine = newEngine()) { + // A one-hour dwell the test cannot possibly elapse. + CursorWebSocketSendLoop loop = newLoop(engine, client, 3_600_000L); + try { + seedMirror(loop, TestUtils.repeat("x", 200)); + for (int i = 1; i <= maxAttempts + 4; i++) { + try { + invokeSetWireBaselineWithCatchUp(loop, engine.ackedFsn() + 1L); + fail("cap gap must raise a retriable CatchUpSendException (attempt " + i + ')'); + } catch (InvocationTargetException e) { + assertEquals("CatchUpSendException", e.getCause().getClass().getSimpleName()); + } + // The producer-facing latch must stay clear on EVERY attempt, + // including the ones past the strike budget. + loop.checkError(); + } + assertTrue("the strikes really did exceed the budget", + readInt(loop, "catchUpCapGapAttempts") > maxAttempts); + + // Backdate the episode anchor past the window: the very next cap gap + // now satisfies BOTH conditions and latches. This pins the AND -- if + // escalation ignored the wall clock the loop would already have + // latched above; if it ignored the strike count it could never latch. + Field anchor = CursorWebSocketSendLoop.class.getDeclaredField("catchUpCapGapFirstNanos"); + anchor.setAccessible(true); + anchor.setLong(loop, System.nanoTime() - TimeUnit.HOURS.toNanos(2)); + try { + invokeSetWireBaselineWithCatchUp(loop, engine.ackedFsn() + 1L); + fail("the escalating cap gap must still raise CatchUpSendException"); + } catch (InvocationTargetException e) { + assertEquals("CatchUpSendException", e.getCause().getClass().getSimpleName()); + } + try { + loop.checkError(); + fail("a cap gap that outlives the escalation window must latch a terminal"); + } catch (LineSenderException terminal) { + assertTrue("terminal must name the exhausted catch-up cap gap: " + terminal.getMessage(), + terminal.getMessage().contains("during catch-up") + && terminal.getMessage().contains("must be resent")); + } + } finally { + loop.close(); + } + } + }); + } + + @Test + public void testTransientCatchUpFailureDoesNotBurnTheCapGapBudget() throws Exception { + // A TRANSIENT catch-up failure (the wire drops mid-catch-up -- a flapping LB, a + // reset) must never increment the cap-gap terminal budget. The budget exists to + // prove a PERSISTENT cluster capability gap; letting a transient feed it means + // enough wire flaps hard-fail a live store-and-forward producer, which is the + // exact failure store-and-forward promises cannot happen. + // + // The production code is correct, but nothing pinned it: the counter is never + // read by the existing transient test, and one transient can never reach a + // 16-strike budget anyway. So drive MORE transients than the whole budget and + // assert the counter never moves and no terminal ever latches. + TestUtils.assertMemoryLeak(() -> { + Field maxField = CursorWebSocketSendLoop.class.getDeclaredField("MAX_CATCHUP_CAP_GAP_ATTEMPTS"); + maxField.setAccessible(true); + int maxAttempts = maxField.getInt(null); + // A cap that FITS (no cap gap), but whose sendBinary always throws: every + // failure here is transport-transient, never a capability gap. + CatchUpCapturingClient client = new CatchUpCapturingClient(0, true); + try (CursorSendEngine engine = newEngine()) { + CursorWebSocketSendLoop loop = newLoop(engine, client, 0L); + try { + seedMirror(loop, "alpha"); + for (int i = 1; i <= maxAttempts + 4; i++) { + try { + invokeSetWireBaselineWithCatchUp(loop, engine.ackedFsn() + 1L); + fail("a transient send failure must raise CatchUpSendException (attempt " + i + ')'); + } catch (InvocationTargetException e) { + assertEquals("CatchUpSendException", e.getCause().getClass().getSimpleName()); + } + loop.checkError(); // a transient is retriable, forever + assertEquals("a transient must NOT burn the cap-gap terminal budget", + 0, readInt(loop, "catchUpCapGapAttempts")); + assertEquals("a transient must NOT anchor a cap-gap episode", + -1L, readLong(loop, "catchUpCapGapFirstNanos")); + } + } finally { + loop.close(); + } + } + }); + } + @Test public void testCatchUpCapGapRetriesUntilBudgetThenLatches() throws Exception { // M1: an entry too large for the fresh server's cap during catch-up (a @@ -539,12 +646,27 @@ private static void invokeSetWireBaselineWithCatchUp(CursorWebSocketSendLoop loo } private CursorWebSocketSendLoop newLoop(CursorSendEngine engine, WebSocketClient client) { + return newLoop(engine, client, 0L); + } + + /** + * As {@link #newLoop(CursorSendEngine, WebSocketClient)} but with an explicit + * cap-gap escalation dwell. {@code 0} (the default here) means count-only + * escalation, which is what the raw constructor gives and what the strike-count + * tests want; the user-facing 5-minute default is applied at the config layer. + */ + private CursorWebSocketSendLoop newLoop( + CursorSendEngine engine, WebSocketClient client, long capGapWindowMillis + ) { return new CursorWebSocketSendLoop( client, engine, 0L, CursorWebSocketSendLoop.DEFAULT_PARK_NANOS, () -> { throw new UnsupportedOperationException("test loop is never started"); }, - 5_000L, 100L, 5_000L, false); + 5_000L, 100L, 5_000L, false, + CursorWebSocketSendLoop.DEFAULT_DURABLE_ACK_KEEPALIVE_INTERVAL_MILLIS, + CursorWebSocketSendLoop.DEFAULT_MAX_HEAD_FRAME_REJECTIONS, + 0L, capGapWindowMillis); } private CursorSendEngine newEngine() { diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java index 677e12e2..b1255aac 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java @@ -41,6 +41,11 @@ public class PersistedSymbolDictTest { + // On-disk geometry, mirrored from PersistedSymbolDict so the layout-derived + // corruption tests below read as arithmetic rather than magic numbers. + private static final int CRC_SIZE = 4; + private static final int HEADER_SIZE = 8; + @Test public void testAppendPersistsAcrossReopen() throws Exception { assertMemoryLeak(() -> { @@ -331,22 +336,88 @@ public void testAppendSymbolsShortWriteThrowsWithoutAdvancingIsIdempotentOnRetry } @Test - public void testBadMagicIsRecreatedEmpty() throws Exception { + public void testBatchAppendWritesOneChunkWithOneChecksum() throws Exception { + // A batch append must write ONE chunk -- one [entryCount][entryBytes] header and + // ONE trailing CRC-32C for the whole range -- not one checksummed record per + // symbol. + // + // This is the load-bearing property of the v3 layout, and it is a producer-thread + // cost, not a cosmetic one: Crc32c.update is a NATIVE call, so a per-entry + // checksum put one JNI transition (plus one sub-cache-line copy and one redundant + // varint decode) on the flush path for every new symbol. On the high-cardinality + // batch this whole feature exists to serve -- one new symbol per row -- that is a + // thousand native calls per flush where the chunk needs one. + // + // Asserted through the file size, which is exact and cheap: N single-byte ASCII + // symbols cost N * ([len varint] + 1 utf8 byte) of entries, wrapped in one chunk. + // Per-entry CRCs would add 4 * N bytes instead of 4. assertMemoryLeak(() -> { Path dir = Files.createTempDirectory("qwp-symdict"); try { - // A file with the right size but garbage content (bad magic). - Path f = dir.resolve(".symbol-dict"); - Files.write(f, new byte[]{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}); + final int n = 26; // 'a'..'z' -- distinct, and one UTF-8 byte each + GlobalSymbolDictionary dict = new GlobalSymbolDictionary(); + for (int i = 0; i < n; i++) { + dict.getOrAddSymbol(String.valueOf((char) ('a' + i))); + } + Assert.assertEquals(n, dict.size()); PersistedSymbolDict d = PersistedSymbolDict.open(dir.toString()); Assert.assertNotNull(d); try { - Assert.assertEquals("bad-magic file recreated empty", 0, d.size()); - d.appendSymbol("X"); - Assert.assertEquals(1, d.size()); + d.appendSymbols(dict, 0, n - 1); + Assert.assertEquals(n, d.size()); } finally { d.close(); } + + int entryBytes = n * 2; // [len=1 varint][1 utf8 byte] + int chunkHeader = 1 + varintSize(entryBytes); // entryCount=50 fits one byte + long expected = HEADER_SIZE + chunkHeader + entryBytes + CRC_SIZE; + Assert.assertEquals( + "a batch append must cost ONE chunk checksum, not one per symbol", + expected, Files.size(dir.resolve(".symbol-dict"))); + + // And it must still read back symbol-for-symbol. + PersistedSymbolDict re = PersistedSymbolDict.open(dir.toString()); + Assert.assertNotNull(re); + try { + Assert.assertEquals(n, re.size()); + ObjList got = re.readLoadedSymbols(); + for (int i = 0; i < n; i++) { + Assert.assertEquals(dict.getSymbol(i), got.getQuick(i)); + } + } finally { + re.close(); + } + } finally { + rmDir(dir); + } + }); + } + + private static int varintSize(int v) { + int n = 1; + while ((v >>>= 7) != 0) { + n++; + } + return n; + } + + @Test + public void testBadMagicDegradesWithoutDestroyingTheFile() throws Exception { + // An unparseable existing file must degrade to null -- the sender falls back + // to full self-sufficient frames -- and must NOT be recreated. open() used to + // fall through to openFresh(), which is O_TRUNC: a single unreadable byte + // destroyed the only copy of load-bearing state. + assertMemoryLeak(() -> { + Path dir = Files.createTempDirectory("qwp-symdict"); + try { + Path f = dir.resolve(".symbol-dict"); + byte[] original = new byte[]{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; + Files.write(f, original); + Assert.assertNull("an unparseable existing dict must degrade to null", + PersistedSymbolDict.open(dir.toString())); + Assert.assertArrayEquals("open() must NOT destroy an existing dictionary", + original, Files.readAllBytes(f)); } finally { rmDir(dir); } @@ -354,36 +425,35 @@ public void testBadMagicIsRecreatedEmpty() throws Exception { } @Test - public void testBadVersionIsRecreatedEmpty() throws Exception { - // A file with correct 'SYD1' magic and a VALID entry but an unknown version - // byte must be recreated empty -- its entries belong to a foreign/future - // format and must not be parsed as v2. Covers the version sub-condition - // specifically (testBadMagicIsRecreatedEmpty covers the magic one). Because - // the seeded "a" is a valid v2 entry, without the version check it would parse - // back (size 1), so this fails. + public void testBadVersionDegradesWithoutDestroyingTheFile() throws Exception { + // A file with correct 'SYD1' magic and a VALID chunk but an unknown version + // byte belongs to a foreign/future format and must not be parsed as v3. + // Covers the version sub-condition specifically (testBadMagicDegrades... covers + // the magic one). + // + // The file must SURVIVE. This is the client-rollback trap: a newer client + // writes v4, ops roll back to this build, and if open() recreated the file the + // v4 dictionary would be gone for good -- rolling forward again could not + // recover it, and every surviving delta frame would be permanently + // unreplayable. Degrading to null instead leaves the bytes for the client that + // does understand them. assertMemoryLeak(() -> { Path dir = Files.createTempDirectory("qwp-symdict"); try { Path f = dir.resolve(".symbol-dict"); - // Build a real v2 dictionary with one valid entry... PersistedSymbolDict seed = PersistedSymbolDict.open(dir.toString()); Assert.assertNotNull(seed); seed.appendSymbol("a"); seed.close(); - // ...then corrupt ONLY the version byte (offset 4) to an unknown value. + // Corrupt ONLY the version byte (offset 4) to an unknown value. byte[] bytes = Files.readAllBytes(f); bytes[4] = (byte) 99; Files.write(f, bytes); - PersistedSymbolDict d = PersistedSymbolDict.open(dir.toString()); - Assert.assertNotNull(d); - try { - Assert.assertEquals("bad-version file recreated empty (entries must not parse)", 0, d.size()); - d.appendSymbol("X"); - Assert.assertEquals(1, d.size()); - } finally { - d.close(); - } + Assert.assertNull("an unknown-version dict must degrade to null", + PersistedSymbolDict.open(dir.toString())); + Assert.assertArrayEquals("open() must NOT destroy a future-version dictionary", + bytes, Files.readAllBytes(f)); } finally { rmDir(dir); } @@ -446,11 +516,11 @@ public void testEmptySymbolRoundTrips() throws Exception { @Test public void testInteriorCorruptionIsCaughtNotSilentlyMisattributed() throws Exception { // A host-crash interior tear (a lost page reading back as zeroes) or a - // stale entry left past the end by a failed truncate can change the bytes - // of a NON-trailing entry. Without the per-entry CRC the parse would + // stale chunk left past the end by a failed truncate can change the bytes + // of a NON-trailing chunk. Without the per-chunk CRC the parse would // accept those bytes, shifting the dense id->symbol map and silently // misattributing symbol-column values on replay. With the CRC the corrupt - // entry fails verification and the parse stops there, so recovery trusts + // chunk fails verification and the parse stops there, so recovery trusts // only the intact prefix (fail-clean: the send loop's torn-dict guard then // forces a resend of the rest). assertMemoryLeak(() -> { @@ -468,25 +538,27 @@ public void testInteriorCorruptionIsCaughtNotSilentlyMisattributed() throws Exce d.close(); } - // Corrupt one byte inside the 3rd entry's UTF-8 (id 2). On-disk - // entry layout is [len varint][utf8][crc32c u32]; a 2-byte ASCII - // symbol is 1 + 2 + 4 = 7 bytes, after the 8-byte header: - // header[0,8) e0[8,15) e1[15,22) e2[22,29) ... - // Offset 23 is "s2"'s first UTF-8 byte; flipping it leaves e2's - // stored CRC stale. + // Corrupt one byte inside the 3rd chunk's entry region (id 2). Each + // appendSymbol writes one single-entry chunk + // ([entryCount][entryBytes][len][utf8][crc32c]), and all five symbols + // are the same width, so the five chunks are equal-sized -- derive the + // stride from the file rather than hard-coding it. The byte flipped is + // the last one before the chunk's trailing CRC, so that CRC goes stale. Path f = dir.resolve(".symbol-dict"); byte[] bytes = Files.readAllBytes(f); - bytes[23] ^= 0x7F; + int chunkLen = (bytes.length - HEADER_SIZE) / 5; + int chunk2 = HEADER_SIZE + 2 * chunkLen; + bytes[chunk2 + chunkLen - CRC_SIZE - 1] ^= 0x7F; Files.write(f, bytes); PersistedSymbolDict re = PersistedSymbolDict.open(dir.toString()); Assert.assertNotNull(re); try { - // Only the intact prefix [s0, s1] is trusted; the corrupt e2 + // Only the intact prefix [s0, s1] is trusted; the corrupt chunk // and everything after it are dropped. No recovered symbol is // the corrupted string -- the tear is DETECTED, never silently // misattributed. - Assert.assertEquals("parse must stop at the corrupt interior entry", 2, re.size()); + Assert.assertEquals("parse must stop at the corrupt interior chunk", 2, re.size()); ObjList s = re.readLoadedSymbols(); Assert.assertEquals("s0", s.getQuick(0)); Assert.assertEquals("s1", s.getQuick(1)); @@ -611,31 +683,40 @@ public void testRemoveOrphanDeletesFile() throws Exception { } @Test - public void testTooLargeToReopenRecreatesEmpty() throws Exception { - // A dictionary that legitimately grew past Integer.MAX_VALUE cannot be read - // into one int-sized buffer; open() must recreate it empty (fail-clean, like - // every other unreadable-file case) rather than truncate the multi-GB file - // via a negative/zero (int) length cast. A fault facade reports the huge - // length without needing a real 2GB file on disk. + public void testTooLargeToReopenDegradesWithoutOpeningTheFile() throws Exception { + // A dictionary at or past Integer.MAX_VALUE cannot be read into one int-sized + // buffer, so open() must short-circuit to null BEFORE touching the file. + // + // The facade reports 2^32 + 100 -- deliberately NOT Integer.MAX_VALUE + 1. + // That value casts to Integer.MIN_VALUE, which Unsafe.malloc rejects anyway, + // so openExisting's catch would produce the same null and the test could not + // tell the guard from its absence (it was exactly that vacuous before). A + // length of 2^32 + k instead casts to a SMALL POSITIVE prefix -- the branch + // the guard's own javadoc calls out -- under which openExisting would happily + // malloc, read a truncated prefix, and parse it. + // + // The assertion that pins the guard is therefore NOT the null (both paths give + // null) but that the file is never even OPENED: with the guard, open() returns + // before openRW; without it, openExisting opens the file and parses garbage. assertMemoryLeak(() -> { Path dir = Files.createTempDirectory("qwp-symdict"); try { - // Seed a small real file so exists() is true and the facade's huge - // length() drives the > Integer.MAX_VALUE reopen guard. PersistedSymbolDict seed = PersistedSymbolDict.open(dir.toString()); Assert.assertNotNull(seed); - seed.appendSymbol("a"); - seed.close(); - - PersistedSymbolDict d = PersistedSymbolDict.open(new HugeLengthFacade(), dir.toString()); - Assert.assertNotNull(d); - try { - Assert.assertEquals("a >2GB dictionary must recreate empty", 0, d.size()); - d.appendSymbol("X"); - Assert.assertEquals(1, d.size()); - } finally { - d.close(); + for (int i = 0; i < 40; i++) { + seed.appendSymbol("sym" + i); } + seed.close(); + Path f = dir.resolve(".symbol-dict"); + byte[] before = Files.readAllBytes(f); + + HugeLengthFacade ff = new HugeLengthFacade(); + Assert.assertNull("a >=2GB dictionary must degrade to null", + PersistedSymbolDict.open(ff, dir.toString())); + Assert.assertEquals("the guard must short-circuit BEFORE opening the file", + 0, ff.openRwCalls); + Assert.assertArrayEquals("open() must NOT destroy or trim an oversized dictionary", + before, Files.readAllBytes(f)); } finally { rmDir(dir); } @@ -693,15 +774,19 @@ public void testTornTrailingEntrySelfHeals() throws Exception { } @Test - public void testTruncateFailureRecreatesEmpty() throws Exception { - // A host crash can leave a torn/stale tail past the last complete entry. + public void testTruncateFailureDegradesWithoutDestroyingTheFile() throws Exception { + // A host crash can leave a torn/stale tail past the last complete chunk. // open() drops it with a truncate; if that truncate FAILS (a read-only // remount, a Windows share lock), the file still exposes the stale bytes, - // whose self-consistent per-entry CRC a later shifted parse could accept as - // a real symbol. So a failed truncate must make the file UNTRUSTED -- - // open() recreates it empty (fail-clean) rather than returning a dict laid - // over stale bytes. Drive the truncate failure with a facade and assert the - // reopened dictionary is empty, not the [one, two] prefix. + // whose self-consistent chunk CRC a later shifted parse could accept as a real + // symbol. So a failed truncate must make the file UNTRUSTED -- open() degrades + // to null (the sender falls back to full self-sufficient frames) rather than + // returning a dict laid over stale bytes. + // + // It must NOT recreate the file, which is what it used to do: a read-only + // remount is transient, and destroying the [one, two] prefix on the way past it + // makes every surviving delta frame permanently unreplayable. Degrading leaves + // the bytes for the next attempt, once the mount is writable again. assertMemoryLeak(() -> { Path dir = Files.createTempDirectory("qwp-symdict"); try { @@ -710,26 +795,31 @@ public void testTruncateFailureRecreatesEmpty() throws Exception { d.appendSymbol("two"); d.close(); - // Append a torn trailing record (length prefix 5, only 2 bytes) so - // the reopen parses [one, two], then finds validLen < len and tries - // to truncate the tail -- the branch under test. + // Append a torn trailing record so the reopen parses [one, two], then + // finds validLen < len and tries to truncate the tail -- the branch + // under test. Path f = dir.resolve(".symbol-dict"); long cleanLen = Files.size(f); // header + "one" + "two", no tail Files.write(f, new byte[]{(byte) 5, (byte) 'x', (byte) 'y'}, StandardOpenOption.APPEND); Assert.assertEquals("torn tail present before reopen", cleanLen + 3, Files.size(f)); + byte[] before = Files.readAllBytes(f); // Reopen through a facade whose truncate() fails. - PersistedSymbolDict re = PersistedSymbolDict.open(new FailingTruncateFacade(), dir.toString()); + Assert.assertNull("a dict whose torn tail cannot be trimmed must degrade to null", + PersistedSymbolDict.open(new FailingTruncateFacade(), dir.toString())); + Assert.assertArrayEquals("a failed truncate must NOT destroy the dictionary", + before, Files.readAllBytes(f)); + + // And once the filesystem is writable again, the SAME file recovers its + // intact prefix in full -- which is the whole point of not destroying it. + PersistedSymbolDict re = PersistedSymbolDict.open(dir.toString()); Assert.assertNotNull(re); try { - // The failed truncate made the file untrusted, so open() recreated - // it empty rather than trusting the [one, two] prefix over a stale - // tail: size()==0, not 2, and no recovered symbols. - Assert.assertEquals("failed truncate must recreate the dictionary empty", 0, re.size()); - Assert.assertEquals(0, re.readLoadedSymbols().size()); - // openFresh rewrote a bare 8-byte header (magic + version), so both - // the two entries and the torn tail are gone. - Assert.assertEquals("recreated file is a bare header", 8L, Files.size(f)); + Assert.assertEquals("the intact prefix survives the transient", 2, re.size()); + ObjList s = re.readLoadedSymbols(); + Assert.assertEquals("one", s.getQuick(0)); + Assert.assertEquals("two", s.getQuick(1)); + Assert.assertEquals("the torn tail is trimmed once truncate works", cleanLen, Files.size(f)); } finally { re.close(); } @@ -894,12 +984,28 @@ public boolean truncate(int fd, long size) { /** * Reports a dictionary length past {@link Integer#MAX_VALUE} -- reproducing a * dictionary that legitimately grew beyond 2GB, which {@code open()} cannot read - * into one int-sized buffer and must recreate empty. + * into one int-sized buffer and must refuse before touching the file. + *

    + * The length is {@code 2^32 + 100}, NOT {@code Integer.MAX_VALUE + 1}. The latter + * casts to {@code Integer.MIN_VALUE}, which {@code Unsafe.malloc} rejects on its + * own, so {@code openExisting}'s catch would produce the same {@code null} with or + * without the guard -- which is precisely what made the old version of this test + * vacuous. {@code 2^32 + k} casts to a small POSITIVE prefix instead, so without + * the guard {@code openExisting} really would open the file and parse a truncated + * prefix of it. {@link #openRwCalls} is what catches that. */ private static final class HugeLengthFacade extends DelegatingFilesFacade { + int openRwCalls; + @Override public long length(String path) { - return (long) Integer.MAX_VALUE + 1L; + return (1L << 32) + 100L; + } + + @Override + public int openRW(String path) { + openRwCalls++; + return super.openRW(path); } } From dd48d4336f9e8f3296dc28ab3e89308b10726c84 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Tue, 14 Jul 2026 16:17:22 +0100 Subject: [PATCH 56/96] Assert the new cap-gap config key is honored WsSenderConfigHonoredTest enforces that every key registered in ConfigSchema is actually plumbed through to the sender. The new catchup_cap_gap_min_escalation_window_millis key needed its assertion. --- .../io/questdb/client/test/impl/WsSenderConfigHonoredTest.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/core/src/test/java/io/questdb/client/test/impl/WsSenderConfigHonoredTest.java b/core/src/test/java/io/questdb/client/test/impl/WsSenderConfigHonoredTest.java index dba5765f..f6287825 100644 --- a/core/src/test/java/io/questdb/client/test/impl/WsSenderConfigHonoredTest.java +++ b/core/src/test/java/io/questdb/client/test/impl/WsSenderConfigHonoredTest.java @@ -75,6 +75,8 @@ public void testEveryIngressKeyIsHonored() { assertHonored("max_background_drainers=6", "max_background_drainers", 6); assertHonored("max_frame_rejections=6", "max_frame_rejections", 6); assertHonored("poison_min_escalation_window_millis=7500", "poison_min_escalation_window_millis", 7500L); + assertHonored("catchup_cap_gap_min_escalation_window_millis=90000", + "catchup_cap_gap_min_escalation_window_millis", 90000L); assertHonored("error_inbox_capacity=128", "error_inbox_capacity", 128); assertHonored("connection_listener_inbox_capacity=64", "connection_listener_inbox_capacity", 64); assertHonored("token=ey.abc", "token", "ey.abc"); From 521b48d85da397c8bfb33cded799bd8ae4e5c08a Mon Sep 17 00:00:00 2001 From: glasstiger Date: Tue, 14 Jul 2026 16:46:27 +0100 Subject: [PATCH 57/96] Close the untested gaps around the symbol-dict guards Four behaviours the delta symbol-dictionary work relies on had no test that could fail if they regressed. Each mutation below leaves the whole suite green today. Pin the split path's write-ahead ordering. Moving the persist after the publish in flushPendingRowsSplit left every test passing, because the only split-path test checked the dictionary after a SUCCESSFUL flush -- where the symbols land either way. Only a publish that FAILS while the symbols are new can tell the orderings apart, so the new test caps the server batch size to force the split and shrinks the segment so the split frame cannot be published. With the ordering broken, a process crash in that window would leave a recorded frame whose deltaStart exceeds the recovered dictionary, and recovery would quarantine a slot that should have drained cleanly. Pin the persist-failure translation. A short write to .symbol-dict -- a full disk, an exhausted quota -- throws a low-level IllegalStateException that persistNewSymbolsBeforePublish wraps in a LineSenderException. Deleting the wrap left the suite green, yet the raw exception would sail past every caller's catch around flush(). Nothing could reach that path: PersistedSymbolDict has facade-aware overloads but CursorSendEngine called only the FilesFacade.INSTANCE ones, so the engine now takes a FilesFacade and the test drives a real short write through the real producer path. Pin the sealed-segment walk. Deleting the sealed half of SegmentRing.maxSymbolDeltaEnd left the suite green because every test fits its slot in one active segment. It is not a benign wrong answer: recoveredMaxSymbolId collapses to -1, and the seed guard fires on `>= pd.size()`, so a value that is too low never fires at all -- a torn dictionary is trusted and the producer silently reuses ids the surviving frames already define. The walk earns its keep exactly when the active segment holds nothing but an aborted deferred tail, which is the crash the guard was built for, so the test reproduces that shape. Hoist DelegatingFilesFacade into test/tools so a fault seam can be shared rather than stamped into each test that needs one. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../client/sf/cursor/CursorSendEngine.java | 33 +++- .../qwp/client/DeltaDictRecoveryTest.java | 71 ++++++++ .../qwp/client/SelfSufficientFramesTest.java | 85 +++++++++ .../sf/cursor/CursorSendEngineTest.java | 6 +- ...CursorWebSocketSendLoopOrphanTailTest.java | 65 +++++++ .../sf/cursor/PersistedSymbolDictTest.java | 136 +------------- .../test/tools/DelegatingFilesFacade.java | 170 ++++++++++++++++++ 7 files changed, 424 insertions(+), 142 deletions(-) create mode 100644 core/src/test/java/io/questdb/client/test/tools/DelegatingFilesFacade.java diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java index a2e20d35..ffd964f3 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java @@ -27,8 +27,10 @@ import io.questdb.client.cutlass.qwp.protocol.QwpConstants; import io.questdb.client.std.Compat; import io.questdb.client.std.Files; +import io.questdb.client.std.FilesFacade; import io.questdb.client.std.ObjList; import io.questdb.client.std.QuietCloseable; +import org.jetbrains.annotations.TestOnly; import java.util.concurrent.locks.LockSupport; @@ -154,9 +156,29 @@ public CursorSendEngine(String sfDir, long segmentSizeBytes) { */ public CursorSendEngine(String sfDir, long segmentSizeBytes, long maxTotalBytes, long appendDeadlineNanos) { + this(sfDir, segmentSizeBytes, maxTotalBytes, appendDeadlineNanos, FilesFacade.INSTANCE); + } + + /** + * As {@link #CursorSendEngine(String, long, long, long)}, but with an explicit + * {@link FilesFacade} for the persisted symbol dictionary. + *

    + * The seam exists so a test can drive a dictionary I/O fault -- a short write from + * a full disk or an exhausted quota -- through the REAL producer path + * ({@code flush()} -> the write-ahead persist), and assert it surfaces as a + * {@code LineSenderException} like every other flush-path failure rather than as a + * raw {@code IllegalStateException} that would sail past every caller's + * {@code catch (LineSenderException)}. Nothing else could reach that translation: + * {@code PersistedSymbolDict} has facade-aware overloads, but the engine used to + * call only the {@code FilesFacade.INSTANCE} ones, so no fault could be injected + * from outside. + */ + @TestOnly + public CursorSendEngine(String sfDir, long segmentSizeBytes, + long maxTotalBytes, long appendDeadlineNanos, FilesFacade dictFf) { this(sfDir, segmentSizeBytes, new SegmentManager(segmentSizeBytes, SegmentManager.DEFAULT_POLL_NANOS, maxTotalBytes), - true, appendDeadlineNanos); + true, appendDeadlineNanos, dictFf); } /** @@ -165,11 +187,12 @@ public CursorSendEngine(String sfDir, long segmentSizeBytes, * ownership of the manager. Uses the default append deadline. */ public CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager manager) { - this(sfDir, segmentSizeBytes, manager, false, DEFAULT_APPEND_DEADLINE_NANOS); + this(sfDir, segmentSizeBytes, manager, false, DEFAULT_APPEND_DEADLINE_NANOS, + FilesFacade.INSTANCE); } private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager manager, - boolean ownsManager, long appendDeadlineNanos) { + boolean ownsManager, long appendDeadlineNanos, FilesFacade dictFf) { // sfDir == null → memory-only mode (non-SF async ingest). Same // cursor architecture, no disk involvement; segments // live in malloc'd native memory. @@ -278,7 +301,7 @@ private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager man // Load the persisted symbol dictionary so delta-encoded frames // in this recovered slot can be re-registered on the fresh // server before replay. Null on open failure -> delta disabled. - persistedDictInProgress = PersistedSymbolDict.open(sfDir); + persistedDictInProgress = PersistedSymbolDict.open(dictFf, sfDir); long baseSeed = lowestBase - 1; long watermarkFsn = watermarkInProgress != null ? watermarkInProgress.read() @@ -389,7 +412,7 @@ private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager man // Windows share lock); if the clean open itself fails, // persistedSymbolDict stays null and the sender falls back to full // self-sufficient frames, which is also safe. - persistedDictInProgress = PersistedSymbolDict.openClean(sfDir); + persistedDictInProgress = PersistedSymbolDict.openClean(dictFf, sfDir); } MmapSegment initial; String initialPath = null; diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java index 31153408..4be99883 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java @@ -26,6 +26,9 @@ import io.questdb.client.Sender; import io.questdb.client.cutlass.line.LineSenderException; +import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender; +import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine; +import io.questdb.client.test.tools.DelegatingFilesFacade; import io.questdb.client.cutlass.qwp.client.sf.cursor.PersistedSymbolDict; import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer; import io.questdb.client.test.tools.TestUtils; @@ -580,6 +583,56 @@ public void testCommitMessageDoesNotShipUnpersistedLeakedSymbol() throws Excepti }); } + @Test + public void testPersistFailureSurfacesAsLineSenderException() throws Exception { + // A dictionary write that fails mid-flush -- a full disk, an exhausted quota -- + // must reach the caller as a LineSenderException, like every other flush-path + // failure. PersistedSymbolDict throws a low-level IllegalStateException on a short + // write; persistNewSymbolsBeforePublish wraps it. Without the wrap the raw + // IllegalStateException sails straight past every user's + // `catch (LineSenderException)` around flush() and takes the application down. + // + // Nothing could reach that translation before: PersistedSymbolDict has + // facade-aware overloads, but CursorSendEngine called only the + // FilesFacade.INSTANCE ones, so no test could inject a dictionary I/O fault + // through the real producer path. The engine now takes a FilesFacade for exactly + // this, and the sender is built on that engine directly -- the same + // QwpWebSocketSender.connect(..., CursorSendEngine) entry point Sender.build uses. + assertMemoryLeak(() -> { + try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) { + int port = silent.getPort(); + silent.start(); + Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS)); + + String slot = Paths.get(sfDir, "default").toString(); + Assert.assertEquals(0, io.questdb.client.std.Files.mkdir(sfDir, + io.questdb.client.std.Files.DIR_MODE_DEFAULT)); + ShortDictWriteFacade ff = new ShortDictWriteFacade(); + // The engine owns the dictionary; the fault facade reaches only it, so the + // segment files still write normally and the ONLY failure is the persist. + CursorSendEngine engine = new CursorSendEngine( + slot, 4L * 1024 * 1024, CursorSendEngine.DEFAULT_APPEND_DEADLINE_NANOS, + CursorSendEngine.DEFAULT_APPEND_DEADLINE_NANOS, ff); + try (Sender sender = QwpWebSocketSender.connect( + "localhost", port, null, 0, 0, 0L, null, false, engine)) { + ff.armed = true; // the next dictionary append short-writes + sender.table("m").symbol("s", "boom").longColumn("v", 1L).atNow(); + try { + sender.flush(); + Assert.fail("a short write to the symbol dictionary must fail the flush"); + } catch (LineSenderException expected) { + Assert.assertTrue("the persist failure must be reported as a sender error, " + + "not leak a raw IllegalStateException: " + expected.getMessage(), + expected.getMessage().contains("failed to persist symbol dictionary before publish")); + } + } catch (LineSenderException ignored) { + // close() re-flushes the still-buffered row; the facade has disarmed, so + // this normally succeeds. Either way it is not what we assert. + } + } + }); + } + @Test public void testFailedPublishDoesNotDuplicatePersistedSymbols() throws Exception { // Regression: persistNewSymbolsBeforePublish is a write-ahead -- it runs @@ -1256,4 +1309,22 @@ private static byte[] buildAck(long seq) { return buf; } } + /** + * Short-writes the FIRST dictionary append after it is armed, modelling a disk that + * fills mid-flush. Offset 0 is left alone so the file header still writes -- only the + * entry append fails, which is the path under test. + */ + private static final class ShortDictWriteFacade extends DelegatingFilesFacade { + boolean armed; + + @Override + public long write(int fd, long addr, long len, long offset) { + if (armed && offset > 0 && len > 1) { + armed = false; + return INSTANCE.write(fd, addr, len - 1, offset); + } + return INSTANCE.write(fd, addr, len, offset); + } + } + } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java index 5729aa44..c0d27e09 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java @@ -26,6 +26,8 @@ import io.questdb.client.Sender; import io.questdb.client.cutlass.line.LineSenderException; +import io.questdb.client.std.ObjList; +import io.questdb.client.cutlass.qwp.client.sf.cursor.PersistedSymbolDict; import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer; import io.questdb.client.test.tools.TestUtils; import org.junit.Assert; @@ -332,6 +334,89 @@ public void testOversizedTableSplitStrandsNothing() throws Exception { }); } + @Test + public void testFileModeSplitPersistsDictBeforeAFAILEDPublish() throws Exception { + // The SPLIT path's write-ahead ordering: persistNewSymbolsBeforePublish must run + // BEFORE sealAndSwapBuffer publishes the frame to the ring, exactly as it does on + // the non-split path. + // + // Its sibling below (testFileModeSplitPersistsDictBeforePublish) checks the dict + // after a SUCCESSFUL flush, which proves nothing about ORDER -- the symbols land + // in the file either way. Only a publish that FAILS while the symbols are new can + // tell the two apart: with the write-ahead intact the symbols are already durable + // when the publish throws; with the persist moved after the publish, the throw + // beats it and the dictionary is still empty. (The non-split path is pinned this + // way by DeltaDictRecoveryTest.testFailedPublishDoesNotDuplicatePersistedSymbols; + // the split path had no equivalent.) + // + // Why it matters: store-and-forward is process-crash durable. If a split frame + // reaches the ring before its symbols reach .symbol-dict, a JVM crash in that + // window leaves a recorded frame whose deltaStart exceeds the recovered + // dictionary -- so recovery declares the slot unreplayable and quarantines a slot + // that should have drained cleanly. + // + // Setup: the server cap (150) splits the two-table batch, and each split frame + // still FITS that cap, so the split pre-flight passes and the publish is actually + // attempted. sf_max_bytes=64 then makes every frame larger than the segment, so + // appendBlocking fails with PAYLOAD_TOO_LARGE -- deterministically, no + // backpressure timing needed. + Path sfDir = Files.createTempDirectory("qwp-sf-split-persist-fail"); + try { + assertMemoryLeak(() -> { + CapturingHandler handler = new CapturingHandler(); + try (TestWebSocketServer server = new TestWebSocketServer(handler)) { + server.setAdvertisedMaxBatchSize(150); // forces the two-table batch to split + int port = server.getPort(); + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + + String config = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + + ";sf_max_bytes=64;"; + String pad = TestUtils.repeat("x", 60); + Sender sender = Sender.fromConfig(config); + try { + sender.table("t1").symbol("s", "alpha").stringColumn("p", pad).longColumn("v", 1L).atNow(); + sender.table("t2").symbol("s", "bravo").stringColumn("p", pad).longColumn("v", 2L).atNow(); + try { + sender.flush(); + Assert.fail("a split frame larger than the segment must fail to publish"); + } catch (LineSenderException expected) { + // PAYLOAD_TOO_LARGE -- the publish, not the pre-flight. + } + Assert.assertEquals("the publish must fail, so no frame reaches the server", + 0, handler.batches.size()); + + // The write-ahead already ran: both of the batch's new symbols are + // durable even though the frame that references them never + // published. Move the persist after sealAndSwapBuffer and this is 0. + PersistedSymbolDict pd = PersistedSymbolDict.open( + sfDir.resolve("default").toString()); + Assert.assertNotNull(pd); + try { + Assert.assertEquals("the split path must persist its new symbols BEFORE " + + "publishing the frame that references them", 2, pd.size()); + ObjList symbols = pd.readLoadedSymbols(); + Assert.assertEquals("alpha", symbols.getQuick(0)); + Assert.assertEquals("bravo", symbols.getQuick(1)); + } finally { + pd.close(); + } + } finally { + try { + sender.close(); + } catch (LineSenderException ignored) { + // close() re-flushes the still-buffered oversized rows and fails + // again; expected, and not what this test asserts. close() still + // runs its resource cleanup, so no native memory leaks. + } + } + } + }); + } finally { + rmDir(sfDir); + } + } + @Test public void testFileModeSplitPersistsDictBeforePublish() throws Exception { // File-mode store-and-forward + a SPLIT flush: a two-table batch whose combined diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineTest.java index f4de1ff2..0e874ba5 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineTest.java @@ -31,6 +31,7 @@ import io.questdb.client.cutlass.qwp.client.sf.cursor.SegmentManager; import io.questdb.client.cutlass.qwp.client.sf.cursor.SlotLock; import io.questdb.client.std.Files; +import io.questdb.client.std.FilesFacade; import io.questdb.client.std.MemoryTag; import io.questdb.client.std.ObjList; import io.questdb.client.std.Unsafe; @@ -608,11 +609,12 @@ private static void assertSlotCanBeReacquired(String sfDir) { private static Throwable invokeOwnedPrivateConstructorExpectingFailure( String sfDir, long segmentSizeBytes, SegmentManager manager) throws Exception { Constructor ctor = CursorSendEngine.class.getDeclaredConstructor( - String.class, long.class, SegmentManager.class, boolean.class, long.class); + String.class, long.class, SegmentManager.class, boolean.class, long.class, + FilesFacade.class); ctor.setAccessible(true); try { ctor.newInstance(sfDir, segmentSizeBytes, manager, true, - CursorSendEngine.DEFAULT_APPEND_DEADLINE_NANOS); + CursorSendEngine.DEFAULT_APPEND_DEADLINE_NANOS, FilesFacade.INSTANCE); fail("expected constructor failure"); return null; } catch (InvocationTargetException e) { diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopOrphanTailTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopOrphanTailTest.java index bf16f3b0..d19f0e12 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopOrphanTailTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopOrphanTailTest.java @@ -44,6 +44,8 @@ import java.util.List; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; /** @@ -223,6 +225,69 @@ public void testSlowPathReplaysBelowTailThenRetiresAndRecyclesOnce() throws Exce }); } + @Test + public void testRecoveredMaxSymbolIdSpansSealedSegments() throws Exception { + // The torn-dict detector must walk the SEALED segments too, not just the active + // one. Every other test in the suite fits its whole slot in a single active + // segment, so the sealed walk in SegmentRing.maxSymbolDeltaEnd was dead code as + // far as the tests were concerned: deleting it left them all green while + // recoveredMaxSymbolId silently collapsed to -1. + // + // -1 is not a benign wrong answer. seedGlobalDictionaryFromPersisted's guard is + // `recoveredMaxSymbolId >= pd.size()`, so a value that is too LOW never fires: a + // torn dictionary is trusted, the producer resumes seeded from the short + // dictionary, and it hands ids the surviving frames already define to different + // symbols -- the exact silent misattribution the whole feature exists to prevent. + // + // The shape that needs the sealed walk is the crash the guard was built for. In a + // plain multi-segment slot the ACTIVE segment holds the newest frames and thus the + // highest ids, so an active-only walk happens to get the right answer. It stops + // being right when the active segment contributes NOTHING to the committed range + // -- here, a producer that died mid-transaction, leaving an aborted deferred tail + // long enough to overflow into a fresh segment. maxSymbolDeltaEnd skips those + // frames (fsn > recoveredCommitBoundaryFsn), so the highest COMMITTED id is left + // sitting in a SEALED segment, reachable only through the sealed walk. + TestUtils.assertMemoryLeak(() -> { + // Small segments so the tail really does roll one. Each frame is + // FRAME_HEADER_SIZE + (QWP HEADER_SIZE + 2) = 22 bytes, against 256 - 24 + // usable, so a segment holds ~10. + try (CursorSendEngine engine = new CursorSendEngine(tmpDir, 256)) { + // Committed delta frames registering ids 0,1,2 -- the highest COMMITTED + // ids in the slot. + for (int i = 0; i < 3; i++) { + appendDeltaFrame(engine, false, i, 1); + } + assertNull("the committed frames must still be in the ACTIVE segment here", + engine.firstSealed()); + // The aborted transaction: deferred frames with no covering commit. Keep + // appending until they overflow the segment, then a few more, so the + // committed frames end up SEALED and the active segment holds nothing but + // the tail. Ids stay small so the test's one-byte varint encoding holds. + int deferredId = 3; + for (int i = 0; i < 64 && engine.firstSealed() == null; i++) { + appendDeltaFrame(engine, true, deferredId++, 1); + } + assertNotNull("the deferred tail must have rolled a segment", engine.firstSealed()); + for (int i = 0; i < 3; i++) { + appendDeltaFrame(engine, true, deferredId++, 1); + } + } + try (CursorSendEngine engine = new CursorSendEngine(tmpDir, 256)) { + assertTrue(engine.wasRecoveredFromDisk()); + assertNotNull("the recovered slot must have a sealed segment", engine.firstSealed()); + assertEquals("the last commit-bearing frame is fsn 2", + 2L, engine.recoveredCommitBoundaryFsn()); + // ids 0,1,2 were introduced by the committed frames, all of which now live + // in a SEALED segment. The active segment holds only the deferred tail, + // which the walk skips -- so an active-only walk returns 0 and this comes + // back -1. + assertEquals("the highest COMMITTED id lives in a SEALED segment; " + + "maxSymbolDeltaEnd must walk the sealed segments to see it", + 2L, engine.recoveredMaxSymbolId()); + } + }); + } + @Test public void testRecoveredMaxSymbolIdExcludesOrphanTailFrames() throws Exception { // recoveredMaxSymbolId must reflect only COMMITTED (transmitted) frames, not diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java index b1255aac..c5222e0e 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java @@ -28,6 +28,7 @@ import io.questdb.client.cutlass.qwp.client.sf.cursor.PersistedSymbolDict; import io.questdb.client.std.FilesFacade; import io.questdb.client.std.ObjList; +import io.questdb.client.test.tools.DelegatingFilesFacade; import io.questdb.client.test.tools.TestUtils; import org.junit.Assert; import org.junit.Test; @@ -833,141 +834,6 @@ private static void rmDir(Path dir) { TestUtils.removeTmpDirRec(dir == null ? null : dir.toString()); } - /** - * A {@link FilesFacade} that delegates every call to {@link FilesFacade#INSTANCE}. - * Subclasses inject a single fault; every other call hits the real filesystem. - */ - private static class DelegatingFilesFacade implements FilesFacade { - @Override - public long allocNativePath(String path) { - return INSTANCE.allocNativePath(path); - } - - @Override - public boolean allocate(int fd, long size) { - return INSTANCE.allocate(fd, size); - } - - @Override - public int close(int fd) { - return INSTANCE.close(fd); - } - - @Override - public boolean exists(String path) { - return INSTANCE.exists(path); - } - - @Override - public void findClose(long findPtr) { - INSTANCE.findClose(findPtr); - } - - @Override - public long findFirst(String dir) { - return INSTANCE.findFirst(dir); - } - - @Override - public long findName(long findPtr) { - return INSTANCE.findName(findPtr); - } - - @Override - public int findNext(long findPtr) { - return INSTANCE.findNext(findPtr); - } - - @Override - public int findType(long findPtr) { - return INSTANCE.findType(findPtr); - } - - @Override - public void freeNativePath(long pathPtr) { - INSTANCE.freeNativePath(pathPtr); - } - - @Override - public int fsync(int fd) { - return INSTANCE.fsync(fd); - } - - @Override - public long length(int fd) { - return INSTANCE.length(fd); - } - - @Override - public long length(String path) { - return INSTANCE.length(path); - } - - @Override - public long length(long pathPtr) { - return INSTANCE.length(pathPtr); - } - - @Override - public int lock(int fd) { - return INSTANCE.lock(fd); - } - - @Override - public int mkdir(String path, int mode) { - return INSTANCE.mkdir(path, mode); - } - - @Override - public int openCleanRW(String path) { - return INSTANCE.openCleanRW(path); - } - - @Override - public int openCleanRW(long pathPtr) { - return INSTANCE.openCleanRW(pathPtr); - } - - @Override - public int openRW(String path) { - return INSTANCE.openRW(path); - } - - @Override - public int openRW(long pathPtr) { - return INSTANCE.openRW(pathPtr); - } - - @Override - public long read(int fd, long addr, long len, long offset) { - return INSTANCE.read(fd, addr, len, offset); - } - - @Override - public boolean remove(String path) { - return INSTANCE.remove(path); - } - - @Override - public boolean remove(long pathPtr) { - return INSTANCE.remove(pathPtr); - } - - @Override - public int rename(String oldPath, String newPath) { - return INSTANCE.rename(oldPath, newPath); - } - - @Override - public boolean truncate(int fd, long size) { - return INSTANCE.truncate(fd, size); - } - - @Override - public long write(int fd, long addr, long len, long offset) { - return INSTANCE.write(fd, addr, len, offset); - } - } /** * Fails every {@link #truncate(int, long)} -- reproducing a host where the diff --git a/core/src/test/java/io/questdb/client/test/tools/DelegatingFilesFacade.java b/core/src/test/java/io/questdb/client/test/tools/DelegatingFilesFacade.java new file mode 100644 index 00000000..516800c4 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/tools/DelegatingFilesFacade.java @@ -0,0 +1,170 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.tools; + +import io.questdb.client.std.FilesFacade; + +/** + * Pass-through {@link FilesFacade} that forwards every call to + * {@link FilesFacade#INSTANCE}. Subclass it and override the ONE method whose + * failure you want to inject -- a short write, a truncate that cannot trim, an + * implausible file length -- so a test can drive a real I/O failure path without a + * real broken disk. + *

    + * {@code FilesFacade} has no default methods, so the delegation boilerplate is + * unavoidable; it lives here once rather than being stamped into each test that + * needs a fault seam. + */ +public class DelegatingFilesFacade implements FilesFacade { + @Override + public long allocNativePath(String path) { + return INSTANCE.allocNativePath(path); + } + + @Override + public boolean allocate(int fd, long size) { + return INSTANCE.allocate(fd, size); + } + + @Override + public int close(int fd) { + return INSTANCE.close(fd); + } + + @Override + public boolean exists(String path) { + return INSTANCE.exists(path); + } + + @Override + public void findClose(long findPtr) { + INSTANCE.findClose(findPtr); + } + + @Override + public long findFirst(String dir) { + return INSTANCE.findFirst(dir); + } + + @Override + public long findName(long findPtr) { + return INSTANCE.findName(findPtr); + } + + @Override + public int findNext(long findPtr) { + return INSTANCE.findNext(findPtr); + } + + @Override + public int findType(long findPtr) { + return INSTANCE.findType(findPtr); + } + + @Override + public void freeNativePath(long pathPtr) { + INSTANCE.freeNativePath(pathPtr); + } + + @Override + public int fsync(int fd) { + return INSTANCE.fsync(fd); + } + + @Override + public long length(int fd) { + return INSTANCE.length(fd); + } + + @Override + public long length(String path) { + return INSTANCE.length(path); + } + + @Override + public long length(long pathPtr) { + return INSTANCE.length(pathPtr); + } + + @Override + public int lock(int fd) { + return INSTANCE.lock(fd); + } + + @Override + public int mkdir(String path, int mode) { + return INSTANCE.mkdir(path, mode); + } + + @Override + public int openCleanRW(String path) { + return INSTANCE.openCleanRW(path); + } + + @Override + public int openCleanRW(long pathPtr) { + return INSTANCE.openCleanRW(pathPtr); + } + + @Override + public int openRW(String path) { + return INSTANCE.openRW(path); + } + + @Override + public int openRW(long pathPtr) { + return INSTANCE.openRW(pathPtr); + } + + @Override + public long read(int fd, long addr, long len, long offset) { + return INSTANCE.read(fd, addr, len, offset); + } + + @Override + public boolean remove(String path) { + return INSTANCE.remove(path); + } + + @Override + public boolean remove(long pathPtr) { + return INSTANCE.remove(pathPtr); + } + + @Override + public int rename(String oldPath, String newPath) { + return INSTANCE.rename(oldPath, newPath); + } + + @Override + public boolean truncate(int fd, long size) { + return INSTANCE.truncate(fd, size); + } + + @Override + public long write(int fd, long addr, long len, long offset) { + return INSTANCE.write(fd, addr, len, offset); + } +} From aeb92b559467fbeb1c4516e5ac6dc6c8fffa80c3 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Tue, 14 Jul 2026 20:14:10 +0100 Subject: [PATCH 58/96] Set aside the slots recovery cannot rescue, instead of failing The full-dictionary rebuild and the unreplayable-slot quarantine answer two halves of the same question and have to be ordered, not merged. Rebuilding the dictionary from the surviving frames' own delta sections rescues most of the slots that used to fail, so it must run FIRST -- a quarantine that pre-judged the slot on its own coverage arithmetic would set aside slots recovery can still replay in full, and strand data that drains perfectly. What it cannot rescue it still throws on, and that throw still escaped build(). senderId is stable and a not-fully-drained slot is retained on close, so every retry re-recovered the same slot and threw again: the application could not construct a Sender at all, and so could not even BUFFER new rows. One already-lost batch became an unbounded outage of everything after it, which inverts the guarantee store-and-forward exists to give. So seedGlobalDictionaryFromPersisted is now the single authority on whether a slot is replayable -- it is the only code that has tried every source of truth -- and its give-up is typed. build() waits for that verdict rather than forming its own, and on it sets the slot aside: the bytes are kept for forensics and resend, the .failed sentinel tells the orphan drainer the copy is human-in-the-loop, and the producer continues on a clean slot. The engine's own isRecoveredDictionaryIncomplete() guess goes away with it. Not one frame of a set-aside slot ever reaches the wire; that safety property is unchanged and still asserted. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../main/java/io/questdb/client/Sender.java | 62 ++++++--- .../qwp/client/QwpWebSocketSender.java | 6 +- .../client/sf/cursor/CursorSendEngine.java | 31 ----- .../sf/cursor/UnreplayableSlotException.java | 51 ++++++++ .../qwp/client/DeltaDictRecoveryTest.java | 123 ++++++++++++++++-- 5 files changed, 214 insertions(+), 59 deletions(-) create mode 100644 core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/UnreplayableSlotException.java diff --git a/core/src/main/java/io/questdb/client/Sender.java b/core/src/main/java/io/questdb/client/Sender.java index 447d5fad..b674cf1b 100644 --- a/core/src/main/java/io/questdb/client/Sender.java +++ b/core/src/main/java/io/questdb/client/Sender.java @@ -40,6 +40,7 @@ import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorWebSocketSendLoop; import io.questdb.client.cutlass.qwp.client.sf.cursor.OrphanScanner; import io.questdb.client.cutlass.qwp.client.sf.cursor.PersistedSymbolDict; +import io.questdb.client.cutlass.qwp.client.sf.cursor.UnreplayableSlotException; import io.questdb.client.impl.ConfStringParser; import io.questdb.client.impl.ConfigString; import io.questdb.client.impl.ConfigView; @@ -1560,11 +1561,6 @@ public Sender build() { CursorSendEngine cursorEngine = new CursorSendEngine( slotPath, actualSfMaxBytes, actualSfMaxTotalBytes, actualSfAppendDeadlineNanos); - if (cursorEngine.isRecoveredDictionaryIncomplete()) { - cursorEngine = quarantineTornSlot( - cursorEngine, sfDir, senderId, slotPath, actualSfMaxBytes, - actualSfMaxTotalBytes, actualSfAppendDeadlineNanos); - } int actualErrorInboxCapacity = errorInboxCapacity != PARAMETER_NOT_SET_EXPLICITLY ? errorInboxCapacity : io.questdb.client.cutlass.qwp.client.sf.cursor.SenderErrorDispatcher.DEFAULT_CAPACITY; @@ -1576,7 +1572,15 @@ public Sender build() { for (int i = 0, n = hosts.size(); i < n; i++) { wsEndpoints.add(new QwpWebSocketSender.Endpoint(hosts.getQuick(i), ports.getQuick(i))); } - QwpWebSocketSender connected; + // The recovery seed inside connect() is the authority on whether a recovered + // slot can be replayed: it rebuilds the dictionary from its intact prefix and + // then from the surviving frames' own delta sections, and throws + // UnreplayableSlotException only once neither source holds the missing ids. + // Quarantining on anything weaker would set aside slots that recovery can + // still rescue, so build() waits for that verdict rather than pre-judging it. + QwpWebSocketSender connected = null; + boolean quarantined = false; + while (connected == null) { try { connected = QwpWebSocketSender.connect( wsEndpoints, @@ -1603,6 +1607,32 @@ public Sender build() { actualPoisonMinEscalationWindowMillis, actualCatchUpCapGapMinEscalationWindowMillis ); + } catch (UnreplayableSlotException e) { + // The one failure build() recovers from. The slot's frames reference ids + // that nothing still holds, so they can never go on the wire -- but that is + // no reason to take the producer down with them. Before this, the throw + // escaped build() and, because senderId is stable and a not-fully-drained + // slot is retained on close, every retry re-recovered the same slot and + // threw again: the application could not construct a Sender at all, so it + // could not even BUFFER new rows. An already-lost batch became an unbounded + // outage of everything after it. + // + // Set the slot aside instead, keep its bytes for forensics and resend, and + // start the producer on a clean one. Once only: a second such failure would + // mean the FRESH slot is unreplayable, which cannot happen, so let it out + // rather than loop. + if (quarantined || slotPath == null) { + try { + cursorEngine.close(); + } catch (Throwable ignored) { + // best-effort + } + throw e; + } + quarantined = true; + cursorEngine = quarantineTornSlot( + cursorEngine, e, sfDir, senderId, slotPath, actualSfMaxBytes, + actualSfMaxTotalBytes, actualSfAppendDeadlineNanos); } catch (Throwable t) { // connect() failed before ownership of cursorEngine // transferred — close it ourselves. @@ -1613,6 +1643,7 @@ public Sender build() { } throw t; } + } // connect() succeeded — `connected` now owns cursorEngine // via setCursorEngine(engine, true). From here on, ANY // failure must close `connected` (which closes the engine @@ -3001,18 +3032,17 @@ private static long parseSizeValue(@NotNull StringSink value, @NotNull String na * throw -- loudly, and never silently dropping bytes. */ private static CursorSendEngine quarantineTornSlot( - CursorSendEngine torn, String sfDir, String senderId, String slotPath, + CursorSendEngine torn, UnreplayableSlotException cause, String sfDir, + String senderId, String slotPath, long sfMaxBytes, long sfMaxTotalBytes, long sfAppendDeadlineNanos ) { - long recoveredMaxSymbolId = torn.recoveredMaxSymbolId(); - PersistedSymbolDict dict = torn.getPersistedSymbolDict(); - int coverage = dict != null ? dict.size() : 0; - String detail = "recovered store-and-forward symbol dictionary cannot cover the surviving " - + "frames (likely a host crash tore its unsynced tail): frames reference symbol id " - + recoveredMaxSymbolId + " but the recovered dictionary holds only " + coverage - + " id(s)"; - // Release the slot lock and the dictionary fd before renaming. The slot is not - // fully drained, so close() retains every file. + // The verdict, and the reason, come from the recovery seed -- the only code that + // has tried every source of truth. Recomputing them here would mean a second, + // independently-drifting notion of "unreplayable". + final String detail = cause.getMessage(); + // Release the slot lock and the dictionary fd before renaming. connect()'s failure + // path already closed the engine; close() is idempotent, so make it explicit rather + // than depend on that. torn.close(); String quarantinePath = null; diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index fb9a7f52..47319455 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -47,6 +47,7 @@ import io.questdb.client.cutlass.qwp.client.sf.cursor.DefaultSenderErrorHandler; import io.questdb.client.cutlass.qwp.client.sf.cursor.DefaultSenderProgressHandler; import io.questdb.client.cutlass.qwp.client.sf.cursor.PersistedSymbolDict; +import io.questdb.client.cutlass.qwp.client.sf.cursor.UnreplayableSlotException; import io.questdb.client.cutlass.qwp.client.sf.cursor.SenderConnectionDispatcher; import io.questdb.client.cutlass.qwp.client.sf.cursor.SenderErrorDispatcher; import io.questdb.client.cutlass.qwp.client.sf.cursor.SenderProgressDispatcher; @@ -3926,7 +3927,10 @@ private void seedGlobalDictionaryFromPersisted(PersistedSymbolDict pd) { ObjList fromFrames = new ObjList<>(); long coverage = cursorEngine.collectReplaySymbolsAbove(baseline, fromFrames); if (coverage < 0) { - throw new LineSenderException( + // Typed, because Sender.build() sets such a slot aside instead of failing: this + // is the point at which every source of truth has been tried and none of them + // holds the missing ids. See UnreplayableSlotException. + throw new UnreplayableSlotException( "recovered store-and-forward symbol dictionary is incomplete and cannot be " + "rebuilt from the surviving frames (likely a host crash tore its unsynced " + "tail): the frames reference symbol ids below their own delta start, which " diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java index ffd964f3..9f3add89 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java @@ -767,37 +767,6 @@ public boolean isDeltaDictEnabled() { return sfDir == null || persistedSymbolDict != null; } - /** - * Whether this RECOVERED slot's surviving frames reference symbol ids the - * recovered dictionary cannot cover -- i.e. the slot cannot be drained by a - * producer that shares its id space, and a producer seeded from the short - * dictionary would reuse ids those frames already define. - *

    - * True in two situations, both of which mean the same thing to the caller: - *

      - *
    • the dictionary opened but is SHORT of the frames (a host/power crash - * tore its unsynced tail -- see {@code PersistedSymbolDict}'s durability - * note); or
      • - *
    • the dictionary did not open at all ({@code persistedSymbolDict == null} - * -- an unreadable or torn side-file, fd exhaustion, a read-only remount) - * while the surviving frames are DELTA frames that need one. Coverage is - * then zero, so any frame referencing an id at all is unreplayable.
      • - *
    - * Always false for a fresh (non-recovered) slot: {@code recoveredMaxSymbolId} is - * -1 there, and -1 is below every coverage. - *

    - * The caller must NOT drain such a slot with a live producer attached. The - * foreground sender quarantines it and starts on a fresh slot - * ({@code Sender.build()}); the orphan drainer reaches the same verdict - * independently via the send loop's replay guard and marks the slot failed. - * Either way the recorded bytes stay on disk and the affected data must be - * resent -- but the producer keeps producing, which is the whole point of - * store-and-forward. - */ - public boolean isRecoveredDictionaryIncomplete() { - long coverage = persistedSymbolDict != null ? persistedSymbolDict.size() : 0; - return recoveredMaxSymbolId >= coverage; - } /** * Pass-through to {@link SegmentRing#nextSealedAfter(MmapSegment)}. diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/UnreplayableSlotException.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/UnreplayableSlotException.java new file mode 100644 index 00000000..00c0d351 --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/UnreplayableSlotException.java @@ -0,0 +1,51 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.qwp.client.sf.cursor; + +import io.questdb.client.cutlass.line.LineSenderException; + +/** + * A recovered store-and-forward slot whose symbol dictionary cannot be rebuilt at all -- + * neither from the persisted side-file's intact prefix nor from the surviving frames' own + * delta sections. Its frames reference ids that nothing still holds, so replaying them + * would null-pad the gap on the server and silently misattribute symbol values. + *

    + * This is the ONE verdict {@code seedGlobalDictionaryFromPersisted} reaches after it has + * tried every source of truth it has, so it is the only signal that may quarantine a slot. + * A distinct type, rather than a message match, because the difference between "this slot + * is unreplayable" and any other {@link LineSenderException} out of the connect path is the + * difference between setting a slot aside and silently discarding one that was fine. + *

    + * It stays a {@code LineSenderException} so that a caller which does NOT handle it -- the + * background drainer, a test constructing the sender directly -- keeps the old fail-clean + * behaviour rather than seeing a new checked type. Only {@code Sender.build()} treats it as + * recoverable, by setting the slot aside and starting the producer on a fresh one. + */ +public class UnreplayableSlotException extends LineSenderException { + + public UnreplayableSlotException(CharSequence message) { + super(message); + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java index 4be99883..3b94d171 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java @@ -155,6 +155,71 @@ public void testRecoveredSlotReplaysDeltaFramesAgainstFreshServer() throws Excep }); } + @Test + public void testUnreplayableSlotIsSetAsideAndTheProducerKeepsProducing() throws Exception { + // The point of setting an unreplayable slot aside rather than failing: the + // producer stays ALIVE and its new data still reaches the server. This is the + // property store-and-forward exists to guarantee, and the one bricking build() + // destroyed -- one host crash and the application could never construct a Sender + // again, so every row after the tear was lost too, not just the torn batch. + // + // Asserted end to end against a real wire: the fresh rows arrive and the server's + // reconstructed dictionary holds EXACTLY the post-recovery symbols. Not one + // symbol from the unreplayable slot appears -- so the producer kept producing + // AND the torn frames stayed off the wire. + assertMemoryLeak(() -> { + // Phase 1: record delta frames, then lose the whole dictionary (host crash). + try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) { + int port = silent.getPort(); + silent.start(); + Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS)); + String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + + ";close_flush_timeout_millis=0;"; + try (Sender s1 = Sender.fromConfig(cfg)) { + for (int i = 0; i < 6; i++) { + s1.table("m").symbol("s", "torn-" + i).longColumn("v", i).atNow(); + s1.flush(); + } + } + } + java.nio.file.Path slot = Paths.get(sfDir, "default"); + java.nio.file.Path dict = slot.resolve(".symbol-dict"); + java.nio.file.Files.write(dict, + Arrays.copyOf(java.nio.file.Files.readAllBytes(dict), 8)); // header only: 0 symbols + writeAckWatermark(slot.resolve(".ack-watermark"), 2); + + // Phase 2: the recovering sender sets the slot aside and keeps producing. + DictReconstructingHandler handler = new DictReconstructingHandler(); + try (TestWebSocketServer good = new TestWebSocketServer(handler)) { + int port = good.getPort(); + good.start(); + Assert.assertTrue(good.awaitStart(5, TimeUnit.SECONDS)); + + String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";"; + try (Sender s2 = Sender.fromConfig(cfg)) { + s2.table("m").symbol("s", "fresh-a").longColumn("v", 100).atNow(); + s2.table("m").symbol("s", "fresh-b").longColumn("v", 101).atNow(); + s2.flush(); + long deadline = System.currentTimeMillis() + 10_000; + while (System.currentTimeMillis() < deadline && handler.dictSnapshot().size() < 2) { + Thread.sleep(20); + } + Assert.assertEquals("the new rows must reach the server", + 2, handler.dictSnapshot().size()); + } + + assertUnreplayableSlotSetAside(); + List dict2 = handler.dictSnapshot(); + Assert.assertEquals("the server must hold exactly the post-recovery symbols", + Arrays.asList("fresh-a", "fresh-b"), dict2); + for (String s : dict2) { + Assert.assertFalse("no symbol of the unreplayable slot may reach the server: " + s, + s.startsWith("torn-")); + } + } + }); + } + @Test public void testTornDictionaryFailsCleanlyInsteadOfCorrupting() throws Exception { assertMemoryLeak(() -> { @@ -196,19 +261,20 @@ public void testTornDictionaryFailsCleanlyInsteadOfCorrupting() throws Exception Assert.assertTrue(good.awaitStart(5, TimeUnit.SECONDS)); String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";"; - LineSenderException terminal = null; - try { - Sender s2 = Sender.fromConfig(cfg); - s2.close(); - } catch (LineSenderException e) { - terminal = e; - } + // The dictionary cannot be rebuilt from any source, so this slot is genuinely + // unreplayable -- and build() SETS IT ASIDE rather than failing. Failing here + // is what bricked the sender: senderId is stable and a not-fully-drained slot + // is retained on close, so every retry re-recovered the same slot and threw + // again, and the application could not construct a Sender at all -- it could + // not even buffer new rows. Now the producer comes up on a clean slot and the + // unreplayable bytes are kept aside for resend. + Sender.fromConfig(cfg).close(); + + // The safety property is unchanged: not one frame of the unreplayable slot + // may reach the server. Assert.assertEquals("no frame may be replayed to a fresh server with a torn dictionary", 0, handler.frames.get()); - Assert.assertNotNull("a torn dictionary must surface a terminal error", terminal); - Assert.assertTrue(terminal.getMessage(), - terminal.getMessage().contains("cannot be rebuilt from the surviving frames") - && terminal.getMessage().contains("resend")); + assertUnreplayableSlotSetAside(); } }); } @@ -1327,4 +1393,39 @@ public long write(int fd, long addr, long len, long offset) { } } + + /** + * The unreplayable-slot contract: a recovered slot whose symbol dictionary cannot be + * rebuilt -- not from its own intact prefix, not from the surviving frames' delta + * sections -- is SET ASIDE, never silently drained and never allowed to brick the + * sender. + *

    + * The bytes are kept for forensics and resend, the {@code .failed} sentinel tells the + * orphan drainer to treat the copy as human-in-the-loop rather than retry it forever, + * and the producer continues on a fresh, empty slot. + */ + private void assertUnreplayableSlotSetAside() { + java.nio.file.Path aside = Paths.get(sfDir, "default.unreplayable-0"); + Assert.assertTrue("the unreplayable slot must be set aside, not deleted: " + aside, + java.nio.file.Files.isDirectory(aside)); + Assert.assertTrue("the set-aside slot must keep its recorded frames for resend", + hasSegmentFile(aside)); + Assert.assertTrue("the set-aside slot must carry the .failed sentinel", + java.nio.file.Files.exists(aside.resolve(".failed"))); + Assert.assertTrue("the sender must continue on a live slot", + java.nio.file.Files.isDirectory(Paths.get(sfDir, "default"))); + } + + private static boolean hasSegmentFile(java.nio.file.Path dir) { + java.io.File[] files = dir.toFile().listFiles(); + if (files != null) { + for (java.io.File f : files) { + if (f.getName().endsWith(".sfa") && f.length() > 0) { + return true; + } + } + } + return false; + } + } From ff53e1f0a284b82c4b0f1f86c08e18856a2a0fdc Mon Sep 17 00:00:00 2001 From: glasstiger Date: Tue, 14 Jul 2026 21:00:11 +0100 Subject: [PATCH 59/96] Rebuild the dictionary from the frames still on disk A slot whose symbol dictionary was damaged was being condemned as unreplayable while every symbol it needed was sitting on disk. The sender told the user to resend data that had never been lost, and after the previous commit it set the slot aside instead. Three things kept the rebuild from ever running. setCursorEngine gated the producer's seed on deltaDictEnabled. That flag is false exactly when the slot's dictionary failed to open -- fd exhaustion, a read-only remount, ENOSPC -- which is precisely when the frames on disk are the only surviving copy of the symbols. The rebuild was dead code for the very case it was written for. collectReplaySymbolsAbove walked from ackedFsn+1, so it skipped the acked frames. But an acked frame is not trimmed the instant it is acked: trim unlinks whole SEALED segments. The frames that registered the early ids are almost always still there, and they are exactly the ids the replay set's deltas start above. It walks every frame on disk now, and a slot whose registering frames really HAVE been trimmed still reports the gap, because then nothing holds them. And the send loop's mirror seeded only from the dictionary file, never from the frames -- so even with the other two fixed, the loop would still condemn (deltaStart > sentDictCount) a slot the producer had just seeded successfully. The mirror is the loop's model of what the SERVER holds and it is what the catch-up ships, so it now seeds from the same two sources in the same order, and the two land on the same number by construction. Only when a surviving frame actually depends on ids it does not carry: at maxSymbolDeltaStart == 0 every frame re-registers from id 0 as it replays, so a full-dict slot stays catch-up-free. The recovery tests now assert the strong form -- the fresh server's reconstructed dictionary must come back COMPLETE and in order, which can only happen if the catch-up carried the ids the acked frames still hold. A null-padded or shifted dictionary fails there, which is the corruption the old guard condemned these slots to avoid. It never had to. The quarantine now covers only what is genuinely unrecoverable: the registering frames trimmed away and the dictionary torn, so nothing anywhere holds those ids. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../qwp/client/QwpWebSocketSender.java | 6 +- .../client/sf/cursor/CursorSendEngine.java | 40 +- .../sf/cursor/CursorWebSocketSendLoop.java | 38 ++ .../qwp/client/DeltaDictRecoveryTest.java | 376 +++++++----------- 4 files changed, 216 insertions(+), 244 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index 47319455..d7ba19cf 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -2258,7 +2258,11 @@ public void setCursorEngine(CursorSendEngine engine, boolean takeOwnership) { // persisted dictionary so newly ingested symbols continue from the // recovered ids (rather than colliding with them at 0), and the delta // baseline resumes where the crashed session left off. - if (deltaDictEnabled && engine.wasRecoveredFromDisk()) { + // NOT gated on deltaDictEnabled. That flag is false exactly when the slot's dictionary + // failed to open -- which is precisely when the frames on disk are the only surviving + // copy of the symbols and the rebuild matters most. Gating the seed on it made the + // rebuild dead code for the very case it was written for. + if (engine != null && engine.wasRecoveredFromDisk()) { seedGlobalDictionaryFromPersisted(engine.getPersistedSymbolDict()); } } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java index 9f3add89..37adf4a0 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java @@ -108,6 +108,14 @@ public final class CursorSendEngine implements QuietCloseable { // against the recovered dictionary size to fail clean instead. Computed once // in the constructor's recovery branch; -1 elsewhere. private long recoveredMaxSymbolId = -1L; + // Highest deltaStart across the recovered COMMITTED frames; 0 when none carries a symbol + // dictionary. ZERO means every surviving frame is SELF-SUFFICIENT -- it re-registers its + // dictionary from id 0 -- so the slot replays with no dictionary at all and the send loop + // needs no catch-up. ABOVE zero means at least one frame is a true delta whose ids depend + // on registrations it does not itself carry, so the loop must seed its mirror (and ship a + // catch-up) before replaying. Both the full-dict-fallback discard below and the send + // loop's mirror seeding key off this. + private long recoveredMaxSymbolDeltaStart; // FSN of the last frame of a recovered orphaned deferred tail, or -1 when // the recovered ring has no such tail. When >= 0, frames // [recoveredCommitBoundaryFsn + 1 .. recoveredOrphanTipFsn] all carry @@ -380,14 +388,15 @@ private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager man // dictionary. The recoveredMaxSymbolId >= size guard means this never // fires for a slot whose dictionary is intact, nor for an empty slot // (recoveredMaxSymbolId == -1). Single-threaded; before the I/O loop. - if (persistedDictInProgress != null - && recoveredMaxSymbolId >= persistedDictInProgress.size() - && recovered.maxSymbolDeltaStart( + this.recoveredMaxSymbolDeltaStart = recovered.maxSymbolDeltaStart( QwpConstants.MAGIC_MESSAGE, QwpConstants.HEADER_OFFSET_FLAGS, QwpConstants.FLAG_DELTA_SYMBOL_DICT, QwpConstants.HEADER_SIZE, - recoveredCommitBoundaryFsn) == 0L) { + recoveredCommitBoundaryFsn); + if (persistedDictInProgress != null + && recoveredMaxSymbolId >= persistedDictInProgress.size() + && recoveredMaxSymbolDeltaStart == 0L) { persistedDictInProgress.close(); persistedDictInProgress = null; } @@ -715,7 +724,14 @@ public long collectReplaySymbolsAbove(int baseline, ObjList out) { QwpConstants.HEADER_OFFSET_FLAGS, QwpConstants.FLAG_DELTA_SYMBOL_DICT, QwpConstants.HEADER_SIZE, - ackedFsn() + 1L, + // Walk from the LOWEST frame still on disk, not from ackedFsn+1. An acked frame + // is not trimmed the instant it is acked -- trim drops whole SEALED segments -- + // so the symbols it registered are usually still sitting right there, and they + // are exactly the ids the replay set's deltas start above. Skipping them threw + // away the only surviving copy of those symbols and condemned a slot that had + // everything it needed. A slot whose registering frames really HAVE been + // trimmed still reports the gap, because then no frame on disk holds them. + 0L, recoveredCommitBoundaryFsn, baseline, out @@ -842,6 +858,20 @@ public long recoveredMaxSymbolId() { return recoveredMaxSymbolId; } + /** + * Highest {@code deltaStart} across the recovered committed frames; {@code 0} when every + * surviving frame is self-sufficient (or none carries a dictionary at all). + *

    + * The send loop uses this to decide whether it needs a catch-up: at zero, every frame + * re-registers its dictionary from id 0 as it replays, so seeding the mirror -- and + * shipping a catch-up frame off it -- would be pure redundancy. Above zero, at least one + * frame's delta starts above ids it does not itself carry, so the mirror must hold those + * ids before the replay begins or the server null-pads the hole. + */ + public long recoveredMaxSymbolDeltaStart() { + return recoveredMaxSymbolDeltaStart; + } + /** * FSN of the last frame of a recovered orphaned deferred tail, or * {@code -1} when none. See {@link #recoveredCommitBoundaryFsn()}: the diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java index 8e34011d..cf3f1637 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java @@ -43,6 +43,8 @@ import io.questdb.client.std.CharSequenceLongHashMap; import io.questdb.client.std.MemoryTag; import io.questdb.client.std.QuietCloseable; +import io.questdb.client.std.str.Utf8s; +import io.questdb.client.std.ObjList; import io.questdb.client.std.Unsafe; import org.jetbrains.annotations.TestOnly; import org.slf4j.Logger; @@ -699,6 +701,25 @@ public CursorWebSocketSendLoop(WebSocketClient client, CursorSendEngine engine, sentDictCount = pd.size(); } } + // ...and then from the surviving frames' own delta sections, exactly as the producer + // seeds its dictionary. The mirror is the loop's model of what the SERVER holds and it + // is what the catch-up frame ships, so when it stops at the persisted prefix while the + // producer's baseline runs past it, the loop condemns (deltaStart > sentDictCount) a + // slot the producer just seeded successfully. Same two sources, same order, so the two + // land on the same number by construction. + // ...but ONLY when a surviving frame actually depends on ids it does not carry + // (recoveredMaxSymbolDeltaStart > 0). At zero every frame is self-sufficient and + // re-registers its dictionary from id 0 as it replays, so seeding the mirror would buy + // nothing and cost a catch-up frame on every connection -- full-dict slots must stay + // catch-up-free. + if (engine.recoveredMaxSymbolDeltaStart() > 0L) { + ObjList fromFrames = new ObjList<>(); + if (engine.collectReplaySymbolsAbove(sentDictCount, fromFrames) >= 0) { + for (int i = 0, n = fromFrames.size(); i < n; i++) { + appendSymbolToMirror(fromFrames.getQuick(i)); + } + } + } this.fsnAtZero = fsnAtZero; this.parkNanos = parkNanos; this.reconnectFactory = reconnectFactory; @@ -2268,6 +2289,23 @@ private void accumulateSentDict(long payloadAddr, int payloadLen, int deltaStart sentDictCount += (int) newCount; } + /** + * Appends one symbol to the sent-dictionary mirror in wire form + * ({@code [len varint][utf8]}), advancing {@code sentDictCount}. Seeds the mirror at + * construction; {@link #accumulateSentDict} extends it from live frames thereafter. + */ + private void appendSymbolToMirror(CharSequence symbol) { + int utf8Len = Utf8s.utf8Bytes(symbol); + int wireLen = NativeBufferWriter.varintSize(utf8Len) + utf8Len; + ensureSentDictCapacity((long) sentDictBytesLen + wireLen); + long p = NativeBufferWriter.writeVarint(sentDictBytesAddr + sentDictBytesLen, utf8Len); + if (utf8Len > 0) { + Utf8s.strCpyUtf8(symbol, p, utf8Len); + } + sentDictBytesLen += wireLen; + sentDictCount++; + } + private void ensureSentDictCapacity(long required) { if (sentDictBytesCapacity >= required) { return; diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java index 3b94d171..aaaeb66a 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java @@ -156,210 +156,137 @@ public void testRecoveredSlotReplaysDeltaFramesAgainstFreshServer() throws Excep } @Test - public void testUnreplayableSlotIsSetAsideAndTheProducerKeepsProducing() throws Exception { - // The point of setting an unreplayable slot aside rather than failing: the - // producer stays ALIVE and its new data still reaches the server. This is the - // property store-and-forward exists to guarantee, and the one bricking build() - // destroyed -- one host crash and the application could never construct a Sender - // again, so every row after the tear was lost too, not just the torn batch. + public void testTornDictionaryRebuildsFromFramesAcrossTheAckWatermark() throws Exception { + // A host crash truncates the dictionary to nothing, and the ack watermark sits + // mid-stream so the replay set starts at a frame whose delta begins at id 3. // - // Asserted end to end against a real wire: the fresh rows arrive and the server's - // reconstructed dictionary holds EXACTLY the post-recovery symbols. Not one - // symbol from the unreplayable slot appears -- so the producer kept producing - // AND the torn frames stayed off the wire. + // The old guard called that unreplayable and told the user to resend. It is not: ids + // 0..2 are still sitting in the ACKED frames on disk. Trim unlinks whole SEALED + // segments, not individual acked frames, so those six tiny frames are all still in + // the one active segment. Rebuild from them and the slot drains in full. assertMemoryLeak(() -> { - // Phase 1: record delta frames, then lose the whole dictionary (host crash). - try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) { - int port = silent.getPort(); - silent.start(); - Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS)); - String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir - + ";close_flush_timeout_millis=0;"; - try (Sender s1 = Sender.fromConfig(cfg)) { - for (int i = 0; i < 6; i++) { - s1.table("m").symbol("s", "torn-" + i).longColumn("v", i).atNow(); - s1.flush(); - } - } - } + recordSixDeltaFrames(); java.nio.file.Path slot = Paths.get(sfDir, "default"); java.nio.file.Path dict = slot.resolve(".symbol-dict"); java.nio.file.Files.write(dict, - Arrays.copyOf(java.nio.file.Files.readAllBytes(dict), 8)); // header only: 0 symbols + Arrays.copyOf(java.nio.file.Files.readAllBytes(dict), 8)); // header only writeAckWatermark(slot.resolve(".ack-watermark"), 2); - - // Phase 2: the recovering sender sets the slot aside and keeps producing. - DictReconstructingHandler handler = new DictReconstructingHandler(); - try (TestWebSocketServer good = new TestWebSocketServer(handler)) { - int port = good.getPort(); - good.start(); - Assert.assertTrue(good.awaitStart(5, TimeUnit.SECONDS)); - - String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";"; - try (Sender s2 = Sender.fromConfig(cfg)) { - s2.table("m").symbol("s", "fresh-a").longColumn("v", 100).atNow(); - s2.table("m").symbol("s", "fresh-b").longColumn("v", 101).atNow(); - s2.flush(); - long deadline = System.currentTimeMillis() + 10_000; - while (System.currentTimeMillis() < deadline && handler.dictSnapshot().size() < 2) { - Thread.sleep(20); - } - Assert.assertEquals("the new rows must reach the server", - 2, handler.dictSnapshot().size()); - } - - assertUnreplayableSlotSetAside(); - List dict2 = handler.dictSnapshot(); - Assert.assertEquals("the server must hold exactly the post-recovery symbols", - Arrays.asList("fresh-a", "fresh-b"), dict2); - for (String s : dict2) { - Assert.assertFalse("no symbol of the unreplayable slot may reach the server: " + s, - s.startsWith("torn-")); - } - } + assertSlotRecoversWithCompleteDictionary(); }); } @Test - public void testTornDictionaryFailsCleanlyInsteadOfCorrupting() throws Exception { + public void testUnopenableDictRebuildsFromFramesFromIdZero() throws Exception { + // Same rebuild, but the dictionary cannot be OPENED at all (fd exhaustion, a + // read-only remount, ENOSPC -- modelled by planting a directory in its place). The + // producer's seed used to be gated on the dictionary having opened, which made the + // whole rebuild dead code for exactly this case. assertMemoryLeak(() -> { - // Phase 1: each row introduces a new symbol, so frame i carries deltaStart=i. - // Silent server + close-fast leaves all frames unacked in the slot. - try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) { - int port = silent.getPort(); - silent.start(); - Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS)); - String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir - + ";close_flush_timeout_millis=0;"; - try (Sender s1 = Sender.fromConfig(cfg)) { - for (int i = 0; i < 6; i++) { - s1.table("m").symbol("s", "sym-" + i).longColumn("v", i).atNow(); - s1.flush(); - } - } - } + recordSixDeltaFrames(); + java.nio.file.Path slot = Paths.get(sfDir, "default"); + java.nio.file.Path dict = slot.resolve(".symbol-dict"); + java.nio.file.Files.delete(dict); + java.nio.file.Files.createDirectory(dict); + writeAckWatermark(slot.resolve(".ack-watermark"), 0); + assertSlotRecoversWithCompleteDictionary(); + }); + } - // Simulate a host/power crash: the segment frames survive but the WHOLE - // persisted dictionary is lost (truncate .symbol-dict to its 8-byte header, - // 0 symbols), and the ack watermark was left mid-stream (FSN 2). The - // surviving frames still reference the lost ids. + @Test + public void testUnopenableDictRebuildsFromFramesAcrossTheAckWatermark() throws Exception { + // The hardest of the three: the dictionary is unopenable AND the replay set starts at + // deltaStart=3, so ids 0..2 exist nowhere except the acked frames on disk. + assertMemoryLeak(() -> { + recordSixDeltaFrames(); java.nio.file.Path slot = Paths.get(sfDir, "default"); java.nio.file.Path dict = slot.resolve(".symbol-dict"); - byte[] header = Arrays.copyOf(java.nio.file.Files.readAllBytes(dict), 8); - java.nio.file.Files.write(dict, header); + java.nio.file.Files.delete(dict); + java.nio.file.Files.createDirectory(dict); writeAckWatermark(slot.resolve(".ack-watermark"), 2); - - // Phase 2: recover against a fresh counting server. With the whole - // dictionary lost (size 0) while the surviving frames still reference its - // ids, seedGlobalDictionaryFromPersisted must refuse the resume at BUILD -- - // its size==0 short-circuit must NOT skip the torn-dict guard -- rather than - // let the producer resume unseeded and silently misattribute reused ids. - CountingHandler handler = new CountingHandler(); - try (TestWebSocketServer good = new TestWebSocketServer(handler)) { - int port = good.getPort(); - good.start(); - Assert.assertTrue(good.awaitStart(5, TimeUnit.SECONDS)); - - String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";"; - // The dictionary cannot be rebuilt from any source, so this slot is genuinely - // unreplayable -- and build() SETS IT ASIDE rather than failing. Failing here - // is what bricked the sender: senderId is stable and a not-fully-drained slot - // is retained on close, so every retry re-recovered the same slot and threw - // again, and the application could not construct a Sender at all -- it could - // not even buffer new rows. Now the producer comes up on a clean slot and the - // unreplayable bytes are kept aside for resend. - Sender.fromConfig(cfg).close(); - - // The safety property is unchanged: not one frame of the unreplayable slot - // may reach the server. - Assert.assertEquals("no frame may be replayed to a fresh server with a torn dictionary", - 0, handler.frames.get()); - assertUnreplayableSlotSetAside(); - } + assertSlotRecoversWithCompleteDictionary(); }); } @Test - public void testTornDictionaryOneIdGapFailsCleanly() throws Exception { - // One-id-gap boundary variant of - // testUnopenablePersistedDictStillGuardsAgainstReplayingDeltaFrames: the first - // replayed frame starts EXACTLY ONE id past the empty mirror -- the tightest - // gap the I/O-thread replay guard must still reject (deltaStart == - // sentDictCount + 1). It reaches that guard via the unopenable-dictionary path - // (deltaDictEnabled=false), NOT a size-0 openable dict, which now fails clean - // earlier in seedGlobalDictionaryFromPersisted. A one-entry-short tail is the - // common host-crash outcome, so this pins the guard's false-negative edge: a - // "deltaStart > size + 1" mutation would let this single-id gap through and - // null-pad the missing symbol on the server. + public void testTrimmedRegisteringFramesAreUnreplayableAndTheSlotIsSetAside() throws Exception { + // The genuinely unrecoverable slot -- and the only one left, now that a torn dictionary + // rebuilds from the frames that are still on disk. + // + // Here the frames that REGISTERED the early ids are gone for good: trim unlinked their + // whole segment once they were acked (modelled by deleting sf-initial.sfa, which is + // exactly what SegmentManager does). The dictionary is torn away too, so nothing + // anywhere still holds those ids, and the surviving frames' deltas start above them. + // Replaying would null-pad the hole and silently misattribute symbol values. + // + // So the slot is unreplayable -- and it is SET ASIDE, not allowed to brick the sender. + // Failing here is what bricked it: senderId is stable and a not-fully-drained slot is + // retained on close, so every retry re-recovered the same slot and threw again, and the + // application could not construct a Sender at all -- it could not even buffer new rows. assertMemoryLeak(() -> { - // Phase 1: each row introduces a new symbol (frame i carries deltaStart=i). try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) { int port = silent.getPort(); silent.start(); Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS)); + // Small segments so the frames roll into several files and the earliest ones + // can be trimmed away independently. String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir - + ";close_flush_timeout_millis=0;"; + + ";sf_max_bytes=256;close_flush_timeout_millis=0;"; try (Sender s1 = Sender.fromConfig(cfg)) { - for (int i = 0; i < 6; i++) { + for (int i = 0; i < 12; i++) { s1.table("m").symbol("s", "sym-" + i).longColumn("v", i).atNow(); s1.flush(); } } } - // Make the persisted dictionary UNOPENABLE (replace .symbol-dict with a - // directory, so both openRW and openCleanRW fail and the engine reports - // deltaDictEnabled=false), then stamp the watermark at FSN 0 so recovery - // replays from FSN 1 -- a frame with deltaStart=1. The mirror is unseeded - // (size 0), so id 0 is the single missing entry: deltaStart(1) == - // sentDictCount(0) + 1. No catch-up is sent, so any counted frame would be - // the gapped data frame. java.nio.file.Path slot = Paths.get(sfDir, "default"); + Assert.assertTrue("the frames must have rolled into more than one segment", + countSegmentFiles(slot) > 1); + // TRIM the segment that holds the earliest frames -- munmap + unlink is exactly what + // SegmentManager does once they are acked. Their symbols are now gone from disk. + java.nio.file.Files.delete(slot.resolve("sf-initial.sfa")); + // ...and the dictionary is torn away as well, so nothing holds them at all. java.nio.file.Path dict = slot.resolve(".symbol-dict"); - java.nio.file.Files.delete(dict); - java.nio.file.Files.createDirectory(dict); - writeAckWatermark(slot.resolve(".ack-watermark"), 0); + java.nio.file.Files.write(dict, + Arrays.copyOf(java.nio.file.Files.readAllBytes(dict), 8)); - // Phase 2: recover against a fresh counting server. The guard must fire on - // the very first replay frame (deltaStart 1 > recovered size 0) and fail - // terminally rather than send a frame that null-pads id 0. - CountingHandler handler = new CountingHandler(); + DictReconstructingHandler handler = new DictReconstructingHandler(); try (TestWebSocketServer good = new TestWebSocketServer(handler)) { int port = good.getPort(); good.start(); Assert.assertTrue(good.awaitStart(5, TimeUnit.SECONDS)); - String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";"; - LineSenderException terminal = null; - Sender s2 = Sender.fromConfig(cfg); - try { + // build() must SUCCEED -- on a fresh slot -- and the producer must keep working. + try (Sender s2 = Sender.fromConfig(cfg)) { + s2.table("m").symbol("s", "after-recovery").longColumn("v", 99).atNow(); + s2.flush(); long deadline = System.currentTimeMillis() + 10_000; - while (System.currentTimeMillis() < deadline && terminal == null) { - try { - s2.flush(); - Thread.sleep(20); - } catch (LineSenderException e) { - terminal = e; - } - } - } finally { - try { - s2.close(); - } catch (LineSenderException e) { - if (terminal == null) { - terminal = e; - } + while (System.currentTimeMillis() < deadline && handler.maxDictSize() < 1) { + Thread.sleep(20); } } - Assert.assertEquals("a one-id gap must still block replay to a fresh server", - 0, handler.frames.get()); - Assert.assertNotNull("a one-id-short dictionary must surface a terminal error", terminal); - Assert.assertTrue(terminal.getMessage(), - terminal.getMessage().contains("delta start 1 exceeds recovered dictionary size 0")); + assertUnreplayableSlotSetAside(); + // The producer's new data reaches the server, and not one symbol of the + // unreplayable slot does. + Assert.assertEquals("the producer must keep producing on its fresh slot", + Arrays.asList("after-recovery"), handler.dictSnapshot()); } }); } + private static int countSegmentFiles(java.nio.file.Path dir) { + java.io.File[] files = dir.toFile().listFiles(); + int n = 0; + if (files != null) { + for (java.io.File f : files) { + if (f.getName().endsWith(".sfa")) { + n++; + } + } + } + return n; + } + @Test public void testRecoveredSenderContinuesIngestingNewSymbols() throws Exception { // M2 regression: seedGlobalDictionaryFromPersisted resumes the producer's @@ -516,87 +443,6 @@ public void testUnopenablePersistedDictReplaysSelfSufficientFrameSequence() thro }); } - @Test - public void testUnopenablePersistedDictStillGuardsAgainstReplayingDeltaFrames() throws Exception { - // C1 regression: when a recovered disk slot's persisted dictionary cannot be - // OPENED (fd exhaustion, a read-only remount, ENOSPC -- simulated here by a - // .symbol-dict that is a DIRECTORY, so both openRW and openCleanRW fail), - // CursorSendEngine.isDeltaDictEnabled() returns false. The recorded frames - // are still DELTA frames, and replaying them against a fresh - // empty-dictionary server would null-pad the missing ids and SILENTLY - // corrupt the table. The torn-dictionary guard must fire regardless of - // deltaDictEnabled -- pre-fix it was gated on that very flag, so the - // corrupting frame sailed through unguarded. Unlike - // testTornDictionaryFailsCleanlyInsteadOfCorrupting (dict present but empty, - // deltaDictEnabled=true), here the dict is UNOPENABLE (deltaDictEnabled=false). - assertMemoryLeak(() -> { - // Phase 1: each row introduces a new symbol => frame i carries deltaStart=i. - try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) { - int port = silent.getPort(); - silent.start(); - Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS)); - String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir - + ";close_flush_timeout_millis=0;"; - try (Sender s1 = Sender.fromConfig(cfg)) { - for (int i = 0; i < 6; i++) { - s1.table("m").symbol("s", "sym-" + i).longColumn("v", i).atNow(); - s1.flush(); - } - } - } - - // Make the persisted dictionary UNOPENABLE: replace the .symbol-dict file - // with a directory of the same name so PersistedSymbolDict.open() returns - // null (both openRW and openCleanRW fail) and the engine reports - // deltaDictEnabled=false. Stamp the watermark at FSN 2 so replay starts - // at FSN 3 -- a frame whose delta starts at id 3, with ids 0..2 living - // only in the now-unreadable dictionary. - java.nio.file.Path slot = Paths.get(sfDir, "default"); - java.nio.file.Path dict = slot.resolve(".symbol-dict"); - java.nio.file.Files.delete(dict); - java.nio.file.Files.createDirectory(dict); - writeAckWatermark(slot.resolve(".ack-watermark"), 2); - - // Phase 2: recover against a fresh counting server. The guard must fire - // (frame deltaStart 3 > recovered dictionary size 0) and fail terminally - // rather than send a gapped frame that would corrupt the table. - CountingHandler handler = new CountingHandler(); - try (TestWebSocketServer good = new TestWebSocketServer(handler)) { - int port = good.getPort(); - good.start(); - Assert.assertTrue(good.awaitStart(5, TimeUnit.SECONDS)); - - String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";"; - LineSenderException terminal = null; - Sender s2 = Sender.fromConfig(cfg); - try { - long deadline = System.currentTimeMillis() + 10_000; - while (System.currentTimeMillis() < deadline && terminal == null) { - try { - s2.flush(); - Thread.sleep(20); - } catch (LineSenderException e) { - terminal = e; - } - } - } finally { - try { - s2.close(); - } catch (LineSenderException e) { - if (terminal == null) { - terminal = e; - } - } - } - Assert.assertEquals("no delta frame may be replayed when the persisted dictionary is unopenable", - 0, handler.frames.get()); - Assert.assertNotNull("an unopenable dictionary must surface a terminal error", terminal); - Assert.assertTrue(terminal.getMessage(), - terminal.getMessage().contains("symbol dictionary is incomplete")); - } - }); - } - @Test public void testCommitMessageDoesNotShipUnpersistedLeakedSymbol() throws Exception { // C3 regression: sendCommitMessage does NOT write-ahead-persist the @@ -1428,4 +1274,58 @@ private static boolean hasSegmentFile(java.nio.file.Path dir) { return false; } + + /** + * Phase 1 for the recovery tests: six frames, each introducing exactly one new symbol + * (sym-0 .. sym-5), left unacked on disk by a silent server and a fast close. + */ + private void recordSixDeltaFrames() throws Exception { + try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) { + int port = silent.getPort(); + silent.start(); + Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS)); + String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + + ";close_flush_timeout_millis=0;"; + try (Sender s1 = Sender.fromConfig(cfg)) { + for (int i = 0; i < 6; i++) { + s1.table("m").symbol("s", "sym-" + i).longColumn("v", i).atNow(); + s1.flush(); + } + } + } + } + + /** + * Recovers the slot against a fresh server and asserts the dictionary it ends up holding + * is COMPLETE and IN ORDER. + *

    + * This is the strong form, and it is the point: the fresh server starts with an empty + * dictionary, so ids the replayed frames' deltas start ABOVE can only come from a catch-up + * frame -- which can only carry them if the mirror was seeded from the frames still on + * disk. A dictionary that came back null-padded (the server's response to an id it has + * never seen) or shifted by one would fail here, and that is precisely the corruption the + * old guard condemned these slots to avoid. It never had to. + */ + private void assertSlotRecoversWithCompleteDictionary() throws Exception { + DictReconstructingHandler handler = new DictReconstructingHandler(); + try (TestWebSocketServer good = new TestWebSocketServer(handler)) { + int port = good.getPort(); + good.start(); + Assert.assertTrue(good.awaitStart(5, TimeUnit.SECONDS)); + String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";"; + try (Sender s2 = Sender.fromConfig(cfg)) { + long deadline = System.currentTimeMillis() + 10_000; + while (System.currentTimeMillis() < deadline && handler.maxDictSize() < 6) { + Thread.sleep(20); + } + s2.flush(); + } + Assert.assertEquals( + "every id is still held by a frame on disk, so the catch-up must rebuild the " + + "dictionary COMPLETE and gap-free -- not null-pad it", + Arrays.asList("sym-0", "sym-1", "sym-2", "sym-3", "sym-4", "sym-5"), + handler.dictSnapshot()); + } + } + } From 3b73bc815d0c560f9aec93a6e739b4dbbd53b77f Mon Sep 17 00:00:00 2001 From: glasstiger Date: Tue, 14 Jul 2026 21:08:12 +0100 Subject: [PATCH 60/96] Pin the producer's recovery seed on the unopenable-dictionary path The seed's ungating had no test: nothing wrote a NEW symbol after recovering a slot whose dictionary could not be opened, which is the only place the difference shows. It shows on the wire. Re-gate the seed and the resumed producer restarts its id space at 0, on top of ids the surviving frames already define -- the server's dictionary comes back as [after-recovery, sym-1, sym-2, sym-3, sym-4, sym-5] with sym-0 overwritten. Two symbols on one id, and the send loop's mirror still reads that id as sym-0, so mirror and producer disagree about what it MEANS. That is the misattribution the whole dictionary machinery exists to prevent. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../qwp/client/DeltaDictRecoveryTest.java | 51 +++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java index aaaeb66a..e4c36c30 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java @@ -287,6 +287,57 @@ private static int countSegmentFiles(java.nio.file.Path dir) { return n; } + @Test + public void testUnopenableDictSeedsTheProducerAboveTheRecoveredIds() throws Exception { + // The producer must resume ABOVE the ids the recovered frames already define, even when + // the dictionary could not be opened. + // + // seedGlobalDictionaryFromPersisted used to be gated on deltaDictEnabled, which is false + // exactly here -- so the producer restarted its id space at 0, on top of ids the + // surviving frames define. The send loop's mirror (rebuilt from those frames) still read + // id 0 as sym-0 while the producer meant something else by it: the two disagree about + // what an id MEANS, which is the whole failure mode the dictionary machinery exists to + // prevent. + // + // Visible on the wire: the new symbol must land ABOVE the recovered ones, not on top of + // sym-0. + assertMemoryLeak(() -> { + recordSixDeltaFrames(); + java.nio.file.Path slot = Paths.get(sfDir, "default"); + java.nio.file.Path dict = slot.resolve(".symbol-dict"); + java.nio.file.Files.delete(dict); + java.nio.file.Files.createDirectory(dict); + writeAckWatermark(slot.resolve(".ack-watermark"), 2); + + DictReconstructingHandler handler = new DictReconstructingHandler(); + try (TestWebSocketServer good = new TestWebSocketServer(handler)) { + int port = good.getPort(); + good.start(); + Assert.assertTrue(good.awaitStart(5, TimeUnit.SECONDS)); + String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";"; + try (Sender s2 = Sender.fromConfig(cfg)) { + long deadline = System.currentTimeMillis() + 10_000; + while (System.currentTimeMillis() < deadline && handler.maxDictSize() < 6) { + Thread.sleep(20); + } + // ...and now the resumed producer introduces a symbol of its own. + s2.table("m").symbol("s", "after-recovery").longColumn("v", 99).atNow(); + s2.flush(); + deadline = System.currentTimeMillis() + 10_000; + while (System.currentTimeMillis() < deadline && handler.maxDictSize() < 7) { + Thread.sleep(20); + } + } + Assert.assertEquals( + "the resumed producer must take the NEXT id, not reuse id 0 -- reusing it " + + "puts two symbols on one id and silently misattributes values", + Arrays.asList("sym-0", "sym-1", "sym-2", "sym-3", "sym-4", "sym-5", + "after-recovery"), + handler.dictSnapshot()); + } + }); + } + @Test public void testRecoveredSenderContinuesIngestingNewSymbols() throws Exception { // M2 regression: seedGlobalDictionaryFromPersisted resumes the producer's From 98a4ac033db47aac1e422686b3b225fae17341cd Mon Sep 17 00:00:00 2001 From: glasstiger Date: Tue, 14 Jul 2026 23:32:11 +0100 Subject: [PATCH 61/96] Don't key the cap-gap episode off nanoTime's sign sendDictCatchUp tested `catchUpCapGapFirstNanos < 0` to decide whether a symbol-dictionary cap-gap episode was already open. A System.nanoTime() instant is only meaningful as a difference: its origin is arbitrary and the spec permits negative values, so its sign cannot encode state. CursorWebSocketSendLoopCatchUpAlignmentTest backdates the episode anchor by two hours to prove that a cap gap outliving the escalation window latches a terminal. On Linux nanoTime() is nanos-since-boot, so on a freshly booted CI agent it sits far below two hours of nanos and the backdated anchor came out negative. The loop then read that real anchor as "unset", re-anchored it to now on every strike and pinned episodeNanos at ~0: the dwell was never satisfied, no terminal latched, and the test failed. It fails on every machine with under two hours of uptime and on no long-lived dev box, which is why only CI saw it. Key the episode off the strike count instead, exactly as recordHeadRejectionStrike keys its own off poisonStrikes. -1 survives as a debug marker but no longer decides anything, so no state rides on a timestamp's sign. Add testCapGapEpisodeWithANegativeAnchorStillEscalates, which plants the negative anchor directly and so pins the sentinel whatever the host's uptime; the existing test only reddens where uptime happens to fall under its two-hour backdate. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../sf/cursor/CursorWebSocketSendLoop.java | 34 +++++++--- ...WebSocketSendLoopCatchUpAlignmentTest.java | 65 +++++++++++++++++++ 2 files changed, 89 insertions(+), 10 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java index cf3f1637..9d198fff 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java @@ -301,15 +301,19 @@ public final class CursorWebSocketSendLoop implements QuietCloseable { // nor resets it. NOT reset per connection -- it measures the cap-gap episode // across reconnects so a persistent gap eventually latches. I/O-thread-only. private int catchUpCapGapAttempts; - // System.nanoTime() of the FIRST cap gap of the current episode, or -1 when no - // episode is open. Anchors the escalation dwell. Reset together with - // catchUpCapGapAttempts on a successful catch-up, so a node that accepts the - // dictionary ends the episode outright. Anchoring at the FIRST gap (not at loop - // entry, and not refreshed per attempt) is what keeps a TRANSIENT from burning the - // terminal budget: a transient never reaches the catch-up, so it neither increments - // the counter nor moves the anchor -- it can only lengthen the wall clock, which - // alone can never latch the terminal because the strike count still has to be - // satisfied. I/O-thread-only, like catchUpCapGapAttempts. + // System.nanoTime() of the FIRST cap gap of the current episode. Anchors the + // escalation dwell. Reset together with catchUpCapGapAttempts on a successful + // catch-up, so a node that accepts the dictionary ends the episode outright. + // Anchoring at the FIRST gap (not at loop entry, and not refreshed per attempt) is + // what keeps a TRANSIENT from burning the terminal budget: a transient never reaches + // the catch-up, so it neither increments the counter nor moves the anchor -- it can + // only lengthen the wall clock, which alone can never latch the terminal because the + // strike count still has to be satisfied. I/O-thread-only, like catchUpCapGapAttempts. + // + // -1 marks "no episode open" for a debugger or a heap dump, but it is NOT the test + // for one -- catchUpCapGapAttempts == 0 is (see sendDictCatchUp). A nanoTime instant + // is only meaningful as a difference: its origin is arbitrary and the spec permits + // negative values, so no state may ride on this field's sign. private long catchUpCapGapFirstNanos = -1L; // True once a real ring frame (data or commit) has been sent on the CURRENT // connection, as opposed to only the dictionary catch-up. The catch-up consumes @@ -2442,8 +2446,18 @@ private int sendDictCatchUp() { // an ordinary rolling restart of the larger-cap node. Escalating on the // count alone would therefore hard-fail a live store-and-forward // producer during a routine cluster operation. + // + // The STRIKE COUNT -- never the anchor's sign -- says whether an episode is + // already open. A System.nanoTime() instant is only meaningful as a + // difference: its origin is arbitrary and the spec permits negative values, + // so a sign test cannot tell an unset anchor from a real one. Wherever it + // misreads a real anchor as unset it re-anchors to now on EVERY strike, + // pinning episodeNanos at ~0, and the dwell can then never be satisfied -- + // the terminal never latches, however long the gap truly persists. + // recordHeadRejectionStrike keys its episode off poisonStrikes for the same + // reason. long nowNanos = System.nanoTime(); - if (catchUpCapGapFirstNanos < 0) { + if (catchUpCapGapAttempts == 0) { catchUpCapGapFirstNanos = nowNanos; } catchUpCapGapAttempts++; diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java index 707ef5a8..4e00e1e6 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java @@ -330,6 +330,71 @@ public void testCatchUpCapGapStrikesAloneDoNotLatchWithinTheEscalationWindow() t }); } + @Test + public void testCapGapEpisodeWithANegativeAnchorStillEscalates() throws Exception { + // A cap-gap episode anchored at a NEGATIVE nanoTime instant must escalate like any + // other. A System.nanoTime() value is only meaningful as a difference -- its origin + // is arbitrary and the spec permits negative values -- so no state may ride on the + // anchor's sign. sendDictCatchUp once tested catchUpCapGapFirstNanos < 0 to mean "no + // episode open": that read a negative anchor as unset, re-anchored it to now on every + // strike and pinned episodeNanos at ~0, so the dwell was never satisfied and the + // terminal could never latch, however long the cap gap truly persisted. + // + // That is what reddened CI. The sibling test above backdates the anchor two hours, + // and on Linux nanoTime() is nanos-since-boot: on a CI agent up ten minutes it is + // ~6e11, so "two hours ago" comes out ~ -6.6e12 -- negative. The defect therefore + // only surfaced where uptime is under that backdate: every fresh CI agent, and never + // a long-lived dev box (which is why it passed locally). Planting the negative anchor + // directly pins the sentinel on ANY machine, whatever its uptime. + TestUtils.assertMemoryLeak(() -> { + Field maxField = CursorWebSocketSendLoop.class.getDeclaredField("MAX_CATCHUP_CAP_GAP_ATTEMPTS"); + maxField.setAccessible(true); + int maxAttempts = maxField.getInt(null); + CatchUpCapturingClient client = new CatchUpCapturingClient(160); + try (CursorSendEngine engine = newEngine()) { + // A one-hour dwell, against an anchor two hours back: satisfied on elapsed, + // but only if the negative anchor survives to the subtraction. + CursorWebSocketSendLoop loop = newLoop(engine, client, 3_600_000L); + try { + seedMirror(loop, TestUtils.repeat("x", 200)); + // Satisfy the strike half of the AND, one short of the budget. + for (int i = 1; i < maxAttempts; i++) { + try { + invokeSetWireBaselineWithCatchUp(loop, engine.ackedFsn() + 1L); + fail("cap gap must raise a retriable CatchUpSendException (attempt " + i + ')'); + } catch (InvocationTargetException e) { + assertEquals("CatchUpSendException", e.getCause().getClass().getSimpleName()); + } + loop.checkError(); // dwell unmet => retriable, whatever the count + } + // The episode began two hours ago on a machine booted minutes ago. + Field anchor = CursorWebSocketSendLoop.class.getDeclaredField("catchUpCapGapFirstNanos"); + anchor.setAccessible(true); + anchor.setLong(loop, -TimeUnit.HOURS.toNanos(2)); + + // Both halves now hold, so this strike must latch the terminal. + try { + invokeSetWireBaselineWithCatchUp(loop, engine.ackedFsn() + 1L); + fail("the escalating cap gap must still raise CatchUpSendException"); + } catch (InvocationTargetException e) { + assertEquals("CatchUpSendException", e.getCause().getClass().getSimpleName()); + } + try { + loop.checkError(); + fail("a cap-gap episode anchored at a negative nanoTime instant must still " + + "escalate -- the anchor's sign carries no meaning"); + } catch (LineSenderException terminal) { + assertTrue("terminal must name the exhausted catch-up cap gap: " + terminal.getMessage(), + terminal.getMessage().contains("during catch-up") + && terminal.getMessage().contains("must be resent")); + } + } finally { + loop.close(); + } + } + }); + } + @Test public void testTransientCatchUpFailureDoesNotBurnTheCapGapBudget() throws Exception { // A TRANSIENT catch-up failure (the wire drops mid-catch-up -- a flapping LB, a From 63dc52a55b096e507b42a12ec9efe0f394e4ae91 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Wed, 15 Jul 2026 01:30:19 +0100 Subject: [PATCH 62/96] Free the recovery mirror on a ctor seed throw The CursorWebSocketSendLoop constructor mallocs the sent-dictionary mirror from the persisted dictionary's prefix, then extends it from the surviving frames' deltas. If that second step throws -- a native realloc OOM, or the MAX_SENT_DICT_BYTES ceiling -- the throw leaves the loop unpublished, so neither ensureConnected's catch nor BackgroundDrainer's finally can close() it, and the already-malloc'd mirror leaks. Free it on any throw, mirroring the loopNeverRan free in close()/ioLoop's exit. Also close two test-coverage gaps the review flagged: - The send loop's torn-dict guard (deltaStart > sentDictCount in trySendOne) is the drainer path's sole defense against replaying a frame whose symbols were trimmed away; the foreground path is quarantined earlier, so this fire direction was unexercised. Cover it at the send-loop level against a stub client (the drainer integration path cannot connect on darwin), asserting zero frames ship and the terminal latches. - quarantineTornSlot must fail loudly rather than drop bytes when it cannot set a slot aside. Saturate every quarantine candidate and assert build() throws and preserves the slot on disk. A TestOnly seam (forceMirrorSeedFailureForTest) makes the ctor seed throw deterministically; the unreplayable-slot setup is now a shared helper. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../sf/cursor/CursorWebSocketSendLoop.java | 39 +++- .../qwp/client/DeltaDictRecoveryTest.java | 103 +++++--- ...CursorWebSocketSendLoopMirrorLeakTest.java | 89 +++++++ ...sorWebSocketSendLoopTornDictGuardTest.java | 220 ++++++++++++++++++ 4 files changed, 421 insertions(+), 30 deletions(-) create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopTornDictGuardTest.java diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java index 9d198fff..c705aaac 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java @@ -197,6 +197,14 @@ public final class CursorWebSocketSendLoop implements QuietCloseable { * Throttle "reconnect attempt N failed" WARN logs to one per 5 s. */ private static final long RECONNECT_LOG_THROTTLE_NANOS = 5_000_000_000L; + // Test seam: when true, appendSymbolToMirror throws instead of appending, + // simulating the native realloc OOM (or MAX_SENT_DICT_BYTES ceiling) that + // ensureSentDictCapacity can raise while the constructor seeds the recovery + // mirror. Lets a test prove the constructor frees the already-malloc'd mirror on + // such a throw rather than leaking it. Production never sets it; volatile only so + // a test thread's write is visible to the loop under test. + @TestOnly + static volatile boolean forceMirrorSeedFailureForTest; // Pre-converted to nanos for the comparison in sendDurableAckKeepaliveIfDue. // Zero or negative disables the keepalive entirely. private final long durableAckKeepaliveIntervalNanos; @@ -717,11 +725,30 @@ public CursorWebSocketSendLoop(WebSocketClient client, CursorSendEngine engine, // nothing and cost a catch-up frame on every connection -- full-dict slots must stay // catch-up-free. if (engine.recoveredMaxSymbolDeltaStart() > 0L) { - ObjList fromFrames = new ObjList<>(); - if (engine.collectReplaySymbolsAbove(sentDictCount, fromFrames) >= 0) { - for (int i = 0, n = fromFrames.size(); i < n; i++) { - appendSymbolToMirror(fromFrames.getQuick(i)); + // The prefix seed above may already have malloc'd the mirror, and + // appendSymbolToMirror -> ensureSentDictCapacity can still throw here (a + // native realloc OOM, or the MAX_SENT_DICT_BYTES ceiling). Such a throw + // propagates out of the constructor, so the half-built loop is never + // assigned to a reference -- neither ensureConnected's catch nor + // BackgroundDrainer's finally can close() it -- and the mirror would leak. + // Free it on any throw so the constructor leaves nothing behind, mirroring + // the loopNeverRan free in close() / ioLoop's exit. + try { + ObjList fromFrames = new ObjList<>(); + if (engine.collectReplaySymbolsAbove(sentDictCount, fromFrames) >= 0) { + for (int i = 0, n = fromFrames.size(); i < n; i++) { + appendSymbolToMirror(fromFrames.getQuick(i)); + } } + } catch (Throwable t) { + if (sentDictBytesAddr != 0) { + Unsafe.free(sentDictBytesAddr, sentDictBytesCapacity, MemoryTag.NATIVE_DEFAULT); + sentDictBytesAddr = 0; + sentDictBytesCapacity = 0; + sentDictBytesLen = 0; + sentDictCount = 0; + } + throw t; } } this.fsnAtZero = fsnAtZero; @@ -2299,6 +2326,10 @@ private void accumulateSentDict(long payloadAddr, int payloadLen, int deltaStart * construction; {@link #accumulateSentDict} extends it from live frames thereafter. */ private void appendSymbolToMirror(CharSequence symbol) { + if (forceMirrorSeedFailureForTest) { + // Simulate a native realloc OOM on the seed path (see the field). + throw new LineSenderException("simulated mirror seed allocation failure (test only)"); + } int utf8Len = Utf8s.utf8Bytes(symbol); int wireLen = NativeBufferWriter.varintSize(utf8Len) + utf8Len; ensureSentDictCapacity((long) sentDictBytesLen + wireLen); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java index e4c36c30..e8ff1e99 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java @@ -223,32 +223,7 @@ public void testTrimmedRegisteringFramesAreUnreplayableAndTheSlotIsSetAside() th // retained on close, so every retry re-recovered the same slot and threw again, and the // application could not construct a Sender at all -- it could not even buffer new rows. assertMemoryLeak(() -> { - try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) { - int port = silent.getPort(); - silent.start(); - Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS)); - // Small segments so the frames roll into several files and the earliest ones - // can be trimmed away independently. - String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir - + ";sf_max_bytes=256;close_flush_timeout_millis=0;"; - try (Sender s1 = Sender.fromConfig(cfg)) { - for (int i = 0; i < 12; i++) { - s1.table("m").symbol("s", "sym-" + i).longColumn("v", i).atNow(); - s1.flush(); - } - } - } - - java.nio.file.Path slot = Paths.get(sfDir, "default"); - Assert.assertTrue("the frames must have rolled into more than one segment", - countSegmentFiles(slot) > 1); - // TRIM the segment that holds the earliest frames -- munmap + unlink is exactly what - // SegmentManager does once they are acked. Their symbols are now gone from disk. - java.nio.file.Files.delete(slot.resolve("sf-initial.sfa")); - // ...and the dictionary is torn away as well, so nothing holds them at all. - java.nio.file.Path dict = slot.resolve(".symbol-dict"); - java.nio.file.Files.write(dict, - Arrays.copyOf(java.nio.file.Files.readAllBytes(dict), 8)); + writeAndTearUnreplayableSlot(); DictReconstructingHandler handler = new DictReconstructingHandler(); try (TestWebSocketServer good = new TestWebSocketServer(handler)) { @@ -287,6 +262,82 @@ private static int countSegmentFiles(java.nio.file.Path dir) { return n; } + @Test + public void testQuarantineFailsLoudlyWhenAllSlotNamesSaturated() throws Exception { + // M2 regression: when a recovered slot is genuinely unreplayable, build() sets + // it aside (quarantineTornSlot) and starts the producer on a fresh slot. But if + // it cannot free the slot name -- here every default.unreplayable- candidate + // up to MAX_QUARANTINE_SLOT_ATTEMPTS (64) already exists -- it MUST fail LOUDLY: + // throw and leave the slot's bytes on disk for a manual resend, never silently + // drop data. Only the happy rename path was covered before. + assertMemoryLeak(() -> { + writeAndTearUnreplayableSlot(); + // Saturate every quarantine candidate so quarantineTornSlot's rename loop + // finds no free name. + for (int i = 0; i < 64; i++) { + java.nio.file.Files.createDirectories(Paths.get(sfDir, "default.unreplayable-" + i)); + } + + java.nio.file.Path slot = Paths.get(sfDir, "default"); + try (TestWebSocketServer good = new TestWebSocketServer(new DictReconstructingHandler())) { + int port = good.getPort(); + good.start(); + Assert.assertTrue(good.awaitStart(5, TimeUnit.SECONDS)); + String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";"; + Sender s = null; + try { + s = Sender.fromConfig(cfg); + Assert.fail("build() must throw when the unreplayable slot cannot be set aside"); + } catch (LineSenderException expected) { + Assert.assertTrue("unexpected message: " + expected.getMessage(), + expected.getMessage().contains("too many quarantined slots already under") + && expected.getMessage().contains("moved or removed by hand")); + } finally { + if (s != null) { + s.close(); + } + } + } + // The unreplayable slot's bytes must survive on disk for a manual resend -- + // the guard fails loudly rather than dropping data. + Assert.assertTrue("the slot dir must be preserved", java.nio.file.Files.exists(slot)); + Assert.assertTrue("the slot's segment data must be preserved", + countSegmentFiles(slot) >= 1); + }); + } + + // Writes 12 delta frames (each introducing a new symbol) into the default slot + // across several small segments, then makes the slot GENUINELY unreplayable: trims + // the segment holding the earliest ids (munmap + unlink, exactly what SegmentManager + // does once they are acked) and tears the .symbol-dict down to its header, so the + // surviving frames' deltas start above ids nothing on disk still holds. Recovering + // such a slot throws UnreplayableSlotException. + private void writeAndTearUnreplayableSlot() throws Exception { + try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) { + int port = silent.getPort(); + silent.start(); + Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS)); + // Small segments so the frames roll into several files and the earliest ones + // can be trimmed away independently. + String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + + ";sf_max_bytes=256;close_flush_timeout_millis=0;"; + try (Sender s1 = Sender.fromConfig(cfg)) { + for (int i = 0; i < 12; i++) { + s1.table("m").symbol("s", "sym-" + i).longColumn("v", i).atNow(); + s1.flush(); + } + } + } + java.nio.file.Path slot = Paths.get(sfDir, "default"); + Assert.assertTrue("the frames must have rolled into more than one segment", + countSegmentFiles(slot) > 1); + // TRIM the segment that holds the earliest frames. + java.nio.file.Files.delete(slot.resolve("sf-initial.sfa")); + // ...and tear the dictionary away as well, so nothing holds those ids at all. + java.nio.file.Path dict = slot.resolve(".symbol-dict"); + java.nio.file.Files.write(dict, Arrays.copyOf(java.nio.file.Files.readAllBytes(dict), 8)); + } + @Test public void testUnopenableDictSeedsTheProducerAboveTheRecoveredIds() throws Exception { // The producer must resume ABOVE the ids the recovered frames already define, even when diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopMirrorLeakTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopMirrorLeakTest.java index 5cff81ba..8697eaee 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopMirrorLeakTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopMirrorLeakTest.java @@ -25,6 +25,7 @@ package io.questdb.client.test.cutlass.qwp.client.sf.cursor; import io.questdb.client.Sender; +import io.questdb.client.cutlass.line.LineSenderException; import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine; import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorWebSocketSendLoop; import io.questdb.client.cutlass.qwp.client.sf.cursor.PersistedSymbolDict; @@ -156,6 +157,63 @@ public void testRecycledLoopReSeedsMirrorFromPersistedDict() throws Exception { } } + @Test + public void testCtorFreesSeededMirrorWhenFrameSeedThrows() throws Exception { + // C1 regression: the constructor seeds the recovery mirror in TWO steps -- a + // malloc from the persisted dictionary's intact prefix, then an extension from + // the surviving frames' own deltas (appendSymbolToMirror). If that second step + // throws (a native realloc OOM, or the MAX_SENT_DICT_BYTES ceiling), the throw + // leaves the constructor with the object unpublished, so neither + // ensureConnected's catch nor BackgroundDrainer's finally can ever close() it -- + // and the already-malloc'd prefix mirror leaks. The constructor must free it on + // the throw. Pre-fix, the mirror leaks here and assertMemoryLeak fails. + Path sfDir = Files.createTempDirectory("qwp-mirror-ctor-throw"); + try { + // A torn-dict SUBSET: three delta frames a@0,b@1,c@2 survive on disk, but the + // .symbol-dict is rewritten to hold only [a,b] (a host-crash tail tear). On + // recovery pd.size()==2 seeds (mallocs) the mirror, then the frame-seed + // rebuilds c@2 from the surviving frame -- the append the fault interrupts. + populateThreeFrameSlot(sfDir); + Path slot = sfDir.resolve("default"); + try (PersistedSymbolDict torn = PersistedSymbolDict.openClean(slot.toString())) { + Assert.assertNotNull(torn); + torn.appendSymbol("a"); + torn.appendSymbol("b"); + Assert.assertEquals(2, torn.size()); + } + + assertMemoryLeak(() -> { + try (CursorSendEngine engine = new CursorSendEngine(slot.toString(), 4096)) { + PersistedSymbolDict pd = engine.getPersistedSymbolDict(); + Assert.assertNotNull("recovery must open the torn subset dict", pd); + Assert.assertEquals("prefix seed must malloc a 2-entry mirror", 2, pd.size()); + Assert.assertTrue("the frame-seed path must run (frames out-reach the dict)", + engine.recoveredMaxSymbolDeltaStart() > 0L); + + setMirrorSeedFault(true); + try { + new CursorWebSocketSendLoop( + null, engine, 0, 1_000_000L, + () -> { + throw new IOException("no reconnect in this test"); + }, + 0, 0, 1); + Assert.fail("ctor must propagate the injected mirror-seed failure"); + } catch (LineSenderException expected) { + Assert.assertTrue("unexpected message: " + expected.getMessage(), + expected.getMessage().contains("simulated mirror seed allocation failure")); + } finally { + setMirrorSeedFault(false); + } + // The outer assertMemoryLeak proves the prefix-seeded mirror the ctor + // malloc'd was freed on the throw -- pre-fix it leaks here. + } + }); + } finally { + rmDir(sfDir); + } + } + // Constructs a recovery send loop but does NOT start it: the ctor seeds the // catch-up mirror synchronously, which is all these tests observe. The // reconnect factory is never invoked. @@ -189,12 +247,43 @@ private static void populateRecoverableSlot(Path sfDir) throws Exception { } } + // Three delta frames a@0, b@1, c@2, nothing acked, so all three survive and + // replay from frame 0. Paired with a dictionary truncated to [a,b], this is a + // torn-dict SUBSET whose recovery drives the constructor's frame-seed path. + private static void populateThreeFrameSlot(Path sfDir) throws Exception { + try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) { + int port = silent.getPort(); + silent.start(); + Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS)); + String cfg = "ws::addr=localhost:" + port + + ";sf_dir=" + sfDir + + ";close_flush_timeout_millis=0;"; + try (Sender s = Sender.fromConfig(cfg)) { + s.table("m").symbol("s", "a").longColumn("v", 0).atNow(); + s.flush(); + s.table("m").symbol("s", "b").longColumn("v", 1).atNow(); + s.flush(); + s.table("m").symbol("s", "c").longColumn("v", 2).atNow(); + s.flush(); + } + } + } + private static int readInt(CursorWebSocketSendLoop loop, String name) throws Exception { Field f = CursorWebSocketSendLoop.class.getDeclaredField(name); f.setAccessible(true); return f.getInt(loop); } + // Toggles the loop's @TestOnly mirror-seed fault flag. Reflection because the + // flag is package-private in the production package (this test is in a sibling + // test package), the same non-reflective-path-unavailable reason readInt uses. + private static void setMirrorSeedFault(boolean value) throws Exception { + Field f = CursorWebSocketSendLoop.class.getDeclaredField("forceMirrorSeedFailureForTest"); + f.setAccessible(true); + f.setBoolean(null, value); + } + private static void rmDir(Path dir) throws IOException { TestUtils.removeTmpDirRec(dir == null ? null : dir.toString()); } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopTornDictGuardTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopTornDictGuardTest.java new file mode 100644 index 00000000..7dabc741 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopTornDictGuardTest.java @@ -0,0 +1,220 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client.sf.cursor; + +import io.questdb.client.DefaultHttpClientConfiguration; +import io.questdb.client.Sender; +import io.questdb.client.cutlass.http.client.WebSocketClient; +import io.questdb.client.cutlass.line.LineSenderException; +import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine; +import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorWebSocketSendLoop; +import io.questdb.client.network.PlainSocketFactory; +import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer; +import io.questdb.client.test.tools.TestUtils; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.io.File; +import java.io.IOException; +import java.lang.reflect.Field; +import java.lang.reflect.Method; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Arrays; +import java.util.concurrent.TimeUnit; + +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; + +/** + * White-box coverage of the send loop's torn-dictionary guard + * ({@code trySendOne}: {@code deltaStart > sentDictCount}) firing on a genuine gap. + *

    + * That guard is the drainer path's SOLE defense: an orphan drainer adopts a slot + * without running {@code Sender.build()}'s seed-time guard, so on a slot whose + * registering frames were trimmed away and whose {@code .symbol-dict} was torn, the + * send loop must detect the gap itself, ship ZERO frames, and latch a terminal -- + * never null-pad the hole on the server. The foreground sender is always quarantined + * earlier (at build time), so this fire direction runs only here. Driven at the send + * loop level against a frame-counting stub client so it exercises the guard + * deterministically, without a real network connection. + */ +public class CursorWebSocketSendLoopTornDictGuardTest { + + private String sfDir; + + @Before + public void setUp() { + sfDir = TestUtils.createTmpDir("qdb-torn-dict-guard-"); + } + + @After + public void tearDown() { + TestUtils.removeTmpDir(sfDir); + } + + @Test + public void testGuardFiresOnGenuineGapAndShipsNoFrame() throws Exception { + assertMemoryLeak(() -> { + writeAndTearGappedSlot(); + + CountingClient client = new CountingClient(); + try (CursorSendEngine engine = new CursorSendEngine(sfDir + "/default", 16_384)) { + // The persisted dict was torn away and the registering frames trimmed, so + // the mirror cannot be seeded from either source: sentDictCount stays 0 + // while the first surviving frame's delta starts above it. + CursorWebSocketSendLoop loop = newLoop(engine, client); + try { + setRunning(loop); + // Position at the first unsent frame exactly as the I/O loop does + // before its first send (no catch-up: the mirror is empty). + invoke(loop, "positionCursorForStart"); + + boolean sent = (Boolean) invoke(loop, "trySendOne"); + + Assert.assertFalse("the torn-dict guard must refuse to send the gapped frame", sent); + Assert.assertEquals("no frame may reach the server through a gap", + 0, client.framesSent); + try { + loop.checkError(); + Assert.fail("the guard must latch a terminal error"); + } catch (LineSenderException e) { + Assert.assertTrue("unexpected terminal: " + e.getMessage(), + e.getMessage().contains("incomplete") + && e.getMessage().contains("resend required")); + } + } finally { + loop.close(); + } + } + }); + } + + private static int countSegmentFiles(Path dir) { + File[] files = dir.toFile().listFiles(); + int n = 0; + if (files != null) { + for (File f : files) { + if (f.getName().endsWith(".sfa")) { + n++; + } + } + } + return n; + } + + private static Object invoke(CursorWebSocketSendLoop loop, String method) throws Exception { + Method m = CursorWebSocketSendLoop.class.getDeclaredMethod(method); + m.setAccessible(true); + return m.invoke(loop); + } + + private static void setRunning(CursorWebSocketSendLoop loop) throws Exception { + Field f = CursorWebSocketSendLoop.class.getDeclaredField("running"); + f.setAccessible(true); + f.setBoolean(loop, true); + } + + // Constructs a recovery send loop that is never started -- the test drives + // positionCursorForStart + trySendOne directly. The reconnect factory throws + // because no reconnect is expected before the guard latches its terminal. + private CursorWebSocketSendLoop newLoop(CursorSendEngine engine, WebSocketClient client) { + return new CursorWebSocketSendLoop( + client, engine, 0, 1_000_000L, + () -> { + throw new IOException("no reconnect in this test"); + }, + 0, 0, 1); + } + + // Writes 12 delta frames (each a new symbol) into the default slot across several + // small segments, then makes the slot carry a GENUINE gap: trims the segment + // holding the earliest ids (munmap + unlink, as SegmentManager does once they are + // acked) and tears the .symbol-dict down to its header. The surviving frames' deltas + // then start above ids nothing on disk still holds, so the mirror cannot be rebuilt. + private void writeAndTearGappedSlot() throws Exception { + try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) { + int port = silent.getPort(); + silent.start(); + Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS)); + String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + + ";sf_max_bytes=256;close_flush_timeout_millis=0;"; + try (Sender s = Sender.fromConfig(cfg)) { + for (int i = 0; i < 12; i++) { + s.table("m").symbol("s", "sym-" + i).longColumn("v", i).atNow(); + s.flush(); + } + } + } + Path slot = Paths.get(sfDir, "default"); + Assert.assertTrue("the frames must have rolled into more than one segment", + countSegmentFiles(slot) > 1); + Files.delete(slot.resolve("sf-initial.sfa")); + Path dict = slot.resolve(".symbol-dict"); + Files.write(dict, Arrays.copyOf(Files.readAllBytes(dict), 8)); + } + + // Frame-counting stub transport: completes no real I/O. If the guard ever lets a + // gapped frame through, sendBinary bumps framesSent and the test fails. + private static final class CountingClient extends WebSocketClient { + private int framesSent; + + CountingClient() { + super(DefaultHttpClientConfiguration.INSTANCE, PlainSocketFactory.INSTANCE); + } + + @Override + public int getServerMaxBatchSize() { + return 16_384; + } + + @Override + public int getServerQwpVersion() { + return 1; + } + + @Override + public void sendBinary(long dataPtr, int length) { + framesSent++; + } + + @Override + protected void ioWait(int timeout, int op) { + } + + @Override + protected void setupIoWait() { + } + } + + private static class SilentHandler implements TestWebSocketServer.WebSocketServerHandler { + @Override + public void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] data) { + // never acks -- the sender leaves everything unacked in the slot + } + } +} From 7299925df7c5143e5a822d3dc36aa4359a730edd Mon Sep 17 00:00:00 2001 From: glasstiger Date: Wed, 15 Jul 2026 01:47:39 +0100 Subject: [PATCH 63/96] Correct stale and stranded symbol-dict comments Comment-only follow-ups from the delta symbol-dict review; no behaviour change. - quarantineTornSlot marks the set-aside slot .failed unconditionally, so the orphan drainer never re-examines it. Fix the two comments that claimed the drainer could still deliver a quarantined slot; state that it is a human-in-the-loop copy whose bytes are kept for a manual resend. - sendCommitMessage's full-dict-mode comment claimed the commit frame stays self-sufficient, but the prior flush reset currentBatchMaxSymbolId to -1, so the frame carries an empty delta. Say so -- it is harmless, the preceding data frames already registered the dictionary. - Move three javadoc blocks stranded above the wrong method back onto their own: MmapSegment.maxSymbolDeltaEnd, SegmentRing.maxSymbolDeltaEnd and Sender.closeFlushTimeoutMillis were left undocumented, and the stranded @param names no longer matched the method beneath them. - Drop a double blank line in CursorSendEngine. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../main/java/io/questdb/client/Sender.java | 57 +++++++++-------- .../qwp/client/QwpWebSocketSender.java | 4 +- .../client/sf/cursor/CursorSendEngine.java | 1 - .../qwp/client/sf/cursor/MmapSegment.java | 62 +++++++++---------- .../qwp/client/sf/cursor/SegmentRing.java | 16 ++--- 5 files changed, 73 insertions(+), 67 deletions(-) diff --git a/core/src/main/java/io/questdb/client/Sender.java b/core/src/main/java/io/questdb/client/Sender.java index b674cf1b..23fdf6a8 100644 --- a/core/src/main/java/io/questdb/client/Sender.java +++ b/core/src/main/java/io/questdb/client/Sender.java @@ -1013,8 +1013,9 @@ final class LineSenderBuilder { private static final int PROTOCOL_UDP = 3; private static final int PROTOCOL_WEBSOCKET = 2; // Suffix for a slot set aside by quarantineTornSlot. Deliberately NOT the - // sender's own slot name, so OrphanScanner sees the quarantined copy and the - // orphan drainer can still deliver it if its frames turn out to be replayable. + // sender's own slot name, so a restarted sender does not re-adopt it as its own; + // quarantineTornSlot then marks it .failed, so the orphan drainer skips it too and + // the bytes stay put for a human to inspect and resend. private static final String QUARANTINE_SLOT_SUFFIX = ".unreplayable-"; private final ObjList hosts = new ObjList<>(); private final IntList ports = new IntList(); @@ -1757,23 +1758,6 @@ public Sender build() { return sender; } - /** - * close() drain timeout in milliseconds. The sender's {@code close()} - * method blocks up to this many millis waiting for the server to ACK - * every batch already published into the engine before shutting down - * the I/O loop. Default {@code 60000} (60 s) -- generous enough to - * survive real-workload backlogs (slow consumers, catch-up replicas, - * chunky payloads on small server send buffers) without silently - * dropping unacked rows; callers that need a longer pre-close wait - * for a specific submission can call - * {@link Sender#drain(long)} explicitly before close(). - *

    - * Set to {@code 0} or {@code -1} to opt out — close() will not wait - * at all (fast close). Pending data is then lost in memory mode and - * recovered by the next sender in SF mode. - *

    - * WebSocket transport only. - */ /** * Minimum wall-clock time (millis) a symbol-dictionary catch-up CAP GAP must * persist before the sender gives up on it and fails. @@ -1805,6 +1789,23 @@ public LineSenderBuilder catchUpCapGapMinEscalationWindowMillis(long millis) { return this; } + /** + * close() drain timeout in milliseconds. The sender's {@code close()} + * method blocks up to this many millis waiting for the server to ACK + * every batch already published into the engine before shutting down + * the I/O loop. Default {@code 60000} (60 s) -- generous enough to + * survive real-workload backlogs (slow consumers, catch-up replicas, + * chunky payloads on small server send buffers) without silently + * dropping unacked rows; callers that need a longer pre-close wait + * for a specific submission can call + * {@link Sender#drain(long)} explicitly before close(). + *

    + * Set to {@code 0} or {@code -1} to opt out — close() will not wait + * at all (fast close). Pending data is then lost in memory mode and + * recovered by the next sender in SF mode. + *

    + * WebSocket transport only. + */ public LineSenderBuilder closeFlushTimeoutMillis(long timeoutMillis) { if (protocol != PARAMETER_NOT_SET_EXPLICITLY && protocol != PROTOCOL_WEBSOCKET) { throw new LineSenderException("close_flush_timeout_millis is only supported for WebSocket transport"); @@ -3019,13 +3020,17 @@ private static long parseSizeValue(@NotNull StringSink value, @NotNull String na * already-lost batch for an unbounded outage of everything after it, which inverts * the one guarantee store-and-forward exists to give. *

    - * So: rename the slot aside instead. The bytes are preserved for forensics and for - * the orphan drainer, which reaches the same verdict independently (its send loop's - * replay guard fires and it marks the slot {@code .failed}) -- and which, on a slot - * that turns out to be drainable after all (frames written in full-dictionary - * fallback mode are self-sufficient), simply drains it. The new name is NOT the - * sender's own slot name, so {@code OrphanScanner} will consider it. The producer, - * meanwhile, starts on a clean empty slot and never notices. + * So: rename the slot aside instead, and mark it {@code .failed}. The verdict is + * authoritative -- the recovery seed already tried every source of truth (the + * persisted prefix AND the surviving frames' own deltas), and the orphan drainer's + * own replay guard uses that same walk, so there is nothing a drainer could rebuild + * that the seed did not. {@code markFailed} (below) therefore quarantines the copy + * for a human rather than leaving a drainer to retry an unreplayable slot forever; + * a full-dictionary-fallback slot never reaches here, because its dictionary is + * discarded at recovery and it never throws. The bytes are preserved on disk for + * forensics and a manual resend, and the new name -- NOT the sender's own slot name + * -- keeps a restarted sender from re-adopting it. The producer, meanwhile, starts + * on a clean empty slot and never notices. *

    * If the rename fails (a Windows share lock, a read-only mount) there is no way to * free the slot name without destroying data, so fall back to the old behaviour and diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index d7ba19cf..afdb8114 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -3751,7 +3751,9 @@ private void sendCommitMessage() { // what has already been sent -- and therefore already persisted. In delta // mode pass sentMaxSymbolId, yielding an empty delta // [sentMaxSymbolId+1 .. sentMaxSymbolId]; in full-dict mode keep - // currentBatchMaxSymbolId so the frame stays self-sufficient. Any symbol a + // currentBatchMaxSymbolId (reset to -1 by the prior flush, so the commit frame + // carries an empty delta too -- harmless, since the preceding full-dict data + // frames already registered the dictionary on this connection). Any symbol a // cancelled row leaked is picked up (and persisted) by the next real flush, // whose persistNewSymbolsBeforePublish resumes from pd.size(). int commitBatchMaxId = deltaDictEnabled ? sentMaxSymbolId : currentBatchMaxSymbolId; diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java index 37adf4a0..6b1d741a 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java @@ -783,7 +783,6 @@ public boolean isDeltaDictEnabled() { return sfDir == null || persistedSymbolDict != null; } - /** * Pass-through to {@link SegmentRing#nextSealedAfter(MmapSegment)}. */ diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java index 5ad281e1..b2ee674b 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java @@ -524,37 +524,6 @@ public long findLastFrameFsnWithoutPayloadFlag(int flagsOffset, int flagMask, in return best; } - /** - * Highest {@code deltaStart + deltaCount} (one past the highest symbol id) any - * delta-flagged frame in this segment carries, or {@code 0} only when NO - * delta-flagged frame is present. A frame with {@code deltaCount == 0} (a commit - * or a symbol-reusing frame -- the encoder always sets the delta flag) still - * contributes its {@code deltaStart}, i.e. the producer's baseline at encode - * time, NOT 0. That is deliberate: it anchors the torn-dict guard at that - * baseline so a dictionary torn below it is detected -- a conservative - * over-strand (it may over-reject a frame whose rows reference only surviving - * ids), never an under-strand that could silently shift the dense id map. - * Read-only walk over the recovered frames, used - * once at recovery to detect a persisted {@code .symbol-dict} torn (host crash, - * out-of-order page loss) below the ids the surviving frames still reference: if - * the max here reaches at or beyond the recovered dictionary size, a resuming - * producer -- seeded from the shorter dictionary -- would re-use ids the frames - * already define. Only frames at or below {@code maxFsnInclusive} count: frames - * above it are the aborted orphan-deferred tail, which {@code trySendOne} retires - * without ever transmitting, so their ids must not inflate the result (a resuming - * producer never reuses them on the wire). The frame layout mirrors - * {@link #findLastFrameFsnWithoutPayloadFlag}: the QWP message header - * ({@code qwpHeaderSize} bytes) is followed by the delta section - * {@code [deltaStart varint][deltaCount varint]...}. - * - * @param headerMagic the QWP message magic identifying a well-formed frame - * @param flagsOffset byte offset of the flags field within the QWP header - * @param flagDeltaMask the FLAG_DELTA_SYMBOL_DICT bit - * @param qwpHeaderSize the QWP message header size (delta section starts past it) - * @param maxFsnInclusive highest frame FSN to consider; frames above it are the - * retired orphan-deferred tail -- pass - * {@code recoveredCommitBoundaryFsn} - */ /** * Rebuilds, from the frames' own delta sections, the symbols a replay of * {@code [minFsnInclusive .. maxFsnInclusive]} would register ABOVE {@code coverage}, @@ -670,6 +639,37 @@ public long collectReplaySymbolsAbove( return coverage; } + /** + * Highest {@code deltaStart + deltaCount} (one past the highest symbol id) any + * delta-flagged frame in this segment carries, or {@code 0} only when NO + * delta-flagged frame is present. A frame with {@code deltaCount == 0} (a commit + * or a symbol-reusing frame -- the encoder always sets the delta flag) still + * contributes its {@code deltaStart}, i.e. the producer's baseline at encode + * time, NOT 0. That is deliberate: it anchors the torn-dict guard at that + * baseline so a dictionary torn below it is detected -- a conservative + * over-strand (it may over-reject a frame whose rows reference only surviving + * ids), never an under-strand that could silently shift the dense id map. + * Read-only walk over the recovered frames, used + * once at recovery to detect a persisted {@code .symbol-dict} torn (host crash, + * out-of-order page loss) below the ids the surviving frames still reference: if + * the max here reaches at or beyond the recovered dictionary size, a resuming + * producer -- seeded from the shorter dictionary -- would re-use ids the frames + * already define. Only frames at or below {@code maxFsnInclusive} count: frames + * above it are the aborted orphan-deferred tail, which {@code trySendOne} retires + * without ever transmitting, so their ids must not inflate the result (a resuming + * producer never reuses them on the wire). The frame layout mirrors + * {@link #findLastFrameFsnWithoutPayloadFlag}: the QWP message header + * ({@code qwpHeaderSize} bytes) is followed by the delta section + * {@code [deltaStart varint][deltaCount varint]...}. + * + * @param headerMagic the QWP message magic identifying a well-formed frame + * @param flagsOffset byte offset of the flags field within the QWP header + * @param flagDeltaMask the FLAG_DELTA_SYMBOL_DICT bit + * @param qwpHeaderSize the QWP message header size (delta section starts past it) + * @param maxFsnInclusive highest frame FSN to consider; frames above it are the + * retired orphan-deferred tail -- pass + * {@code recoveredCommitBoundaryFsn} + */ public long maxSymbolDeltaEnd(int headerMagic, int flagsOffset, int flagDeltaMask, int qwpHeaderSize, long maxFsnInclusive) { long maxEnd = 0L; long off = HEADER_SIZE; diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java index 3ec06014..06d1e64d 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java @@ -546,14 +546,6 @@ public synchronized long findLastFsnWithoutPayloadFlag(int flagsOffset, int flag return Math.max(best, fsn); } - /** - * Highest {@code deltaStart + deltaCount} any symbol-dict delta frame at or below - * {@code maxFsnInclusive} references (0 when none). See - * {@link MmapSegment#maxSymbolDeltaEnd}; used once at recovery to detect a - * persisted dictionary torn below the ids the surviving committed frames - * reference. Frames above {@code maxFsnInclusive} are the retired orphan-deferred - * tail and are excluded. - */ /** * Rebuilds the symbols a replay of {@code [minFsnInclusive .. maxFsnInclusive]} would * register ABOVE {@code baseline}, appending them to {@code out} in ascending id order. @@ -589,6 +581,14 @@ public synchronized long collectReplaySymbolsAbove( minFsnInclusive, maxFsnInclusive, coverage, out); } + /** + * Highest {@code deltaStart + deltaCount} any symbol-dict delta frame at or below + * {@code maxFsnInclusive} references (0 when none). See + * {@link MmapSegment#maxSymbolDeltaEnd}; used once at recovery to detect a + * persisted dictionary torn below the ids the surviving committed frames + * reference. Frames above {@code maxFsnInclusive} are the retired orphan-deferred + * tail and are excluded. + */ public synchronized long maxSymbolDeltaEnd(int headerMagic, int flagsOffset, int flagDeltaMask, int qwpHeaderSize, long maxFsnInclusive) { long maxEnd = 0L; for (int i = 0, n = sealedSegments.size(); i < n; i++) { From c0c2ce272d4fed327dbc10532a5355ab5b416795 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Wed, 15 Jul 2026 11:32:21 +0100 Subject: [PATCH 64/96] Preserve symbol dict on a transient stat fault PersistedSymbolDict.open() computed the routing length as ff.length(path) and sent anything below HEADER_SIZE to openFresh(), which O_TRUNCs the file. A stat that fails transiently (an EIO on a flaky disk) returns -1 for a fully populated dictionary, so that routing could truncate the only copy of load-bearing recovery state -- the exact destruction the class's "never recreate over an existing file" contract forbids, and which the openExisting read path already guards against but the routing check did not. A stat error reports -1, distinguishable from a genuine sub-header stub in [0, HEADER_SIZE), so open() now degrades to null when the file exists but its length reads negative: it falls back to full self-sufficient frames and leaves every byte on disk for a later attempt, once the transient clears. Also move appendRawEntries' per-entry (addr, len, count) consistency walk behind an assert. It re-reads every entry's length prefix on the common flush path, yet its sole caller derives count and len from one beginMessage, so it cannot fire today. The assert keeps the structural guard in the client's -ea test suite while eliding it from production, where client apps run without -ea. PersistedSymbolDictTest now covers a transient length fault leaving the file intact and recovering once it clears, and an inconsistent raw-entries triple being rejected under -ea. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../client/sf/cursor/PersistedSymbolDict.java | 95 +++++++++----- .../sf/cursor/PersistedSymbolDictTest.java | 118 ++++++++++++++++++ 2 files changed, 182 insertions(+), 31 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java index 411346d0..835d365a 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java @@ -215,7 +215,22 @@ public static PersistedSymbolDict open(String slotDir) { */ public static PersistedSymbolDict open(FilesFacade ff, String slotDir) { String filePath = slotDir + "/" + FILE_NAME; - long existing = ff.exists(filePath) ? ff.length(filePath) : -1L; + boolean exists = ff.exists(filePath); + long existing = exists ? ff.length(filePath) : -1L; + if (exists && existing < 0) { + // The file is present but its length could not be stat'd (a transient EIO + // on a flaky disk). Do NOT fall through to openFresh below, which O_TRUNCs: + // truncating the only copy of load-bearing state on a TRANSIENT fault is the + // exact destruction the class-level "Never recreate over an existing file" + // note forbids -- and unlike the openExisting read path, this routing check + // otherwise has no guard. A genuine sub-header stub reports a length in + // [0, HEADER_SIZE); only a stat error reports < 0, so the two are + // distinguishable. Degrade to full self-sufficient frames and leave every + // byte on disk for a later attempt, once the transient clears. + LOG.warn("symbol dict {} exists but its length could not be read; " + + "falling back to full-dictionary frames (file left intact)", filePath); + return null; + } if (existing >= HEADER_SIZE) { // A dictionary at or past Integer.MAX_VALUE cannot be read back: // openExisting reads it into ONE int-sized native buffer, and the (int) @@ -310,36 +325,12 @@ public synchronized void appendRawEntries(long addr, int len, int count) { // with the entries it holds, shifting the dense id->symbol map on recovery. // The sole caller derives count and len from one beginMessage, so this cannot // fire today -- but the file this guards is the one the "resend required" - // contract rests on, so it stays. - long src = addr; - long srcLimit = addr + len; - for (int i = 0; i < count; i++) { - long symLen = 0; - int shift = 0; - while (src < srcLimit) { - byte b = Unsafe.getUnsafe().getByte(src++); - symLen |= (long) (b & 0x7F) << shift; - if ((b & 0x80) == 0) { - break; - } - shift += 7; - if (shift > 35) { - // A canonical entry-length varint is <= 5 bytes; a longer - // continuation run is corrupt. The bound check below rejects it. - break; - } - } - src += symLen; // src was just past the len varint - if (src > srcLimit) { - throw new IllegalStateException("malformed raw symbol-dict entries to " + FILE_NAME - + " [entry=" + i + ", count=" + count + ']'); - } - } - if (src != srcLimit) { - throw new IllegalStateException("raw symbol-dict entries under-filled the buffer to " - + FILE_NAME + " [count=" + count + ", len=" + len - + ", consumed=" + (int) (src - addr) + ']'); - } + // contract rests on, so keep the structural guard. Gated behind assert: it + // re-walks every entry's length prefix on the common flush path, and the client + // library runs without -ea in production (embedded in user apps), so this holds + // the guard in the client's own -ea test suite without the per-flush cost in + // production. + assert validateRawEntries(addr, len, count); int hdrLen = NativeBufferWriter.varintSize(count) + NativeBufferWriter.varintSize(len); ensureScratch((long) hdrLen + len + CRC_SIZE); long p = NativeBufferWriter.writeVarint(scratchAddr, count); @@ -693,6 +684,48 @@ private static PersistedSymbolDict openFresh(FilesFacade ff, String filePath) { return new PersistedSymbolDict(ff, fd, HEADER_SIZE, 0, 0L, 0); } + /** + * Verifies that {@code count} wire entries ({@code [len varint][utf8]}) occupy + * exactly {@code len} bytes from {@code addr}. Returns {@code true} when the triple + * is consistent; throws {@link IllegalStateException} (naming the offending entry / + * count / consumed bytes) otherwise. Called only from an {@code assert} in + * {@link #appendRawEntries}: it guards an internal invariant the sole caller cannot + * violate today, so it runs under the test suite's {@code -ea} but is elided from + * the per-flush production path (client apps run without {@code -ea}). + */ + private static boolean validateRawEntries(long addr, int len, int count) { + long src = addr; + long srcLimit = addr + len; + for (int i = 0; i < count; i++) { + long symLen = 0; + int shift = 0; + while (src < srcLimit) { + byte b = Unsafe.getUnsafe().getByte(src++); + symLen |= (long) (b & 0x7F) << shift; + if ((b & 0x80) == 0) { + break; + } + shift += 7; + if (shift > 35) { + // A canonical entry-length varint is <= 5 bytes; a longer + // continuation run is corrupt. The bound check below rejects it. + break; + } + } + src += symLen; // src was just past the len varint + if (src > srcLimit) { + throw new IllegalStateException("malformed raw symbol-dict entries to " + FILE_NAME + + " [entry=" + i + ", count=" + count + ']'); + } + } + if (src != srcLimit) { + throw new IllegalStateException("raw symbol-dict entries under-filled the buffer to " + + FILE_NAME + " [count=" + count + ", len=" + len + + ", consumed=" + (int) (src - addr) + ']'); + } + return true; + } + /** * Grows the append scratch buffer to hold at least {@code required} bytes. *

    diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java index c5222e0e..6c6dcf91 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java @@ -31,6 +31,7 @@ import io.questdb.client.test.tools.DelegatingFilesFacade; import io.questdb.client.test.tools.TestUtils; import org.junit.Assert; +import org.junit.Assume; import org.junit.Test; import java.nio.charset.StandardCharsets; @@ -152,6 +153,56 @@ public void testAppendRawEntriesMatchesAppendSymbols() throws Exception { }); } + @Test + public void testAppendRawEntriesRejectsInconsistentTripleUnderAssertions() throws Exception { + // appendRawEntries validates the (addr,len,count) triple before writing: an + // inconsistent triple would record a chunk whose stored entryCount disagreed + // with its entries and shift the dense id->symbol map on recovery, so it must + // fail loudly rather than corrupt the file. That check is an assert -- gated on + // -ea to keep the per-entry walk off the per-flush production path (client apps + // run without -ea) -- so observe it under the test suite's -ea, and skip if + // assertions happen to be disabled. + boolean assertionsEnabled = false; + assert assertionsEnabled = true; + Assume.assumeTrue("the guard is assert-gated; only observable under -ea", assertionsEnabled); + assertMemoryLeak(() -> { + Path src = Files.createTempDirectory("qwp-symdict-src"); + Path dst = Files.createTempDirectory("qwp-symdict-dst"); + try { + GlobalSymbolDictionary dict = new GlobalSymbolDictionary(); + dict.getOrAddSymbol("AAPL"); // id 0 + dict.getOrAddSymbol("MSFT"); // id 1 + dict.getOrAddSymbol("GOOG"); // id 2 + PersistedSymbolDict encoded = PersistedSymbolDict.open(src.toString()); + encoded.appendSymbols(dict, 0, 2); + encoded.close(); + + // Grab the on-disk entry region (3 entries) verbatim, then feed it back + // with a count of 1: validateRawEntries stops one entry in with + // src < srcLimit and throws "under-filled", before any bytes are written. + PersistedSymbolDict reopened = PersistedSymbolDict.open(src.toString()); + try { + PersistedSymbolDict raw = PersistedSymbolDict.open(dst.toString()); + try { + raw.appendRawEntries(reopened.loadedEntriesAddr(), reopened.loadedEntriesLen(), 1); + Assert.fail("an inconsistent (len,count) triple must be rejected"); + } catch (IllegalStateException expected) { + Assert.assertTrue("message names the under-filled buffer: " + expected.getMessage(), + expected.getMessage().contains("under-filled")); + Assert.assertEquals("a rejected triple must write nothing", 0, raw.size()); + } finally { + raw.close(); + } + } finally { + reopened.close(); + } + } finally { + rmDir(src); + rmDir(dst); + } + }); + } + @Test public void testAppendRawEntriesShortWriteThrowsWithoutAdvancingIsIdempotentOnRetry() throws Exception { // The producer's FAST path persists a frame's already-encoded delta bytes @@ -774,6 +825,60 @@ public void testTornTrailingEntrySelfHeals() throws Exception { }); } + @Test + public void testTransientLengthFaultDegradesWithoutDestroyingTheFile() throws Exception { + // open() routes on ff.length(): a value < HEADER_SIZE falls through to the + // truncating openFresh(). A genuine sub-header stub reports a length in + // [0, HEADER_SIZE), but a TRANSIENT stat failure (an EIO on a flaky disk) + // reports the -1 error sentinel for a fully populated file -- and routing that + // to openFresh would O_TRUNC the only copy of load-bearing state, the exact + // destruction the "Never recreate over an existing file" contract forbids. A + // negative length is distinguishable from a stub, so open() must degrade to + // null (fall back to full self-sufficient frames) and leave every byte on disk + // for a later attempt, once the transient clears. + assertMemoryLeak(() -> { + Path dir = Files.createTempDirectory("qwp-symdict"); + try { + PersistedSymbolDict d = PersistedSymbolDict.open(dir.toString()); + d.appendSymbol("one"); + d.appendSymbol("two"); + d.close(); + + Path f = dir.resolve(".symbol-dict"); + byte[] before = Files.readAllBytes(f); + Assert.assertTrue("a populated dict must exceed the header", before.length > HEADER_SIZE); + + // Reopen through a facade whose length() reports the -1 stat-error + // sentinel for the (present) file -- the branch under test. + PersistedSymbolDict reopened = PersistedSymbolDict.open(new StatFailsLengthFacade(), dir.toString()); + if (reopened != null) { + // Pre-fix: open() fell through to openFresh() and handed back a fresh + // empty dict over the now-truncated file. Close its fd so only the + // assertion below reports the failure, not a leaked descriptor. + reopened.close(); + } + Assert.assertNull("a populated dict whose length cannot be stat'd must degrade to null, not truncate", + reopened); + Assert.assertArrayEquals("a transient length-stat fault must NOT destroy the dictionary", + before, Files.readAllBytes(f)); + + // Once the filesystem recovers, the SAME file reopens its intact content. + PersistedSymbolDict re = PersistedSymbolDict.open(dir.toString()); + Assert.assertNotNull(re); + try { + Assert.assertEquals("the intact content survives the transient", 2, re.size()); + ObjList s = re.readLoadedSymbols(); + Assert.assertEquals("one", s.getQuick(0)); + Assert.assertEquals("two", s.getQuick(1)); + } finally { + re.close(); + } + } finally { + rmDir(dir); + } + }); + } + @Test public void testTruncateFailureDegradesWithoutDestroyingTheFile() throws Exception { // A host crash can leave a torn/stale tail past the last complete chunk. @@ -894,4 +999,17 @@ public long write(int fd, long addr, long len, long offset) { return INSTANCE.write(fd, addr, len, offset); } } + + /** + * Reports a length of -1 -- the stat-error sentinel -- for the dictionary file, + * reproducing a transient stat failure (an EIO on a flaky disk) where the file is + * present but its size cannot be read. open() must treat this as "present but + * unreadable" and degrade to null, NOT route it to the truncating fresh-open path. + */ + private static final class StatFailsLengthFacade extends DelegatingFilesFacade { + @Override + public long length(String path) { + return -1L; + } + } } From 0520dc47731d93c91527e9a41bb41f2658d9105a Mon Sep 17 00:00:00 2001 From: glasstiger Date: Wed, 15 Jul 2026 15:11:50 +0100 Subject: [PATCH 65/96] Don't quarantine a fully-acked torn-dict slot seedGlobalDictionaryFromPersisted walked collectReplaySymbolsAbove from FSN 0 and threw UnreplayableSlotException on the first symbol-dictionary gap regardless of ack state. For a recovered slot whose committed frames were all acked -- so nothing was left to replay -- a gap in that already delivered data still threw. (A host crash tore the unsynced side-file, or it could not be opened, while the low segments that registered the missing ids were trimmed away.) build() then set the slot aside as .unreplayable-, its fully-drained close deleted the delivered bytes the quarantine promises to preserve, and it logged a false "resend required" for data the server had already acknowledged. That gap only matters for frames that will REPLAY. When every committed frame has been acked (ackedFsn >= recoveredCommitBoundaryFsn), the seed now suppresses the throw: it seeds the intact prefix only, leaving the frame-derived symbols unused exactly as the send loop's mirror does on a -1, so the producer baseline and the mirror's sentDictCount still agree by construction, and resumes on the same slot. A genuine gap in UNACKED committed frames still throws, so the real-unreplayable quarantine path is unchanged. Add testFullyAckedTornSlotResumesInPlaceWithoutQuarantine, the acked counterpart to the existing unacked quarantine test: the same on-disk tear, but with the ack watermark stamped at the last committed frame, asserting no set-aside copy exists and the sender resumes in place. The test fails without the production change. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../qwp/client/QwpWebSocketSender.java | 27 +++++++++ .../qwp/client/DeltaDictRecoveryTest.java | 56 +++++++++++++++++++ 2 files changed, 83 insertions(+) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index afdb8114..8c954aba 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -3933,6 +3933,33 @@ private void seedGlobalDictionaryFromPersisted(PersistedSymbolDict pd) { ObjList fromFrames = new ObjList<>(); long coverage = cursorEngine.collectReplaySymbolsAbove(baseline, fromFrames); if (coverage < 0) { + // A gap: the surviving frames reference ids below their own delta start, + // introduced by frames since acked and trimmed away, and the persisted + // dictionary no longer holds them (a host crash tore its unsynced tail, or it + // could not be opened). That gap only matters for frames that will REPLAY. + // When every recovered committed frame is already acked + // (ackedFsn >= recoveredCommitBoundaryFsn), NOTHING replays: the gap is in + // data the server already has, and the retired orphan-deferred tail above the + // commit boundary is never transmitted. Throwing here would raise a false + // "resend required" for delivered data AND -- because such a slot is fully + // drained -- let build()'s connect-path close unlink the (already-delivered) + // bytes the quarantine claims to preserve. So DON'T throw: seed the intact + // prefix only, leaving fromFrames unused exactly as the send loop's mirror + // does on a -1 (it adds nothing), so the producer baseline and the mirror's + // sentDictCount still agree by construction. The producer resumes above the + // prefix and the fully-drained slot is cleaned up on close. + long ackedFsn = cursorEngine.ackedFsn(); + long commitBoundaryFsn = cursorEngine.recoveredCommitBoundaryFsn(); + if (ackedFsn >= commitBoundaryFsn) { + sentMaxSymbolId = globalSymbolDictionary.size() - 1; + LOG.info("recovered store-and-forward slot has a torn/incomplete symbol dictionary, " + + "but every committed frame was already acked so nothing needs replaying; " + + "resuming on the intact prefix without quarantine and without data loss " + + "[recoveredPrefixSize={}, ackedFsn={}, commitBoundaryFsn={}]", + baseline, ackedFsn, commitBoundaryFsn); + return; + } + // Genuine loss: unacked committed frames reference ids nothing still holds. // Typed, because Sender.build() sets such a slot aside instead of failing: this // is the point at which every source of truth has been tried and none of them // holds the missing ids. See UnreplayableSlotException. diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java index e8ff1e99..b4d91895 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java @@ -249,6 +249,62 @@ public void testTrimmedRegisteringFramesAreUnreplayableAndTheSlotIsSetAside() th }); } + @Test + public void testFullyAckedTornSlotResumesInPlaceWithoutQuarantine() throws Exception { + // M1 regression -- the ACKED counterpart to + // testTrimmedRegisteringFramesAreUnreplayableAndTheSlotIsSetAside. The on-disk + // tear is IDENTICAL (earliest segment trimmed, .symbol-dict torn to its header, + // so the surviving frames' deltas start above ids nothing on disk still holds), + // but here every committed frame was already ACKED before the crash. Nothing is + // left to replay, so the "gap" is entirely in data the server already has. + // + // seedGlobalDictionaryFromPersisted must therefore NOT raise + // UnreplayableSlotException. Quarantining a fully-delivered slot would fire a + // false "resend required" alarm AND -- because such a slot is fully drained -- + // let build()'s connect-path close unlink the (already-delivered) bytes the + // quarantine claims to preserve. The slot must resume IN PLACE. + // + // Before the fix the gap detector ignored ack state and threw, so this slot was + // set aside as default.unreplayable-0 with a "resend required" error for data the + // server had already acknowledged. + assertMemoryLeak(() -> { + writeAndTearUnreplayableSlot(); + // Mark every committed frame acked. writeAndTearUnreplayableSlot writes 12 + // frames (FSNs 0..11), so stamping the watermark at 11 makes + // ackedFsn == recoveredCommitBoundaryFsn: a torn dictionary with nothing left + // to replay. (Not higher than 11 -- that would make the resuming producer's + // next frame look pre-acked.) + java.nio.file.Path slot = Paths.get(sfDir, "default"); + writeAckWatermark(slot.resolve(".ack-watermark"), 11); + + DictReconstructingHandler handler = new DictReconstructingHandler(); + try (TestWebSocketServer good = new TestWebSocketServer(handler)) { + int port = good.getPort(); + good.start(); + Assert.assertTrue(good.awaitStart(5, TimeUnit.SECONDS)); + String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";"; + // build() must succeed WITHOUT setting the slot aside, and the producer + // must keep working on the SAME slot. + try (Sender s2 = Sender.fromConfig(cfg)) { + s2.table("m").symbol("s", "after-recovery").longColumn("v", 99).atNow(); + s2.flush(); + long deadline = System.currentTimeMillis() + 10_000; + while (System.currentTimeMillis() < deadline && handler.maxDictSize() < 1) { + Thread.sleep(20); + } + } + } + // The fully-acked slot was NOT quarantined: no set-aside copy exists (the + // inverse of assertUnreplayableSlotSetAside), and the sender resumed on the + // original slot. + Assert.assertFalse("a fully-acked torn slot must NOT be quarantined -- its data " + + "was already delivered, so there is nothing to resend", + java.nio.file.Files.isDirectory(Paths.get(sfDir, "default.unreplayable-0"))); + Assert.assertTrue("the sender must resume on the original slot", + java.nio.file.Files.isDirectory(slot)); + }); + } + private static int countSegmentFiles(java.nio.file.Path dir) { java.io.File[] files = dir.toFile().listFiles(); int n = 0; From 8d6a93243fadfaf57ed23d193e02f27b851fccec Mon Sep 17 00:00:00 2001 From: glasstiger Date: Wed, 15 Jul 2026 21:04:47 +0100 Subject: [PATCH 66/96] Keep foreground cap-gap retries unbounded Separate symbol-dictionary catch-up cap-gap policy by loop owner. Foreground senders now retry until a compatible node returns, while orphan drainers retain bounded quarantine behavior. Preserve existing constructor signatures and add unit and integration coverage for retries beyond the orphan budget and recovery after the server cap is restored. --- .../main/java/io/questdb/client/Sender.java | 12 +- .../qwp/client/QwpWebSocketSender.java | 9 +- .../client/sf/cursor/BackgroundDrainer.java | 9 +- .../sf/cursor/CursorWebSocketSendLoop.java | 150 +++++++++++------- .../qwp/client/DeltaDictCatchUpTest.java | 74 +++------ ...WebSocketSendLoopCatchUpAlignmentTest.java | 74 +++++++-- 6 files changed, 197 insertions(+), 131 deletions(-) diff --git a/core/src/main/java/io/questdb/client/Sender.java b/core/src/main/java/io/questdb/client/Sender.java index 23fdf6a8..f73d3e35 100644 --- a/core/src/main/java/io/questdb/client/Sender.java +++ b/core/src/main/java/io/questdb/client/Sender.java @@ -1760,7 +1760,7 @@ public Sender build() { /** * Minimum wall-clock time (millis) a symbol-dictionary catch-up CAP GAP must - * persist before the sender gives up on it and fails. + * persist before an orphan store-and-forward drainer quarantines its slot. *

    * A cap gap means a symbol already accepted by one node is too large to * re-register on the node the sender just failed over to, because that node @@ -1768,13 +1768,11 @@ public Sender build() { * happen; it takes a heterogeneous or mid-roll cluster, or an operator lowering * the cap below existing data. *

    - * The sender retries such a gap across reconnects rather than failing on sight, - * because the larger-cap node may simply be away -- a rolling restart is the most - * likely reason a failover happened at all. It gives up only once the gap has BOTH - * recurred many times AND persisted for this long, so an ordinary cluster - * operation cannot bring down a live producer. Raise it for a cluster whose + * A foreground sender retries such a gap indefinitely because the larger-cap node + * may simply be away. An orphan drainer may quarantine only once the gap has BOTH + * recurred many times AND persisted for this long. Raise it for a cluster whose * rolling restarts take longer than the 5-minute default; set it to {@code 0} to - * fail as soon as the retry count is exhausted. + * quarantine an orphan slot as soon as the retry count is exhausted. *

    * WebSocket transport only. */ diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index 8c954aba..c22f01ec 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -314,9 +314,9 @@ public class QwpWebSocketSender implements Sender { // maxFrameRejections (connect-string key poison_min_escalation_window_millis). private long poisonMinEscalationWindowMillis = CursorWebSocketSendLoop.DEFAULT_POISON_MIN_ESCALATION_WINDOW_MILLIS; - // Minimum wall-clock dwell a symbol-dict catch-up cap gap must persist before the - // send loop latches a terminal (connect-string key - // catchup_cap_gap_min_escalation_window_millis). See + // Minimum wall-clock dwell a symbol-dict catch-up cap gap must persist before an + // orphan drainer may quarantine its slot (connect-string key + // catchup_cap_gap_min_escalation_window_millis). Foreground senders retry forever. See // CursorWebSocketSendLoop.DEFAULT_CATCHUP_CAP_GAP_MIN_ESCALATION_WINDOW_MILLIS. private long catchUpCapGapMinEscalationWindowMillis = CursorWebSocketSendLoop.DEFAULT_CATCHUP_CAP_GAP_MIN_ESCALATION_WINDOW_MILLIS; @@ -3358,7 +3358,8 @@ private void ensureConnected() { durableAckKeepaliveIntervalMillis, maxFrameRejections, poisonMinEscalationWindowMillis, - catchUpCapGapMinEscalationWindowMillis); + catchUpCapGapMinEscalationWindowMillis, + CursorWebSocketSendLoop.CatchUpCapGapPolicy.RETRY_FOREVER); // Plug the async-delivery sink before start() so the I/O thread // never observes a null dispatcher between recordFatal and // notification — the test for null in dispatchError handles diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/BackgroundDrainer.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/BackgroundDrainer.java index a88429ce..0e7bcd10 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/BackgroundDrainer.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/BackgroundDrainer.java @@ -133,9 +133,9 @@ public final class BackgroundDrainer implements Runnable { // drain loop; mirrors the owner sender's poison_min_escalation_window_millis. private final long poisonMinEscalationWindowMillis; // Minimum wall-clock dwell a symbol-dict catch-up cap gap must persist before this - // drainer's send loop latches a terminal. Same knob the foreground sender uses; the - // orphan drainer honours it too so a rolling restart cannot quarantine an otherwise - // drainable slot on a strike count alone. + // orphan drainer's send loop latches a terminal. Foreground senders retry forever; + // this bounded policy is permitted only so a persistent orphan slot can be + // quarantined for operator intervention. private final long catchUpCapGapMinEscalationWindowMillis; public BackgroundDrainer( @@ -584,7 +584,8 @@ public void run() { durableAckKeepaliveIntervalMillis, maxHeadFrameRejections, poisonMinEscalationWindowMillis, - catchUpCapGapMinEscalationWindowMillis); + catchUpCapGapMinEscalationWindowMillis, + CursorWebSocketSendLoop.CatchUpCapGapPolicy.TERMINAL_AFTER_SETTLE_BUDGET); loop.start(); while (!stopRequestedOrInterrupted()) { diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java index c705aaac..2d94dc76 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java @@ -136,7 +136,7 @@ public final class CursorWebSocketSendLoop implements QuietCloseable { public static final long DEFAULT_POISON_MIN_ESCALATION_WINDOW_MILLIS = 5_000L; /** * Default minimum wall-clock dwell (millis) a symbol-dict catch-up CAP GAP must - * persist before the loop latches a terminal, even once + * persist before an orphan drainer may latch a terminal, even once * {@link #MAX_CATCHUP_CAP_GAP_ATTEMPTS} cap gaps have accrued. Same idea as * {@link #DEFAULT_POISON_MIN_ESCALATION_WINDOW_MILLIS}, for the same reason: a * strike count measures "how many times did we look", not "how long has this been @@ -146,35 +146,34 @@ public final class CursorWebSocketSendLoop implements QuietCloseable { * It matters here because the cap gap's trigger IS a failover. With the reconnect * backoff capped at {@link #DEFAULT_RECONNECT_MAX_BACKOFF_MILLIS} (5 s), 16 cap * gaps accrue in roughly two minutes -- comfortably less than an ordinary rolling - * restart of the larger-cap node. A count-only budget would therefore hard-fail a - * live store-and-forward producer during a routine cluster operation, which is - * exactly the transient the budget exists to ride out. Requiring the dwell as well - * means the terminal needs BOTH "we have looked many times" AND "it has stayed true - * for a long time"; a node that returns inside the window ends the episode and the - * producer never notices. + * restart of the larger-cap node. A count-only budget could therefore quarantine an + * otherwise drainable orphan slot during a routine cluster operation. Requiring the + * dwell as well means the orphan terminal needs BOTH "we have looked many times" AND + * "it has stayed true for a long time". Foreground senders never apply this terminal + * policy: they retry indefinitely until a larger-cap node returns. *

    * 5 minutes -- the same figure as {@link #DEFAULT_RECONNECT_MAX_DURATION_MILLIS}, * this codebase's existing notion of how long a transient outage may plausibly * last. Configurable per sender via the * {@code catchup_cap_gap_min_escalation_window_millis} connect-string key or * {@code LineSenderBuilder.catchUpCapGapMinEscalationWindowMillis(long)}; {@code 0} - * restores count-only escalation. + * restores count-only escalation for orphan drainers. */ public static final long DEFAULT_CATCHUP_CAP_GAP_MIN_ESCALATION_WINDOW_MILLIS = 300_000L; private static final Logger LOG = LoggerFactory.getLogger(CursorWebSocketSendLoop.class); // Settle budget for the symbol-dict catch-up cap gap: how many cap-gap attempts // -- catch-ups that reached a fresh server and found a single dictionary entry // too large for its advertised batch cap -- may occur, with no intervening - // SUCCESSFUL catch-up, before the sender latches a terminal. This is a SANCTIONED - // terminal (a genuine cluster batch-size capability gap), the connect-time analog - // of the orphan drainer's durable-ack capability gap + // SUCCESSFUL catch-up, before an orphan drainer latches a terminal. This is a + // SANCTIONED orphan-only terminal (a genuine cluster batch-size capability gap), + // the connect-time analog of the orphan drainer's durable-ack capability gap // (DEFAULT_MAX_DURABLE_ACK_MISMATCH_ATTEMPTS). A homogeneous cluster never trips // it -- an entry that fit its data frame under a cap always fits its bare catch-up // frame under that same cap -- so it only affects a heterogeneous / rolling-cap // cluster, where a failover to a smaller-cap node can hit it for an entry an - // earlier node accepted. Retrying rides out the transient window until a - // larger-cap node returns; only a persistent gap (this many cap gaps with no - // successful catch-up in between) latches. + // earlier node accepted. A foreground sender retries forever. An orphan drainer + // rides out the transient window until a larger-cap node returns; only a persistent + // gap (this many cap gaps with no successful catch-up in between) latches. // // Budget accounting (satisfies "a transient must never burn the terminal // budget"): catchUpCapGapAttempts increments ONLY inside sendDictCatchUp when a @@ -222,11 +221,11 @@ public final class CursorWebSocketSendLoop implements QuietCloseable { // by the server -- holding stale watermarks across the wire boundary // would falsely advance trim before re-confirmation. private final CharSequenceLongHashMap durableTableWatermarks = new CharSequenceLongHashMap(); - // Pre-converted to nanos. Zero disables the dwell entirely (count-only escalation - // at MAX_CATCHUP_CAP_GAP_ATTEMPTS) -- the raw-constructor default; the user-facing - // 5-minute default is applied at the config layer, mirroring - // poisonMinEscalationWindowNanos. + // Pre-converted to nanos. Consulted only by the orphan terminal policy. Zero disables + // the dwell entirely (count-only escalation at MAX_CATCHUP_CAP_GAP_ATTEMPTS); the + // user-facing 5-minute default is applied at the config layer. private final long catchUpCapGapMinEscalationWindowNanos; + private final CatchUpCapGapPolicy catchUpCapGapPolicy; private final CursorSendEngine engine; private final long parkNanos; // FIFO of OK-acked batches awaiting durable-upload confirmation. Used only @@ -302,16 +301,15 @@ public final class CursorWebSocketSendLoop implements QuietCloseable { private int sentDictBytesCapacity; private int sentDictBytesLen; private int sentDictCount; - // Cap-gap attempts -- catch-ups that reached a node and found an entry too large - // for its batch cap -- since the last SUCCESSFUL catch-up (see - // MAX_CATCHUP_CAP_GAP_ATTEMPTS for the full budget accounting). A successful - // catch-up resets it (sendDictCatchUp); a transient reconnect neither increments - // nor resets it. NOT reset per connection -- it measures the cap-gap episode - // across reconnects so a persistent gap eventually latches. I/O-thread-only. + // Orphan-policy cap-gap attempts -- catch-ups that reached a node and found an entry + // too large for its batch cap -- since the last SUCCESSFUL catch-up (see + // MAX_CATCHUP_CAP_GAP_ATTEMPTS for the full budget accounting). Foreground retries + // never increment it. A successful catch-up resets it; a transient reconnect neither + // increments nor resets it. I/O-thread-only. private int catchUpCapGapAttempts; - // System.nanoTime() of the FIRST cap gap of the current episode. Anchors the - // escalation dwell. Reset together with catchUpCapGapAttempts on a successful - // catch-up, so a node that accepts the dictionary ends the episode outright. + // System.nanoTime() of the FIRST cap gap of the current orphan-policy episode. + // Anchors the escalation dwell. Reset together with catchUpCapGapAttempts on a + // successful catch-up, so a node that accepts the dictionary ends the episode. // Anchoring at the FIRST gap (not at loop entry, and not refreshed per attempt) is // what keeps a TRANSIENT from burning the terminal budget: a transient never reaches // the catch-up, so it neither increments the counter nor moves the anchor -- it can @@ -592,13 +590,9 @@ public CursorWebSocketSendLoop(WebSocketClient client, CursorSendEngine engine, /** * Twelve-arg overload — omits the symbol-dict cap-gap escalation dwell, which - * defaults to {@code 0} (legacy: escalate as soon as - * {@link #MAX_CATCHUP_CAP_GAP_ATTEMPTS} cap gaps accrue, with no minimum - * wall-clock). Like the poison dwell, the user-facing default - * ({@link #DEFAULT_CATCHUP_CAP_GAP_MIN_ESCALATION_WINDOW_MILLIS}) is applied at the - * config layer (Sender / QwpWebSocketSender), so this raw constructor stays - * unopinionated for tests and internal callers that want deterministic strike-count - * escalation. + * defaults to {@code 0}. This and the thirteen-arg overload use the foreground-safe + * retry-forever policy; orphan drainers opt into count-and-dwell escalation through + * the policy-aware master constructor. */ public CursorWebSocketSendLoop(WebSocketClient client, CursorSendEngine engine, long fsnAtZero, long parkNanos, @@ -618,7 +612,11 @@ public CursorWebSocketSendLoop(WebSocketClient client, CursorSendEngine engine, } /** - * Master constructor — also accepts the poison-frame detector threshold + * Thirteen-arg overload. Catch-up cap gaps are retried indefinitely, which is the + * required policy for a foreground store-and-forward sender. Orphan drainers use the + * policy-aware overload below to opt into bounded terminal escalation. + *

    + * Also accepts the poison-frame detector threshold * ({@code max_frame_rejections}): consecutive server-active rejections of * the same head-of-line frame, with no ack progress in between, before the * loop escalates to a typed terminal. Must be {@code >= 1}. Then the minimum @@ -626,11 +624,10 @@ public CursorWebSocketSendLoop(WebSocketClient client, CursorSendEngine engine, * ({@code poison_min_escalation_window_millis}; {@code >= 0}, where 0 = legacy * immediate escalation at the threshold). *

    - * The final argument is the analogous dwell for the symbol-dict catch-up cap gap - * ({@code catchup_cap_gap_min_escalation_window_millis}; {@code >= 0}, where 0 = - * escalate as soon as {@link #MAX_CATCHUP_CAP_GAP_ATTEMPTS} cap gaps accrue). See - * {@link #DEFAULT_CATCHUP_CAP_GAP_MIN_ESCALATION_WINDOW_MILLIS} for why a strike - * count alone is not a safe escalation signal on a live producer. + * The final argument is the analogous dwell for the symbol-dict catch-up cap gap. + * It is retained here for source and binary compatibility but is consulted only when + * the policy-aware overload selects + * {@link CatchUpCapGapPolicy#TERMINAL_AFTER_SETTLE_BUDGET}. */ public CursorWebSocketSendLoop(WebSocketClient client, CursorSendEngine engine, long fsnAtZero, long parkNanos, @@ -643,6 +640,32 @@ public CursorWebSocketSendLoop(WebSocketClient client, CursorSendEngine engine, int maxHeadFrameRejections, long poisonMinEscalationWindowMillis, long catchUpCapGapMinEscalationWindowMillis) { + this(client, engine, fsnAtZero, parkNanos, reconnectFactory, + reconnectMaxDurationMillis, reconnectInitialBackoffMillis, + reconnectMaxBackoffMillis, durableAckMode, + durableAckKeepaliveIntervalMillis, maxHeadFrameRejections, + poisonMinEscalationWindowMillis, catchUpCapGapMinEscalationWindowMillis, + CatchUpCapGapPolicy.RETRY_FOREVER); + } + + /** + * Policy-aware master constructor. Foreground senders must pass + * {@link CatchUpCapGapPolicy#RETRY_FOREVER}; only orphan drainers may pass + * {@link CatchUpCapGapPolicy#TERMINAL_AFTER_SETTLE_BUDGET} and quarantine a slot + * after the attempt and dwell thresholds are both exhausted. + */ + public CursorWebSocketSendLoop(WebSocketClient client, CursorSendEngine engine, + long fsnAtZero, long parkNanos, + ReconnectFactory reconnectFactory, + long reconnectMaxDurationMillis, + long reconnectInitialBackoffMillis, + long reconnectMaxBackoffMillis, + boolean durableAckMode, + long durableAckKeepaliveIntervalMillis, + int maxHeadFrameRejections, + long poisonMinEscalationWindowMillis, + long catchUpCapGapMinEscalationWindowMillis, + CatchUpCapGapPolicy catchUpCapGapPolicy) { if (maxHeadFrameRejections < 1) { throw new IllegalArgumentException( "maxHeadFrameRejections must be >= 1: " + maxHeadFrameRejections); @@ -658,6 +681,10 @@ public CursorWebSocketSendLoop(WebSocketClient client, CursorSendEngine engine, } this.catchUpCapGapMinEscalationWindowNanos = catchUpCapGapMinEscalationWindowMillis * 1_000_000L; + if (catchUpCapGapPolicy == null) { + throw new IllegalArgumentException("catchUpCapGapPolicy must be non-null"); + } + this.catchUpCapGapPolicy = catchUpCapGapPolicy; if (engine == null) { throw new IllegalArgumentException("engine must be non-null"); } @@ -2440,8 +2467,8 @@ private int sendDictCatchUp() { long entryBytes = entryEnd - entryStart; // The exact table-less frame sendCatchUpChunk would build for THIS entry // alone: header + deltaStart varint (the entry's own global id) + - // deltaCount varint (1) + the entry bytes. Terminal only when even that - // solo frame exceeds the cap -- i.e. the entry genuinely cannot be + // deltaCount varint (1) + the entry bytes. A cap gap exists only when even + // that solo frame exceeds the cap -- i.e. the entry genuinely cannot be // re-registered. Testing the real solo frame (not the conservative // packing budget above) is what keeps a HOMOGENEOUS cluster // livelock-free: an entry the producer already shipped in a data frame @@ -2458,18 +2485,17 @@ private int sendDictCatchUp() { // catch-up frame under that same cap), so the only way in is a // heterogeneous / rolling-cap failover to a smaller-cap node. // - // Give the cluster a settle budget instead of latching on first - // sight: a larger-cap node may return, so retry across reconnects - // and only latch a terminal after MAX_CATCHUP_CAP_GAP_ATTEMPTS - // consecutive attempts still find it too large. Under budget the + // A foreground sender retries indefinitely: store-and-forward must + // contain a transient heterogeneous-cluster window instead of surfacing + // it to the producer. Only an orphan drainer may apply the settle budget + // below and quarantine its slot after a persistent gap. Under budget the // throw is RETRIABLE (no recordFatal) -- connectLoop reconnects with - // backoff and re-runs the catch-up, which resets the counter on a - // node that accepts it. A transient reconnect (connect/upgrade - // failure, role reject) never reaches the catch-up, so it neither - // increments nor burns this budget. On exhaustion latch via - // recordFatal, NOT fail() -- failing from inside the catch-up would - // re-enter connectLoop (see CatchUpSendException); the data must be - // resent after the cap is raised. + // backoff and re-runs the catch-up, which resets the counter on a node + // that accepts it. A transient reconnect (connect/upgrade failure, role + // reject) never reaches the catch-up, so it neither increments nor burns + // the orphan budget. On exhaustion latch via recordFatal, NOT fail() -- + // failing from inside the catch-up would re-enter connectLoop (see + // CatchUpSendException); the data must be resent after the cap is raised. // Escalation needs BOTH the strike count AND a minimum wall-clock dwell // (see DEFAULT_CATCHUP_CAP_GAP_MIN_ESCALATION_WINDOW_MILLIS). A count // alone measures how often we looked, not how long the gap has held -- @@ -2487,6 +2513,13 @@ private int sendDictCatchUp() { // the terminal never latches, however long the gap truly persists. // recordHeadRejectionStrike keys its episode off poisonStrikes for the same // reason. + if (catchUpCapGapPolicy == CatchUpCapGapPolicy.RETRY_FOREVER) { + throw new CatchUpSendException(new LineSenderException( + "symbol dictionary entry too large for the server batch cap during catch-up [" + + "frameLen=" + soloFrameLen + ", cap=" + cap + ']' + + "; retrying indefinitely -- a larger-cap node may return")); + } + long nowNanos = System.nanoTime(); if (catchUpCapGapAttempts == 0) { catchUpCapGapFirstNanos = nowNanos; @@ -2815,6 +2848,17 @@ private boolean tryRetireOrphanTail() { return true; } + /** + * Determines whether an oversized symbol-dictionary catch-up entry is always + * retriable or may become terminal after the orphan settle budget. + */ + public enum CatchUpCapGapPolicy { + /** Keep reconnecting until a node with a compatible batch cap returns. */ + RETRY_FOREVER, + /** Quarantine an orphan slot after both the attempt and dwell limits expire. */ + TERMINAL_AFTER_SETTLE_BUDGET + } + /** * Factory used by the I/O loop to build a fresh, connected, upgraded * {@link WebSocketClient} after a wire failure. Implementations close diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictCatchUpTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictCatchUpTest.java index db7603f2..a7b2fb5f 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictCatchUpTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictCatchUpTest.java @@ -25,7 +25,6 @@ package io.questdb.client.test.cutlass.qwp.client; import io.questdb.client.Sender; -import io.questdb.client.cutlass.line.LineSenderException; import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer; import io.questdb.client.test.tools.TestUtils; import org.junit.Assert; @@ -210,17 +209,11 @@ public void testFixedCapNearBoundarySymbolCatchesUpWithoutTerminal() throws Exce } @Test - public void testCatchUpEntryTooLargeForCapFailsTerminally() throws Exception { + public void testForegroundCatchUpCapGapRetriesPastOrphanBudgetAndRecovers() throws Exception { // A dictionary entry that exceeds the reconnect server's per-chunk budget - // (cap - HEADER_SIZE - 16) cannot be shipped as a catch-up chunk. Every - // reachable node re-advertises the same small cap here, so the gap never - // resolves: sendDictCatchUp must retry across the settle budget and then - // latch a clean terminal ("... during catch-up") rather than call fail() - // (which from inside the catch-up re-enters connectLoop). Pre-fix it latched - // on the FIRST cap gap; the settle budget (MAX_CATCHUP_CAP_GAP_ATTEMPTS) - // rides out a transient smaller-cap window first (see the retry sibling in - // CursorWebSocketSendLoopCatchUpAlignmentTest), and only a persistent gap - // exhausts it. Small reconnect backoffs keep the budgeted attempts fast. + // (cap - HEADER_SIZE - 16) cannot be shipped as a catch-up chunk. A live + // foreground sender must keep retrying without surfacing that cluster state to + // the producer, even after the orphan drainer's 16-attempt quarantine budget. // // Connection 1 advertises no cap, so the ~202-byte symbol registers and // enters the sent-dictionary mirror. The handler then shrinks the @@ -237,49 +230,27 @@ public void testCatchUpEntryTooLargeForCapFailsTerminally() throws Exception { Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); String bigSymbol = TestUtils.repeat("x", 200); // ~202-byte dict entry - LineSenderException terminal = null; - // catchup_cap_gap_min_escalation_window_millis=0 restores count-only - // escalation, so the terminal latches as soon as the settle budget's - // strikes are exhausted. The DEFAULT is a 5-minute wall-clock dwell on - // top of the strike count, precisely so a cap gap that lasts only as - // long as a rolling restart cannot hard-fail a live producer -- and a - // test cannot wait that out. Zeroing it here is what makes the terminal - // observable; the dwell itself is covered by - // CursorWebSocketSendLoopCatchUpAlignmentTest. - Sender sender = Sender.fromConfig("ws::addr=localhost:" + port + // Zeroing the dwell makes the old faulty foreground policy terminal at + // its 16th cap gap. Reaching 20 handshakes therefore proves the live loop + // is no longer governed by the orphan budget. + try (Sender sender = Sender.fromConfig("ws::addr=localhost:" + port + ";reconnect_initial_backoff_millis=10;reconnect_max_backoff_millis=50" - + ";catchup_cap_gap_min_escalation_window_millis=0;"); - try { + + ";catchup_cap_gap_min_escalation_window_millis=0;")) { sender.table("t").symbol("s", bigSymbol).longColumn("v", 1L).atNow(); sender.flush(); - // The terminal latches on the I/O thread once the reconnect's - // catch-up hits the oversized entry; it surfaces to the producer - // on a subsequent flush. Poll a bounded time for it. The polling - // rows use a small symbol that fits the shrunk cap, so the - // producer-side cap check never fires and flush() surfaces the - // I/O thread's catch-up terminal via checkError. - long deadline = System.currentTimeMillis() + 10_000; - while (System.currentTimeMillis() < deadline && terminal == null) { - try { - sender.table("t").symbol("s", "y").longColumn("v", 2L).atNow(); - sender.flush(); - Thread.sleep(20); - } catch (LineSenderException e) { - terminal = e; - } - } - } finally { - try { - sender.close(); - } catch (LineSenderException e) { - if (terminal == null) { - terminal = e; - } - } + waitFor(() -> server.handshakeCount() >= 20, 10_000); + + // Producer calls remain usable while the wire is stuck in the cap + // gap. Then restore the larger-cap node and prove the buffered row is + // actually acknowledged, not merely accepted into the local SF ring. + sender.table("t").symbol("s", "y").longColumn("v", 2L).atNow(); + sender.flush(); + server.setAdvertisedMaxBatchSize(0); + sender.table("t").symbol("s", "z").longColumn("v", 3L).atNow(); + long targetFsn = sender.flushAndGetSequence(); + Assert.assertTrue("foreground sender must recover and drain after the cap is restored", + sender.awaitAckedFsn(targetFsn, 5_000)); } - Assert.assertNotNull("an oversized catch-up entry must surface a terminal", terminal); - Assert.assertTrue("terminal must come from the catch-up path, got: " + terminal.getMessage(), - terminal.getMessage().contains("during catch-up")); } }); } @@ -499,7 +470,7 @@ private static int tableCount(byte[] frame) { * registers), then -- once connection 1 has sent something -- shrinks the * advertised batch cap and drops the socket. The reconnect (connection 2) * therefore advertises a cap whose catch-up budget is too small for the - * symbol, exercising the oversized-entry terminal in sendDictCatchUp. + * symbol, exercising the foreground retry path in sendDictCatchUp. */ private static class CapShrinkHandler implements TestWebSocketServer.WebSocketServerHandler { final AtomicInteger connectionsAccepted = new AtomicInteger(); @@ -516,6 +487,7 @@ public synchronized void onBinaryMessage(TestWebSocketServer.ClientHandler clien if (currentClient != client) { currentClient = client; connectionsAccepted.incrementAndGet(); + nextSeq.set(0); } try { client.sendBinary(CatchUpHandler.buildAck(nextSeq.getAndIncrement())); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java index 4e00e1e6..fed94645 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java @@ -267,17 +267,51 @@ public void testCatchUpChunkFrameSizeOverflowFailsLoud() throws Exception { }); } + @Test + public void testForegroundCatchUpCapGapRetriesPastOrphanBudget() throws Exception { + // The foreground policy must never accrue or exhaust the orphan drainer's + // quarantine budget. Drive more cap gaps than that entire budget and assert every + // failure remains retriable to the I/O loop and invisible to the producer. + TestUtils.assertMemoryLeak(() -> { + Field maxField = CursorWebSocketSendLoop.class.getDeclaredField("MAX_CATCHUP_CAP_GAP_ATTEMPTS"); + maxField.setAccessible(true); + int maxAttempts = maxField.getInt(null); + CatchUpCapturingClient client = new CatchUpCapturingClient(160); + try (CursorSendEngine engine = newEngine()) { + CursorWebSocketSendLoop loop = newForegroundLoop(engine, client); + try { + seedMirror(loop, TestUtils.repeat("x", 200)); + for (int i = 1; i <= maxAttempts + 4; i++) { + try { + invokeSetWireBaselineWithCatchUp(loop, engine.ackedFsn() + 1L); + fail("cap gap must raise a retriable CatchUpSendException (attempt " + i + ')'); + } catch (InvocationTargetException e) { + assertEquals("CatchUpSendException", e.getCause().getClass().getSimpleName()); + } + loop.checkError(); + } + assertEquals("foreground retries must not burn the orphan attempt budget", + 0, readInt(loop, "catchUpCapGapAttempts")); + assertEquals("foreground retries must not anchor an orphan cap-gap episode", + -1L, readLong(loop, "catchUpCapGapFirstNanos")); + } finally { + loop.close(); + } + } + }); + } + @Test public void testCatchUpCapGapStrikesAloneDoNotLatchWithinTheEscalationWindow() throws Exception { // The strike count alone must NOT latch a terminal: escalation also requires the // cap-gap episode to have persisted for catchUpCapGapMinEscalationWindowMillis. // - // This is what keeps a routine rolling restart from killing a live producer. + // This keeps a routine rolling restart from quarantining a drainable orphan slot. // MAX_CATCHUP_CAP_GAP_ATTEMPTS strikes accrue in ~2 minutes at the capped // reconnect backoff -- less than the time the larger-cap node is away -- so a - // count-only budget would hard-fail the sender on the very transient the budget + // count-only budget would quarantine the slot on the very transient the budget // exists to ride out. Here we drive far MORE than the budget's strikes inside a - // (deliberately huge) window and assert the sender stays alive. + // deliberately huge window and assert the orphan loop stays retriable. TestUtils.assertMemoryLeak(() -> { Field maxField = CursorWebSocketSendLoop.class.getDeclaredField("MAX_CATCHUP_CAP_GAP_ATTEMPTS"); maxField.setAccessible(true); @@ -446,7 +480,7 @@ public void testCatchUpCapGapRetriesUntilBudgetThenLatches() throws Exception { // the reconnect loop rides it out -- a larger-cap node may return -- and only // after MAX_CATCHUP_CAP_GAP_ATTEMPTS consecutive cap gaps does it recordFatal. // Pre-fix the first cap gap latched a terminal, so one transient failover to a - // smaller-cap node killed the sender. (A successful catch-up resets the budget; + // smaller-cap node quarantined the orphan slot. (A successful catch-up resets the budget; // the other catch-up tests, which use a fitting cap, never trip it.) TestUtils.assertMemoryLeak(() -> { Field maxField = CursorWebSocketSendLoop.class.getDeclaredField("MAX_CATCHUP_CAP_GAP_ATTEMPTS"); @@ -456,13 +490,13 @@ public void testCatchUpCapGapRetriesUntilBudgetThenLatches() throws Exception { // retriable loop below is bounded by maxAttempts, so keying this test purely // off the constant under test makes it TAUTOLOGICAL: a regression of // MAX_CATCHUP_CAP_GAP_ATTEMPTS to 1 -- which is precisely the pre-fix bug this - // test names, a single cap gap killing the sender -- would run the loop ZERO + // test names, a single cap gap quarantining the slot -- would run the loop ZERO // times, the "exhausting" attempt would become the FIRST attempt, and the test // would still pass green. Requiring > 1 makes that regression fail here, and it // also guarantees the loop runs at least once, so the first cap gap is genuinely // asserted retriable rather than vacuously skipped. assertTrue("the cap-gap settle budget must tolerate MORE THAN ONE gap, else a single " - + "transient failover to a smaller-cap node kills the sender " + + "transient failover to a smaller-cap node quarantines the slot " + "[MAX_CATCHUP_CAP_GAP_ATTEMPTS=" + maxAttempts + ']', maxAttempts > 1); // cap 160 => catch-up budget is below a ~216-byte solo frame for a 200-char symbol. @@ -512,7 +546,7 @@ public void testSuccessfulCatchUpResetsCapGapBudget() throws Exception { // gaps across reconnects; a successful catch-up ends the episode and MUST reset // it to 0 (sendDictCatchUp's final line). Otherwise cap gaps interspersed with // successful catch-ups -- a rolling-cap cluster where a larger-cap node comes - // and goes -- would accumulate to a spurious terminal over a long-lived sender. + // and goes -- would accumulate to a spurious terminal over a long-lived orphan drainer. // testCatchUpCapGapRetriesUntilBudgetThenLatches only accrues gaps under one // fixed cap with no success interleaved, so it cannot pin the reset. TestUtils.assertMemoryLeak(() -> { @@ -552,7 +586,7 @@ public void testSuccessfulCatchUpResetsCapGapBudget() throws Exception { // Behavioural proof the budget is genuinely fresh: max-1 more cap // gaps still latch NO terminal (they would if the counter had stayed - // at max-1 -- one more gap would have hit the cap and killed the sender). + // at max-1 -- one more gap would have quarantined the slot). client.cap = 160; for (int i = 1; i < maxAttempts; i++) { try { @@ -716,9 +750,8 @@ private CursorWebSocketSendLoop newLoop(CursorSendEngine engine, WebSocketClient /** * As {@link #newLoop(CursorSendEngine, WebSocketClient)} but with an explicit - * cap-gap escalation dwell. {@code 0} (the default here) means count-only - * escalation, which is what the raw constructor gives and what the strike-count - * tests want; the user-facing 5-minute default is applied at the config layer. + * cap-gap escalation dwell. These white-box tests model an orphan drainer, where + * {@code 0} means count-only quarantine; foreground loops retry indefinitely. */ private CursorWebSocketSendLoop newLoop( CursorSendEngine engine, WebSocketClient client, long capGapWindowMillis @@ -731,7 +764,24 @@ private CursorWebSocketSendLoop newLoop( 5_000L, 100L, 5_000L, false, CursorWebSocketSendLoop.DEFAULT_DURABLE_ACK_KEEPALIVE_INTERVAL_MILLIS, CursorWebSocketSendLoop.DEFAULT_MAX_HEAD_FRAME_REJECTIONS, - 0L, capGapWindowMillis); + 0L, capGapWindowMillis, + CursorWebSocketSendLoop.CatchUpCapGapPolicy.TERMINAL_AFTER_SETTLE_BUDGET); + } + + private CursorWebSocketSendLoop newForegroundLoop( + CursorSendEngine engine, WebSocketClient client + ) { + // Deliberately use the compatibility overload: its safe default must remain the + // foreground RETRY_FOREVER policy for external callers. + return new CursorWebSocketSendLoop( + client, engine, 0L, CursorWebSocketSendLoop.DEFAULT_PARK_NANOS, + () -> { + throw new UnsupportedOperationException("test loop is never started"); + }, + 5_000L, 100L, 5_000L, false, + CursorWebSocketSendLoop.DEFAULT_DURABLE_ACK_KEEPALIVE_INTERVAL_MILLIS, + CursorWebSocketSendLoop.DEFAULT_MAX_HEAD_FRAME_REJECTIONS, + 0L, 0L); } private CursorSendEngine newEngine() { From 84293fa1c92b61398c7f7b0ec0dc905acbaad130 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Wed, 15 Jul 2026 21:20:35 +0100 Subject: [PATCH 67/96] Reset cap-gap budget across unrelated outages --- .../sf/cursor/CursorWebSocketSendLoop.java | 88 +++++++++----- ...WebSocketSendLoopCatchUpAlignmentTest.java | 113 +++++++++++++++++- 2 files changed, 170 insertions(+), 31 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java index 2d94dc76..441edd90 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java @@ -163,8 +163,8 @@ public final class CursorWebSocketSendLoop implements QuietCloseable { private static final Logger LOG = LoggerFactory.getLogger(CursorWebSocketSendLoop.class); // Settle budget for the symbol-dict catch-up cap gap: how many cap-gap attempts // -- catch-ups that reached a fresh server and found a single dictionary entry - // too large for its advertised batch cap -- may occur, with no intervening - // SUCCESSFUL catch-up, before an orphan drainer latches a terminal. This is a + // too large for its advertised batch cap -- may occur consecutively before an + // orphan drainer latches a terminal. This is a // SANCTIONED orphan-only terminal (a genuine cluster batch-size capability gap), // the connect-time analog of the orphan drainer's durable-ack capability gap // (DEFAULT_MAX_DURABLE_ACK_MISMATCH_ATTEMPTS). A homogeneous cluster never trips @@ -173,18 +173,17 @@ public final class CursorWebSocketSendLoop implements QuietCloseable { // cluster, where a failover to a smaller-cap node can hit it for an entry an // earlier node accepted. A foreground sender retries forever. An orphan drainer // rides out the transient window until a larger-cap node returns; only a persistent - // gap (this many cap gaps with no successful catch-up in between) latches. + // gap (this many consecutive cap gaps with no successful catch-up or unrelated + // reconnect state in between) latches. // // Budget accounting (satisfies "a transient must never burn the terminal // budget"): catchUpCapGapAttempts increments ONLY inside sendDictCatchUp when a - // node is reached and an entry is oversized, and resets ONLY when a catch-up fully - // succeeds. A TRANSIENT reconnect (connect refuse, upgrade/role failure) never - // reaches the catch-up, so it NEITHER increments nor resets the counter -- it only - // lengthens the wall-clock settle window. The terminal therefore always requires - // this many GENUINE cap gaps; a transient can never inflate it. Deliberately NOT - // reset on a mere successful RECONNECT: a reconnect to the small-cap node itself - // produces the cap gap, so resetting there would stop a persistent gap from ever - // latching -- the reset must gate on a successful CATCH-UP, not on connecting. + // node is reached and an entry is oversized. A successful catch-up ends the + // episode, as does any unrelated reconnect state (connect refusal, catch-up send + // failure, upgrade/role rejection): otherwise its downtime would count toward the + // wall-clock dwell even though no cap gap was observed. The cap-gap exception + // itself does NOT reset the episode, so consecutive small-cap nodes still prove a + // persistent cluster capability gap and can exhaust the orphan policy. private static final int MAX_CATCHUP_CAP_GAP_ATTEMPTS = 16; // Hard ceiling for the lifetime-monotonic sent-dictionary mirror. The mirror // fields are int, so it cannot exceed Integer.MAX_VALUE bytes; reaching even @@ -302,19 +301,17 @@ public final class CursorWebSocketSendLoop implements QuietCloseable { private int sentDictBytesLen; private int sentDictCount; // Orphan-policy cap-gap attempts -- catch-ups that reached a node and found an entry - // too large for its batch cap -- since the last SUCCESSFUL catch-up (see - // MAX_CATCHUP_CAP_GAP_ATTEMPTS for the full budget accounting). Foreground retries - // never increment it. A successful catch-up resets it; a transient reconnect neither - // increments nor resets it. I/O-thread-only. + // too large for its batch cap -- with no intervening successful catch-up or unrelated + // reconnect state (see MAX_CATCHUP_CAP_GAP_ATTEMPTS for the full budget accounting). + // Foreground retries never increment it. I/O-thread-only. private int catchUpCapGapAttempts; // System.nanoTime() of the FIRST cap gap of the current orphan-policy episode. // Anchors the escalation dwell. Reset together with catchUpCapGapAttempts on a - // successful catch-up, so a node that accepts the dictionary ends the episode. - // Anchoring at the FIRST gap (not at loop entry, and not refreshed per attempt) is - // what keeps a TRANSIENT from burning the terminal budget: a transient never reaches - // the catch-up, so it neither increments the counter nor moves the anchor -- it can - // only lengthen the wall clock, which alone can never latch the terminal because the - // strike count still has to be satisfied. I/O-thread-only, like catchUpCapGapAttempts. + // successful catch-up or an unrelated reconnect state, so only an uninterrupted run + // of cap-gap observations contributes wall-clock dwell. Anchoring at the FIRST gap + // (not refreshed per attempt) lets consecutive small-cap nodes satisfy the dwell; + // resetting after transport/role states prevents their downtime from doing so. + // I/O-thread-only, like catchUpCapGapAttempts. // // -1 marks "no episode open" for a debugger or a heap dump, but it is NOT the test // for one -- catchUpCapGapAttempts == 0 is (see sendDictCatchUp). A nanoTime instant @@ -1436,6 +1433,15 @@ private void clearDurableAckTracking() { lastFrameOrPingNanos = 0L; } + private static boolean isCatchUpCapGap(Throwable t) { + return t instanceof CatchUpSendException && ((CatchUpSendException) t).capGap; + } + + private void resetCatchUpCapGapEpisode() { + catchUpCapGapAttempts = 0; + catchUpCapGapFirstNanos = -1L; + } + /** * Shared per-outage retry loop. Used by {@link #fail(Throwable)} for * mid-flight wire failures (phase="reconnect") and by @@ -1450,6 +1456,13 @@ private void connectLoop(Throwable initial, String phase, long paceFirstAttemptM recordFatal(initial); return; } + // A wire/role state that interrupted an orphan cap-gap run is not evidence that + // the cluster's batch cap stayed incompatible during the outage. Start a fresh + // episode; the cap-gap exception itself is the one reconnect cause that preserves + // the existing attempt count and dwell anchor. + if (!isCatchUpCapGap(initial)) { + resetCatchUpCapGapEpisode(); + } LOG.warn("cursor I/O loop entering {} loop: {}", phase, initial.getMessage()); long outageStartNanos = System.nanoTime(); @@ -1519,6 +1532,9 @@ private void connectLoop(Throwable initial, String phase, long paceFirstAttemptM phase, elapsedMs, attempts, fsnAtZero); return; } + // A null factory result is an unsuccessful connect state, not a cap-gap + // observation. Do not let the time spent retrying it satisfy orphan dwell. + resetCatchUpCapGapEpisode(); } catch (QwpAuthFailedException | WebSocketUpgradeException e) { // Terminal across all configured endpoints per spec sf-client.md // section 13.3: auth (401/403) bypasses reconnect and surfaces as @@ -1595,6 +1611,7 @@ private void connectLoop(Throwable initial, String phase, long paceFirstAttemptM // endpoint, forever. Growing to reconnectMaxBackoffMillis // mirrors the orphan drainer's role-reject path and honours the // documented capped-exponential-backoff contract. + resetCatchUpCapGapEpisode(); lastReconnectError = e; long now = System.nanoTime(); if (now - lastLogNanos >= RECONNECT_LOG_THROTTLE_NANOS) { @@ -1618,6 +1635,9 @@ private void connectLoop(Throwable initial, String phase, long paceFirstAttemptM recordFatal(e); throw (Error) e; } + if (!isCatchUpCapGap(e)) { + resetCatchUpCapGapEpisode(); + } lastReconnectError = e; long now = System.nanoTime(); if (now - lastLogNanos >= RECONNECT_LOG_THROTTLE_NANOS) { @@ -2490,10 +2510,10 @@ private int sendDictCatchUp() { // it to the producer. Only an orphan drainer may apply the settle budget // below and quarantine its slot after a persistent gap. Under budget the // throw is RETRIABLE (no recordFatal) -- connectLoop reconnects with - // backoff and re-runs the catch-up, which resets the counter on a node - // that accepts it. A transient reconnect (connect/upgrade failure, role - // reject) never reaches the catch-up, so it neither increments nor burns - // the orphan budget. On exhaustion latch via recordFatal, NOT fail() -- + // backoff and re-runs the catch-up, which resets the episode on a node + // that accepts it. An unrelated transport/upgrade/role state also resets + // the episode, so its downtime cannot satisfy the orphan dwell. On + // exhaustion latch via recordFatal, NOT fail() -- // failing from inside the catch-up would re-enter connectLoop (see // CatchUpSendException); the data must be resent after the cap is raised. // Escalation needs BOTH the strike count AND a minimum wall-clock dwell @@ -2517,7 +2537,7 @@ private int sendDictCatchUp() { throw new CatchUpSendException(new LineSenderException( "symbol dictionary entry too large for the server batch cap during catch-up [" + "frameLen=" + soloFrameLen + ", cap=" + cap + ']' - + "; retrying indefinitely -- a larger-cap node may return")); + + "; retrying indefinitely -- a larger-cap node may return"), true); } long nowNanos = System.nanoTime(); @@ -2540,7 +2560,7 @@ private int sendDictCatchUp() { if (exhausted) { recordFatal(err); } - throw new CatchUpSendException(err); + throw new CatchUpSendException(err, true); } if (chunkSymbols > 0 && chunkBytes + entryBytes > budget) { sendCatchUpChunk(chunkStartId, chunkSymbols, chunkStartAddr, (int) chunkBytes); @@ -2563,8 +2583,7 @@ private int sendDictCatchUp() { // count and the wall-clock anchor. Resetting only the count would leave a stale // anchor from an old episode, so the very first strike of a LATER episode would // already satisfy the dwell. - catchUpCapGapAttempts = 0; - catchUpCapGapFirstNanos = -1L; + resetCatchUpCapGapEpisode(); return framesSent; } @@ -2888,11 +2907,20 @@ public interface ReconnectFactory { * connectLoop}'s own retry catch (swapClient path), a fresh {@code fail()} * from the I/O loop body (trySendOne path), or dropping the dead client so * the I/O thread reconnects (start path). A JVM {@code Error} is never - * wrapped -- it must stay terminal. + * wrapped -- it must stay terminal. The {@code capGap} marker lets the reconnect + * loop preserve only consecutive incompatible-cap observations; ordinary catch-up + * send failures restart the orphan settle episode. */ private static final class CatchUpSendException extends RuntimeException { + private final boolean capGap; + CatchUpSendException(Throwable cause) { + this(cause, false); + } + + CatchUpSendException(Throwable cause, boolean capGap) { super(cause); + this.capGap = capGap; } } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java index fed94645..43bd9972 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java @@ -27,6 +27,7 @@ import io.questdb.client.DefaultHttpClientConfiguration; import io.questdb.client.cutlass.http.client.WebSocketClient; import io.questdb.client.cutlass.line.LineSenderException; +import io.questdb.client.cutlass.qwp.client.QwpRoleMismatchException; import io.questdb.client.cutlass.qwp.client.WebSocketResponse; import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine; import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorWebSocketSendLoop; @@ -40,12 +41,12 @@ import java.lang.reflect.Field; import java.lang.reflect.InvocationTargetException; -import java.util.concurrent.TimeUnit; import java.lang.reflect.Method; import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.concurrent.TimeUnit; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -472,6 +473,16 @@ public void testTransientCatchUpFailureDoesNotBurnTheCapGapBudget() throws Excep }); } + @Test + public void testTransportOutageRestartsCapGapEpisode() throws Exception { + assertUnrelatedReconnectStateRestartsCapGapEpisode(false); + } + + @Test + public void testRoleRejectRestartsCapGapEpisode() throws Exception { + assertUnrelatedReconnectStateRestartsCapGapEpisode(true); + } + @Test public void testCatchUpCapGapRetriesUntilBudgetThenLatches() throws Exception { // M1: an entry too large for the fresh server's cap during catch-up (a @@ -660,6 +671,76 @@ private static void appendFrames(CursorSendEngine engine, int count) { } } + private void assertUnrelatedReconnectStateRestartsCapGapEpisode(boolean roleReject) throws Exception { + // Accrue an orphan drainer's cap-gap strikes to one short of terminal, then + // simulate a long unrelated outage before another small-cap node appears. The + // outage must end the old episode: its wall-clock duration says nothing about + // whether the cluster's batch cap remained incompatible while no node answered. + TestUtils.assertMemoryLeak(() -> { + Field maxField = CursorWebSocketSendLoop.class.getDeclaredField("MAX_CATCHUP_CAP_GAP_ATTEMPTS"); + maxField.setAccessible(true); + int maxAttempts = maxField.getInt(null); + assertTrue("the cap-gap settle budget must have a retriable interval", maxAttempts > 1); + + int[] reconnectCalls = {0}; + long[] staleAnchor = {Long.MIN_VALUE}; + CursorWebSocketSendLoop[] loopRef = new CursorWebSocketSendLoop[1]; + try (CursorSendEngine engine = newEngine()) { + CursorWebSocketSendLoop loop = new CursorWebSocketSendLoop( + null, engine, 0L, CursorWebSocketSendLoop.DEFAULT_PARK_NANOS, + () -> { + int call = ++reconnectCalls[0]; + if (call < maxAttempts) { + return new CatchUpCapturingClient(160); + } + if (call == maxAttempts) { + assertEquals("precondition: consecutive cap gaps survive reconnect", + maxAttempts - 1, + readInt(loopRef[0], "catchUpCapGapAttempts")); + // Model the elapsed outage without sleeping. With the defect, + // this old anchor survives the unrelated failure and the next + // cap gap immediately satisfies both terminal conditions. + staleAnchor[0] = System.nanoTime() - TimeUnit.HOURS.toNanos(2); + setField(loopRef[0], "catchUpCapGapFirstNanos", staleAnchor[0]); + if (roleReject) { + throw new QwpRoleMismatchException( + "PRIMARY", null, "all endpoints role-rejected"); + } + throw new LineSenderException("transport unavailable"); + } + if (call == maxAttempts + 1) { + // Stop after getServerMaxBatchSize() has driven the final cap + // gap, leaving its fresh episode state observable below. + return new CatchUpCapturingClient(160, false, + () -> setBooleanFieldUnchecked(loopRef[0], "running", false)); + } + throw new AssertionError("unexpected reconnect call " + call); + }, + 5_000L, 0L, 0L, false, + CursorWebSocketSendLoop.DEFAULT_DURABLE_ACK_KEEPALIVE_INTERVAL_MILLIS, + CursorWebSocketSendLoop.DEFAULT_MAX_HEAD_FRAME_REJECTIONS, + 0L, TimeUnit.HOURS.toMillis(1), + CursorWebSocketSendLoop.CatchUpCapGapPolicy.TERMINAL_AFTER_SETTLE_BUDGET); + loopRef[0] = loop; + try { + seedMirror(loop, TestUtils.repeat("x", 200)); + setBooleanField(loop, "running", true); + invokeConnectLoop(loop); + + loop.checkError(); + assertEquals("pre-outage cap gaps must not carry into the new episode", + 1, readInt(loop, "catchUpCapGapAttempts")); + assertTrue("the post-outage cap gap must get a fresh dwell anchor", + readLong(loop, "catchUpCapGapFirstNanos") > staleAnchor[0]); + assertEquals("test must observe gaps, the unrelated state, and a new gap", + maxAttempts + 1, reconnectCalls[0]); + } finally { + loop.close(); + } + } + }); + } + // Builds a QWP delta frame [12-byte header][deltaStart varint][deltaCount // varint][ [len varint][utf8] ... ] for the given symbols. accumulateSentDict // skips the header, so its content is irrelevant; the caller frees the frame. @@ -744,6 +825,13 @@ private static void invokeSetWireBaselineWithCatchUp(CursorWebSocketSendLoop loo m.invoke(loop, replayStart); } + private static void invokeConnectLoop(CursorWebSocketSendLoop loop) throws Exception { + Method m = CursorWebSocketSendLoop.class.getDeclaredMethod( + "connectLoop", Throwable.class, String.class, long.class); + m.setAccessible(true); + m.invoke(loop, new LineSenderException("test reconnect"), "reconnect", 0L); + } + private CursorWebSocketSendLoop newLoop(CursorSendEngine engine, WebSocketClient client) { return newLoop(engine, client, 0L); } @@ -830,6 +918,20 @@ private static void setIntField(Object target, String name, int value) throws Ex f.setInt(target, value); } + private static void setBooleanField(Object target, String name, boolean value) throws Exception { + Field f = CursorWebSocketSendLoop.class.getDeclaredField(name); + f.setAccessible(true); + f.setBoolean(target, value); + } + + private static void setBooleanFieldUnchecked(Object target, String name, boolean value) { + try { + setBooleanField(target, name, value); + } catch (Exception e) { + throw new AssertionError(e); + } + } + private static int varintSize(long value) { int n = 1; while (value > 0x7F) { @@ -856,6 +958,7 @@ private static final class CatchUpCapturingClient extends WebSocketClient { // Mutable so a test can model a rolling-cap cluster: raise it for a node that // accepts the dictionary, lower it for a smaller-cap node that cap-gaps. private int cap; + private final Runnable onCapRead; private final boolean throwOnSend; private int framesSent; @@ -864,13 +967,21 @@ private static final class CatchUpCapturingClient extends WebSocketClient { } CatchUpCapturingClient(int cap, boolean throwOnSend) { + this(cap, throwOnSend, null); + } + + CatchUpCapturingClient(int cap, boolean throwOnSend, Runnable onCapRead) { super(DefaultHttpClientConfiguration.INSTANCE, PlainSocketFactory.INSTANCE); this.cap = cap; this.throwOnSend = throwOnSend; + this.onCapRead = onCapRead; } @Override public int getServerMaxBatchSize() { + if (onCapRead != null) { + onCapRead.run(); + } return cap; } From fe9546fad32a03d487cd91f03e23802b55563ed7 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Wed, 15 Jul 2026 21:39:41 +0100 Subject: [PATCH 68/96] Prevent orphan adoption during slot quarantine --- .../main/java/io/questdb/client/Sender.java | 184 ++++++++++-------- .../client/sf/cursor/BackgroundDrainer.java | 53 ++++- .../qwp/client/sf/cursor/SlotLock.java | 74 +++++-- .../qwp/client/DeltaDictRecoveryTest.java | 106 +++++++++- .../client/sf/cursor/OrphanScannerTest.java | 25 +++ .../qwp/client/sf/cursor/SlotLockTest.java | 24 +++ 6 files changed, 362 insertions(+), 104 deletions(-) diff --git a/core/src/main/java/io/questdb/client/Sender.java b/core/src/main/java/io/questdb/client/Sender.java index f73d3e35..be4fe409 100644 --- a/core/src/main/java/io/questdb/client/Sender.java +++ b/core/src/main/java/io/questdb/client/Sender.java @@ -40,6 +40,7 @@ import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorWebSocketSendLoop; import io.questdb.client.cutlass.qwp.client.sf.cursor.OrphanScanner; import io.questdb.client.cutlass.qwp.client.sf.cursor.PersistedSymbolDict; +import io.questdb.client.cutlass.qwp.client.sf.cursor.SlotLock; import io.questdb.client.cutlass.qwp.client.sf.cursor.UnreplayableSlotException; import io.questdb.client.impl.ConfStringParser; import io.questdb.client.impl.ConfigString; @@ -1012,6 +1013,8 @@ final class LineSenderBuilder { private static final int PROTOCOL_TCP = 0; private static final int PROTOCOL_UDP = 3; private static final int PROTOCOL_WEBSOCKET = 2; + @TestOnly + private static volatile Runnable quarantineAfterCloseHook; // Suffix for a slot set aside by quarantineTornSlot. Deliberately NOT the // sender's own slot name, so a restarted sender does not re-adopt it as its own; // quarantineTornSlot then marks it .failed, so the orphan drainer skips it too and @@ -1559,91 +1562,101 @@ public Sender build() { sfAppendDeadlineMillis == PARAMETER_NOT_SET_EXPLICITLY ? CursorSendEngine.DEFAULT_APPEND_DEADLINE_NANOS : sfAppendDeadlineMillis * 1_000_000L; - CursorSendEngine cursorEngine = new CursorSendEngine( - slotPath, actualSfMaxBytes, - actualSfMaxTotalBytes, actualSfAppendDeadlineNanos); - int actualErrorInboxCapacity = errorInboxCapacity != PARAMETER_NOT_SET_EXPLICITLY - ? errorInboxCapacity - : io.questdb.client.cutlass.qwp.client.sf.cursor.SenderErrorDispatcher.DEFAULT_CAPACITY; - int actualConnectionListenerInboxCapacity = connectionListenerInboxCapacity != PARAMETER_NOT_SET_EXPLICITLY - ? connectionListenerInboxCapacity - : io.questdb.client.cutlass.qwp.client.sf.cursor.SenderConnectionDispatcher.DEFAULT_CAPACITY; - List wsEndpoints = - new ArrayList<>(hosts.size()); - for (int i = 0, n = hosts.size(); i < n; i++) { - wsEndpoints.add(new QwpWebSocketSender.Endpoint(hosts.getQuick(i), ports.getQuick(i))); - } - // The recovery seed inside connect() is the authority on whether a recovered - // slot can be replayed: it rebuilds the dictionary from its intact prefix and - // then from the surviving frames' own delta sections, and throws - // UnreplayableSlotException only once neither source holds the missing ids. - // Quarantining on anything weaker would set aside slots that recovery can - // still rescue, so build() waits for that verdict rather than pre-judging it. QwpWebSocketSender connected = null; - boolean quarantined = false; - while (connected == null) { - try { - connected = QwpWebSocketSender.connect( - wsEndpoints, - wsTlsConfig, - actualAutoFlushRows, - actualAutoFlushBytes, - actualAutoFlushIntervalNanos, - wsAuthHeader, - requestDurableAck, - cursorEngine, - actualCloseFlushTimeoutMillis, - actualReconnectMaxDurationMillis, - actualReconnectInitialBackoffMillis, - actualReconnectMaxBackoffMillis, - actualInitialConnectMode, - errorHandler, - actualErrorInboxCapacity, - actualDurableAckKeepaliveIntervalMillis, - authTimeoutMillis, - connectTimeoutMillis == PARAMETER_NOT_SET_EXPLICITLY ? 0 : connectTimeoutMillis, - connectionListener, - actualConnectionListenerInboxCapacity, - actualMaxFrameRejections, - actualPoisonMinEscalationWindowMillis, - actualCatchUpCapGapMinEscalationWindowMillis - ); - } catch (UnreplayableSlotException e) { - // The one failure build() recovers from. The slot's frames reference ids - // that nothing still holds, so they can never go on the wire -- but that is - // no reason to take the producer down with them. Before this, the throw - // escaped build() and, because senderId is stable and a not-fully-drained - // slot is retained on close, every retry re-recovered the same slot and - // threw again: the application could not construct a Sender at all, so it - // could not even BUFFER new rows. An already-lost batch became an unbounded - // outage of everything after it. - // - // Set the slot aside instead, keep its bytes for forensics and resend, and - // start the producer on a clean one. Once only: a second such failure would - // mean the FRESH slot is unreplayable, which cannot happen, so let it out - // rather than loop. - if (quarantined || slotPath == null) { + // The parent-anchored logical lock is stable across a slot rename. Keep it + // from before the directory-local lock is acquired until connect() has either + // adopted that engine or quarantine has closed, renamed and recreated it. + // This closes the inode-swap window in which an already-queued orphan drainer + // could otherwise acquire the renamed directory's old .lock and later operate + // on the fresh slot through the original pathname. + try (SlotLock logicalSlotLock = slotPath == null + ? null + : SlotLock.acquireLogical(slotPath)) { + CursorSendEngine cursorEngine = new CursorSendEngine( + slotPath, actualSfMaxBytes, + actualSfMaxTotalBytes, actualSfAppendDeadlineNanos); + int actualErrorInboxCapacity = errorInboxCapacity != PARAMETER_NOT_SET_EXPLICITLY + ? errorInboxCapacity + : io.questdb.client.cutlass.qwp.client.sf.cursor.SenderErrorDispatcher.DEFAULT_CAPACITY; + int actualConnectionListenerInboxCapacity = connectionListenerInboxCapacity != PARAMETER_NOT_SET_EXPLICITLY + ? connectionListenerInboxCapacity + : io.questdb.client.cutlass.qwp.client.sf.cursor.SenderConnectionDispatcher.DEFAULT_CAPACITY; + List wsEndpoints = + new ArrayList<>(hosts.size()); + for (int i = 0, n = hosts.size(); i < n; i++) { + wsEndpoints.add(new QwpWebSocketSender.Endpoint(hosts.getQuick(i), ports.getQuick(i))); + } + // The recovery seed inside connect() is the authority on whether a recovered + // slot can be replayed: it rebuilds the dictionary from its intact prefix and + // then from the surviving frames' own delta sections, and throws + // UnreplayableSlotException only once neither source holds the missing ids. + // Quarantining on anything weaker would set aside slots that recovery can + // still rescue, so build() waits for that verdict rather than pre-judging it. + boolean quarantined = false; + while (connected == null) { try { - cursorEngine.close(); - } catch (Throwable ignored) { - // best-effort + connected = QwpWebSocketSender.connect( + wsEndpoints, + wsTlsConfig, + actualAutoFlushRows, + actualAutoFlushBytes, + actualAutoFlushIntervalNanos, + wsAuthHeader, + requestDurableAck, + cursorEngine, + actualCloseFlushTimeoutMillis, + actualReconnectMaxDurationMillis, + actualReconnectInitialBackoffMillis, + actualReconnectMaxBackoffMillis, + actualInitialConnectMode, + errorHandler, + actualErrorInboxCapacity, + actualDurableAckKeepaliveIntervalMillis, + authTimeoutMillis, + connectTimeoutMillis == PARAMETER_NOT_SET_EXPLICITLY ? 0 : connectTimeoutMillis, + connectionListener, + actualConnectionListenerInboxCapacity, + actualMaxFrameRejections, + actualPoisonMinEscalationWindowMillis, + actualCatchUpCapGapMinEscalationWindowMillis + ); + } catch (UnreplayableSlotException e) { + // The one failure build() recovers from. The slot's frames reference ids + // that nothing still holds, so they can never go on the wire -- but that is + // no reason to take the producer down with them. Before this, the throw + // escaped build() and, because senderId is stable and a not-fully-drained + // slot is retained on close, every retry re-recovered the same slot and + // threw again: the application could not construct a Sender at all, so it + // could not even BUFFER new rows. An already-lost batch became an unbounded + // outage of everything after it. + // + // Set the slot aside instead, keep its bytes for forensics and resend, and + // start the producer on a clean one. Once only: a second such failure would + // mean the FRESH slot is unreplayable, which cannot happen, so let it out + // rather than loop. + if (quarantined || slotPath == null) { + try { + cursorEngine.close(); + } catch (Throwable ignored) { + // best-effort + } + throw e; + } + quarantined = true; + cursorEngine = quarantineTornSlot( + cursorEngine, e, sfDir, senderId, slotPath, actualSfMaxBytes, + actualSfMaxTotalBytes, actualSfAppendDeadlineNanos); + } catch (Throwable t) { + // connect() failed before ownership of cursorEngine + // transferred — close it ourselves. + try { + cursorEngine.close(); + } catch (Throwable ignored) { + // best-effort + } + throw t; } - throw e; } - quarantined = true; - cursorEngine = quarantineTornSlot( - cursorEngine, e, sfDir, senderId, slotPath, actualSfMaxBytes, - actualSfMaxTotalBytes, actualSfAppendDeadlineNanos); - } catch (Throwable t) { - // connect() failed before ownership of cursorEngine - // transferred — close it ourselves. - try { - cursorEngine.close(); - } catch (Throwable ignored) { - // best-effort - } - throw t; - } } // connect() succeeded — `connected` now owns cursorEngine // via setCursorEngine(engine, true). From here on, ANY @@ -3047,6 +3060,10 @@ private static CursorSendEngine quarantineTornSlot( // path already closed the engine; close() is idempotent, so make it explicit rather // than depend on that. torn.close(); + Runnable hook = quarantineAfterCloseHook; + if (hook != null) { + hook.run(); + } String quarantinePath = null; for (int i = 0; i < MAX_QUARANTINE_SLOT_ATTEMPTS; i++) { @@ -3074,6 +3091,11 @@ private static CursorSendEngine quarantineTornSlot( return new CursorSendEngine(slotPath, sfMaxBytes, sfMaxTotalBytes, sfAppendDeadlineNanos); } + @TestOnly + public static void setQuarantineAfterCloseHookForTest(Runnable hook) { + quarantineAfterCloseHook = hook; + } + private static int resolveIPv4(String host) { try { byte[] addr = InetAddress.getByName(host).getAddress(); diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/BackgroundDrainer.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/BackgroundDrainer.java index 0e7bcd10..8e287b4a 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/BackgroundDrainer.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/BackgroundDrainer.java @@ -44,7 +44,9 @@ *

    * Lifecycle: *

      - *
    1. Acquire the slot's {@code .lock}; skip silently on contention.
      2. + *
    2. Acquire the parent-anchored logical-slot lock and revalidate the + * scanner snapshot; skip silently on contention or a stale snapshot.
      3. + *
    3. Acquire the slot's {@code .lock}, then release the logical lock.
      4. *
    4. Open a {@link CursorSendEngine} on the slot — recovery picks up * every {@code .sfa} file already on disk.
      5. *
    5. Open a fresh {@link WebSocketClient} via the supplied factory @@ -529,27 +531,54 @@ private boolean stopRequestedOrInterrupted() { @Override public void run() { runnerThread = Thread.currentThread(); + SlotLock logicalSlotLock = null; CursorSendEngine engine = null; WebSocketClient client = null; CursorWebSocketSendLoop loop = null; try { - // The engine acquires the slot's .lock itself — we don't need - // (and must not) double-lock it. If another sender or drainer - // holds it, the engine constructor throws and we exit silently - // (no .failed sentinel — contention is expected, not an error). + // Scanner results are only snapshots. Serialize adoption against + // a producer's close -> quarantine rename -> fresh-slot recreate + // transition, then revalidate while that stable parent-anchored + // lock is held. The slot's own .lock inode moves with a rename and + // cannot provide this guarantee by itself. + if (slotPath != null) { + try { + logicalSlotLock = SlotLock.acquireLogical(slotPath); + } catch (IllegalStateException t) { + if (isLockContention(t)) { + LOG.info("orphan logical slot already locked, skipping: {} ({})", + slotPath, t.getMessage()); + outcome = DrainOutcome.LOCKED_BY_OTHER; + return; + } + throw t; + } + if (!OrphanScanner.isCandidateOrphan(slotPath)) { + LOG.info("orphan candidate changed before adoption, skipping: {}", slotPath); + outcome = DrainOutcome.SUCCESS; + return; + } + } + + // The engine acquires the directory-local .lock itself. Keep the + // lock order logical -> local, and release the short-lived logical + // lock only after the engine has secured stable ownership. try { engine = new CursorSendEngine(slotPath, segmentSizeBytes, sfMaxTotalBytes, CursorSendEngine.DEFAULT_APPEND_DEADLINE_NANOS); } catch (IllegalStateException t) { - String msg = t.getMessage(); - if (msg != null && msg.contains("already in use")) { + if (isLockContention(t)) { LOG.info("orphan slot already locked, skipping: {} ({})", - slotPath, msg); + slotPath, t.getMessage()); outcome = DrainOutcome.LOCKED_BY_OTHER; return; } throw t; } + if (logicalSlotLock != null) { + logicalSlotLock.close(); + logicalSlotLock = null; + } long target = engine.publishedFsn(); if (engine.ackedFsn() >= target) { LOG.info("orphan slot already drained: {} (acked={} target={})", @@ -724,12 +753,20 @@ public void run() { + "slot lock releases when it exits", slotPath); } } + if (logicalSlotLock != null) { + logicalSlotLock.close(); + } // Don't let a later requestStop() unpark an unrelated task that // the pool's executor may have scheduled onto this same thread. runnerThread = null; } } + private static boolean isLockContention(IllegalStateException error) { + String message = error.getMessage(); + return message != null && message.contains("already in use"); + } + /** * Plug an observer for durable-ack-related events. {@code null} clears * any previously installed listener. See {@link BackgroundDrainerListener} diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SlotLock.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SlotLock.java index 0b8379de..eb6281a7 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SlotLock.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SlotLock.java @@ -33,10 +33,14 @@ import java.nio.charset.StandardCharsets; /** - * Advisory exclusive lock for a single SF slot directory. + * Advisory exclusive locks for a single SF slot. *

      - * One {@code .lock} file per slot, held via {@code flock}/{@code LockFileEx} - * for the entire lifetime of the engine that owns the slot. The lock is + * {@link #acquire(String)} locks a {@code .lock} file inside the slot directory + * for the entire lifetime of the engine that owns it. {@link #acquireLogical(String)} + * locks a sibling file under the parent SF directory for short-lived pathname + * transitions and orphan adoption; because it is outside the slot directory, + * it remains stable if that directory is renamed. Both use + * {@code flock}/{@code LockFileEx}. A lock is * automatically released when the fd is closed — including on hard process * exit, since the kernel cleans up file locks for terminated processes. *

      @@ -58,6 +62,7 @@ public final class SlotLock implements QuietCloseable { private static final String LOCK_FILE_NAME = ".lock"; private static final String LOCK_PID_FILE_NAME = ".lock.pid"; + private static final String LOGICAL_LOCK_DIR_NAME = ".slot-locks"; private final String slotDir; private int fd; @@ -75,18 +80,42 @@ private SlotLock(String slotDir, int fd) { * or lock contention. */ public static SlotLock acquire(String slotDir) { - if (slotDir == null || slotDir.isEmpty()) { - throw new IllegalArgumentException("slotDir must not be empty"); - } - if (!Files.exists(slotDir)) { - int rc = Files.mkdir(slotDir, Files.DIR_MODE_DEFAULT); - if (rc != 0) { - throw new IllegalStateException( - "could not create slot dir: " + slotDir + " rc=" + rc); - } - } + validateSlotDir(slotDir); + ensureDirectory(slotDir, "slot dir"); String lockPath = slotDir + "/" + LOCK_FILE_NAME; String pidPath = slotDir + "/" + LOCK_PID_FILE_NAME; + return acquireAt(slotDir, lockPath, pidPath); + } + + /** + * Acquires the stable logical lock for {@code slotDir}. Unlike the + * directory-local {@code .lock}, this lock is anchored in the parent SF + * directory, so renaming the slot cannot move the lock inode away from the + * logical slot name it guards. + *

      + * Callers use this as a short-lived transition/adoption lock, always before + * acquiring the directory-local lock. In particular it must cover an + * unreplayable slot's close -> rename -> recreate transition, preventing a + * queued orphan drainer from adopting the renamed inode and later touching + * the fresh directory through the old pathname. + */ + public static SlotLock acquireLogical(String slotDir) { + validateSlotDir(slotDir); + int separator = slotDir.lastIndexOf('/'); + if (separator < 1 || separator == slotDir.length() - 1) { + throw new IllegalArgumentException( + "slotDir must contain a parent and slot name: " + slotDir); + } + String parentDir = slotDir.substring(0, separator); + String slotName = slotDir.substring(separator + 1); + String logicalLockDir = parentDir + "/" + LOGICAL_LOCK_DIR_NAME; + ensureDirectory(logicalLockDir, "logical slot lock dir"); + String lockPath = logicalLockDir + "/" + slotName + ".lock"; + String pidPath = logicalLockDir + "/" + slotName + ".lock.pid"; + return acquireAt(slotDir, lockPath, pidPath); + } + + private static SlotLock acquireAt(String slotDir, String lockPath, String pidPath) { int fd = Files.openRW(lockPath); if (fd < 0) { throw new IllegalStateException( @@ -111,6 +140,25 @@ public static SlotLock acquire(String slotDir) { } } + private static void ensureDirectory(String path, String description) { + if (!Files.exists(path)) { + int rc = Files.mkdir(path, Files.DIR_MODE_DEFAULT); + // Multiple senders may create the shared parent lock directory + // concurrently. Treat EEXIST as success, just as the builder does + // for the SF root itself. + if (rc != 0 && !Files.exists(path)) { + throw new IllegalStateException( + "could not create " + description + ": " + path + " rc=" + rc); + } + } + } + + private static void validateSlotDir(String slotDir) { + if (slotDir == null || slotDir.isEmpty()) { + throw new IllegalArgumentException("slotDir must not be empty"); + } + } + /** Slot dir this lock guards. */ public String slotDir() { return slotDir; diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java index b4d91895..d14c97a8 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java @@ -27,10 +27,13 @@ import io.questdb.client.Sender; import io.questdb.client.cutlass.line.LineSenderException; import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender; +import io.questdb.client.cutlass.qwp.client.sf.cursor.BackgroundDrainer; import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine; -import io.questdb.client.test.tools.DelegatingFilesFacade; +import io.questdb.client.cutlass.qwp.client.sf.cursor.OrphanScanner; import io.questdb.client.cutlass.qwp.client.sf.cursor.PersistedSymbolDict; +import io.questdb.client.std.ObjList; import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer; +import io.questdb.client.test.tools.DelegatingFilesFacade; import io.questdb.client.test.tools.TestUtils; import org.junit.After; import org.junit.Assert; @@ -46,9 +49,11 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; -import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; @@ -249,6 +254,103 @@ public void testTrimmedRegisteringFramesAreUnreplayableAndTheSlotIsSetAside() th }); } + @Test + public void testQueuedOrphanCannotAdoptSlotWhileQuarantineRecreatesItsName() throws Exception { + // Sender 1 leaves a genuinely unreplayable default slot. Sender 2 recovers it, + // closes the directory-local lock, quarantines the directory and recreates the + // default pathname. A scanner may already have queued that pathname before sender + // 2 starts; the queued drainer must not acquire the old lock inode in the close -> + // rename gap and later issue path-based writes/unlinks against sender 2's fresh slot. + assertMemoryLeak(() -> { + writeAndTearUnreplayableSlot(); + ObjList scannerSnapshot = OrphanScanner.scan(sfDir, "other-sender"); + Assert.assertEquals(1, scannerSnapshot.size()); + String staleSnapshotPath = scannerSnapshot.get(0); + + CountDownLatch producerInRenameGap = new CountDownLatch(1); + CountDownLatch allowQuarantineRename = new CountDownLatch(1); + AtomicReference recoveredSender = new AtomicReference<>(); + AtomicReference recoveryFailure = new AtomicReference<>(); + Thread recoveryThread = null; + Sender.LineSenderBuilder.setQuarantineAfterCloseHookForTest(() -> { + producerInRenameGap.countDown(); + try { + if (!allowQuarantineRename.await(15, TimeUnit.SECONDS)) { + throw new AssertionError("timed out waiting to finish quarantine rename"); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new AssertionError("quarantine hook interrupted", e); + } + }); + try { + DictReconstructingHandler handler = new DictReconstructingHandler(); + try (TestWebSocketServer good = new TestWebSocketServer(handler)) { + int port = good.getPort(); + good.start(); + Assert.assertTrue(good.awaitStart(5, TimeUnit.SECONDS)); + String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";"; + + recoveryThread = new Thread(() -> { + try { + recoveredSender.set(Sender.fromConfig(cfg)); + } catch (Throwable t) { + recoveryFailure.set(t); + } + }, "qwp-quarantine-recovery"); + recoveryThread.start(); + Assert.assertTrue("recovering sender did not reach the quarantine gap", + producerInRenameGap.await(10, TimeUnit.SECONDS)); + + BackgroundDrainer queuedDrainer = new BackgroundDrainer( + staleSnapshotPath, 256, 8192, () -> null, + 1000, 1, 10, true, 0); + Thread drainerThread = new Thread(queuedDrainer, "qwp-queued-orphan"); + drainerThread.start(); + drainerThread.join(5_000); + Assert.assertFalse("queued drainer must not wait on or adopt the old inode", + drainerThread.isAlive()); + Assert.assertEquals(BackgroundDrainer.DrainOutcome.LOCKED_BY_OTHER, + queuedDrainer.outcome()); + + allowQuarantineRename.countDown(); + recoveryThread.join(10_000); + Assert.assertFalse("recovering sender did not finish", recoveryThread.isAlive()); + if (recoveryFailure.get() != null) { + throw new AssertionError("recovering sender failed", recoveryFailure.get()); + } + Sender sender = recoveredSender.get(); + Assert.assertNotNull(sender); + sender.table("m").symbol("s", "after-race").longColumn("v", 99).atNow(); + sender.flush(); + long deadline = System.currentTimeMillis() + 10_000; + while (System.currentTimeMillis() < deadline && handler.maxDictSize() < 1) { + Thread.sleep(20); + } + sender.close(); + recoveredSender.set(null); + + assertUnreplayableSlotSetAside(); + Assert.assertEquals("fresh producer slot must remain intact and usable", + Arrays.asList("after-race"), handler.dictSnapshot()); + } + } finally { + allowQuarantineRename.countDown(); + if (recoveryThread != null) { + recoveryThread.join(10_000); + if (recoveryThread.isAlive()) { + recoveryThread.interrupt(); + } + } + Sender sender = recoveredSender.getAndSet(null); + if (sender != null) { + sender.close(); + } + Sender.LineSenderBuilder.setQuarantineAfterCloseHookForTest(null); + } + }); + } + @Test public void testFullyAckedTornSlotResumesInPlaceWithoutQuarantine() throws Exception { // M1 regression -- the ACKED counterpart to diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/OrphanScannerTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/OrphanScannerTest.java index 162474fe..a39bef67 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/OrphanScannerTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/OrphanScannerTest.java @@ -24,6 +24,7 @@ package io.questdb.client.test.cutlass.qwp.client.sf.cursor; +import io.questdb.client.cutlass.qwp.client.sf.cursor.BackgroundDrainer; import io.questdb.client.cutlass.qwp.client.sf.cursor.OrphanScanner; import io.questdb.client.std.Files; import io.questdb.client.std.IntList; @@ -321,6 +322,30 @@ public void testIsCandidateOrphanDirect() throws Exception { }); } + @Test + public void testDrainerRevalidatesStaleScannerSnapshotBeforeCreatingEngine() throws Exception { + TestUtils.assertMemoryLeak(() -> { + String slot = sfDir + "/stale"; + assertEquals(0, Files.mkdir(slot, Files.DIR_MODE_DEFAULT)); + String segment = slot + "/sf-0001.sfa"; + touchFile(segment); + assertTrue(OrphanScanner.isCandidateOrphan(slot)); + + // The scanner has already queued this pathname, but another owner + // drains/removes its last segment before the worker gets to run. + assertTrue(Files.remove(segment)); + BackgroundDrainer drainer = new BackgroundDrainer( + slot, 1024, 8192, () -> { + throw new AssertionError("a stale candidate must not connect"); + }, 1000, 1, 10, true, 0); + drainer.run(); + + assertEquals(BackgroundDrainer.DrainOutcome.SUCCESS, drainer.outcome()); + assertFalse("stale adoption must not create the directory-local lock", + Files.exists(slot + "/.lock")); + }); + } + private static void touchFile(String path) { int fd = Files.openRW(path); if (fd >= 0) Files.close(fd); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SlotLockTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SlotLockTest.java index 644a7463..6643bff8 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SlotLockTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SlotLockTest.java @@ -103,6 +103,30 @@ public void testCloseReleasesLock() throws Exception { }); } + @Test + public void testLogicalLockRemainsContendedAcrossSlotRenameAndRecreate() throws Exception { + TestUtils.assertMemoryLeak(() -> { + String slot = parentDir + "/rename"; + String moved = parentDir + "/rename.quarantined"; + assertEquals(0, Files.mkdir(slot, Files.DIR_MODE_DEFAULT)); + + try (SlotLock ignored = SlotLock.acquireLogical(slot)) { + assertEquals(0, Files.rename(slot, moved)); + assertEquals(0, Files.mkdir(slot, Files.DIR_MODE_DEFAULT)); + + try (SlotLock unexpected = SlotLock.acquireLogical(slot)) { + fail("logical slot lock must survive rename and recreate"); + } catch (IllegalStateException expected) { + assertTrue(expected.getMessage().contains("already in use")); + } + } + + try (SlotLock reacquired = SlotLock.acquireLogical(slot)) { + assertEquals(slot, reacquired.slotDir()); + } + }); + } + @Test public void testTwoDifferentSlotsCoexist() throws Exception { TestUtils.assertMemoryLeak(() -> { From 6328467c85add2f5823612d59ae4c60ee8d433b1 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Wed, 15 Jul 2026 21:52:14 +0100 Subject: [PATCH 69/96] Verify delivery after torn-slot recovery --- .../client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java index d14c97a8..f9f0e968 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java @@ -394,6 +394,10 @@ public void testFullyAckedTornSlotResumesInPlaceWithoutQuarantine() throws Excep while (System.currentTimeMillis() < deadline && handler.maxDictSize() < 1) { Thread.sleep(20); } + Assert.assertEquals( + "the resumed sender must deliver new data from the original slot", + Arrays.asList("after-recovery"), + handler.dictSnapshot()); } } // The fully-acked slot was NOT quarantined: no set-aside copy exists (the From 5617d7906775cfe21c5befde7557637ac8dd8259 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Wed, 15 Jul 2026 22:21:06 +0100 Subject: [PATCH 70/96] Optimize symbol dictionary recovery and catch-up --- .../client/sf/cursor/CursorSendEngine.java | 84 ++++-- .../sf/cursor/CursorWebSocketSendLoop.java | 71 ++++- .../qwp/client/sf/cursor/MmapSegment.java | 14 + .../client/sf/cursor/PersistedSymbolDict.java | 189 ++++++++++++- .../sf/cursor/RecoveredFrameAnalysis.java | 261 ++++++++++++++++++ .../qwp/client/sf/cursor/SegmentRing.java | 18 ++ ...WebSocketSendLoopCatchUpAlignmentTest.java | 32 +++ ...CursorWebSocketSendLoopOrphanTailTest.java | 52 ++++ .../sf/cursor/PersistedSymbolDictTest.java | 34 +++ 9 files changed, 716 insertions(+), 39 deletions(-) create mode 100644 core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/RecoveredFrameAnalysis.java diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java index 6b1d741a..5fdc129c 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java @@ -137,6 +137,10 @@ public final class CursorSendEngine implements QuietCloseable { // constructor, closed by {@link #close()}. When null in disk mode the engine // reports delta encoding as unavailable and the sender keeps full-dict frames. private final PersistedSymbolDict persistedSymbolDict; + // Engine-owned output of the single ordered recovery walk. It is retained + // because both producer seeding and every recycled send loop need the same + // frame-rebuilt symbol suffix. Null for fresh and memory-only engines. + private final RecoveredFrameAnalysis recoveredFrameAnalysis; // close() is publicly callable from any thread (Sender.close from a user // thread, JVM shutdown hooks, test cleanup). volatile + synchronized // close() makes the check-and-set atomic and gives readers a fence. @@ -248,6 +252,7 @@ private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager man SegmentRing ringInProgress = null; AckWatermark watermarkInProgress = null; PersistedSymbolDict persistedDictInProgress = null; + RecoveredFrameAnalysis recoveredFrameAnalysisInProgress = null; try { // Disk mode: try to recover any *.sfa files left behind by a prior // session before deciding to start fresh. Without this the engine @@ -329,6 +334,12 @@ private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager man if (seed >= 0) { recovered.acknowledge(seed); } + // Fold the whole recovered ring once. The result checkpoints all + // running metadata and raw symbol bytes at each commit-bearing + // frame, so its final snapshot excludes an orphan deferred tail + // without requiring a second bounded scan. + recoveredFrameAnalysisInProgress = recovered.analyzeRecovery( + persistedDictInProgress == null ? 0 : persistedDictInProgress.size()); // Locate the last commit-bearing frame below a potentially // orphaned FLAG_DEFER_COMMIT tail. A producer that crashed (or // closed) mid-transaction leaves deferred frames with no @@ -338,12 +349,7 @@ private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager man // drainOnClose), and (b) replaying them into a NEW session's // commit would resurrect half a transaction -- see the WARN // below. Computed before the I/O loop or producer append. - this.recoveredCommitBoundaryFsn = recovered.findLastFsnWithoutPayloadFlag( - QwpConstants.HEADER_OFFSET_FLAGS, - QwpConstants.FLAG_DEFER_COMMIT, - QwpConstants.MAGIC_MESSAGE, - QwpConstants.HEADER_SIZE - ); + this.recoveredCommitBoundaryFsn = recoveredFrameAnalysisInProgress.commitBoundaryFsn(); if (publishedFsn >= 0 && recoveredCommitBoundaryFsn < publishedFsn) { this.recoveredOrphanTipFsn = publishedFsn; LOG.warn("recovered SF log ends with {} deferred frame(s) whose transaction was never " @@ -363,12 +369,7 @@ private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager man // this and over-reject an otherwise-recoverable slot. maxSymbolDeltaEnd // returns 0 when no such frame carries a symbol, yielding -1 here. // Computed before the I/O loop or producer append; single-threaded. - this.recoveredMaxSymbolId = recovered.maxSymbolDeltaEnd( - QwpConstants.MAGIC_MESSAGE, - QwpConstants.HEADER_OFFSET_FLAGS, - QwpConstants.FLAG_DELTA_SYMBOL_DICT, - QwpConstants.HEADER_SIZE, - recoveredCommitBoundaryFsn) - 1L; + this.recoveredMaxSymbolId = recoveredFrameAnalysisInProgress.maxDeltaEnd() - 1L; // Full-dict-fallback recovery. When the persisted .symbol-dict is a // SUBSET of the ids the surviving frames reference // (recoveredMaxSymbolId >= its size) YET every such frame is @@ -388,12 +389,7 @@ private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager man // dictionary. The recoveredMaxSymbolId >= size guard means this never // fires for a slot whose dictionary is intact, nor for an empty slot // (recoveredMaxSymbolId == -1). Single-threaded; before the I/O loop. - this.recoveredMaxSymbolDeltaStart = recovered.maxSymbolDeltaStart( - QwpConstants.MAGIC_MESSAGE, - QwpConstants.HEADER_OFFSET_FLAGS, - QwpConstants.FLAG_DELTA_SYMBOL_DICT, - QwpConstants.HEADER_SIZE, - recoveredCommitBoundaryFsn); + this.recoveredMaxSymbolDeltaStart = recoveredFrameAnalysisInProgress.maxDeltaStart(); if (persistedDictInProgress != null && recoveredMaxSymbolId >= persistedDictInProgress.size() && recoveredMaxSymbolDeltaStart == 0L) { @@ -451,6 +447,7 @@ private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager man this.ring = ringInProgress; this.watermark = watermarkInProgress; this.persistedSymbolDict = persistedDictInProgress; + this.recoveredFrameAnalysis = recoveredFrameAnalysisInProgress; } catch (Throwable t) { // Stop an owned manager before freeing the ring and watermark it may // touch, then release the slot lock. Each cleanup is in its own @@ -482,6 +479,12 @@ private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager man } catch (Throwable ignored) { } } + if (recoveredFrameAnalysisInProgress != null) { + try { + recoveredFrameAnalysisInProgress.close(); + } catch (Throwable ignored) { + } + } if (acquiredLock != null) { try { acquiredLock.close(); @@ -667,6 +670,12 @@ public synchronized void close() { } catch (Throwable ignored) { } } + if (recoveredFrameAnalysis != null) { + try { + recoveredFrameAnalysis.close(); + } catch (Throwable ignored) { + } + } if (fullyDrained) { try { unlinkAllSegmentFiles(sfDir); @@ -716,6 +725,11 @@ public synchronized void close() { * transmitted, so their ids never reach a server and must not inflate the baseline. */ public long collectReplaySymbolsAbove(int baseline, ObjList out) { + if (recoveredFrameAnalysis != null + && recoveredFrameAnalysis.baseline() == baseline) { + recoveredFrameAnalysis.appendDecodedSymbols(out); + return recoveredFrameAnalysis.coverage(); + } if (ring == null) { return baseline; } @@ -738,6 +752,40 @@ public long collectReplaySymbolsAbove(int baseline, ObjList out) { ); } + long recoveredSymbolCoverage(int baseline) { + return checkedRecoveryAnalysis(baseline).coverage(); + } + + int recoveredSymbolSuffixCount(int baseline) { + return checkedRecoveryAnalysis(baseline).rawCount(); + } + + int recoveredSymbolSuffixLen(int baseline) { + return checkedRecoveryAnalysis(baseline).rawLen(); + } + + void copyRecoveredSymbolSuffix(int baseline, long target) { + RecoveredFrameAnalysis analysis = checkedRecoveryAnalysis(baseline); + int len = analysis.rawLen(); + if (len > 0) { + io.questdb.client.std.Unsafe.getUnsafe().copyMemory(analysis.rawAddr(), target, len); + } + } + + @TestOnly + public long recoveryFramesVisited() { + return recoveredFrameAnalysis == null ? 0L : recoveredFrameAnalysis.framesVisited(); + } + + private RecoveredFrameAnalysis checkedRecoveryAnalysis(int baseline) { + if (recoveredFrameAnalysis == null || recoveredFrameAnalysis.baseline() != baseline) { + throw new IllegalStateException("recovery symbol baseline mismatch [expected=" + + (recoveredFrameAnalysis == null ? "none" : recoveredFrameAnalysis.baseline()) + + ", actual=" + baseline + ']'); + } + return recoveredFrameAnalysis; + } + /** * Pass-through to {@link SegmentRing#findSegmentContaining(long)}. */ diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java index 441edd90..5c59823c 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java @@ -44,7 +44,6 @@ import io.questdb.client.std.MemoryTag; import io.questdb.client.std.QuietCloseable; import io.questdb.client.std.str.Utf8s; -import io.questdb.client.std.ObjList; import io.questdb.client.std.Unsafe; import org.jetbrains.annotations.TestOnly; import org.slf4j.Logger; @@ -195,7 +194,7 @@ public final class CursorWebSocketSendLoop implements QuietCloseable { * Throttle "reconnect attempt N failed" WARN logs to one per 5 s. */ private static final long RECONNECT_LOG_THROTTLE_NANOS = 5_000_000_000L; - // Test seam: when true, appendSymbolToMirror throws instead of appending, + // Test seam: when true, recovery mirror seeding throws before growing, // simulating the native realloc OOM (or MAX_SENT_DICT_BYTES ceiling) that // ensureSentDictCapacity can raise while the constructor seeds the recovery // mirror. Lets a test prove the constructor frees the already-malloc'd mirror on @@ -300,6 +299,13 @@ public final class CursorWebSocketSendLoop implements QuietCloseable { private int sentDictBytesCapacity; private int sentDictBytesLen; private int sentDictCount; + // Reusable staging frame for dictionary catch-up chunks. A reconnect may split a + // large dictionary into many chunks, and every later reconnect repeats that split; + // retaining one growable loop-owned buffer turns the old malloc/free-per-chunk path + // into amortized growth only. I/O-thread-owned, freed beside sentDictBytesAddr. + private long catchUpFrameAddr; + private int catchUpFrameCapacity; + private int catchUpFrameGrowthCount; // Orphan-policy cap-gap attempts -- catch-ups that reached a node and found an entry // too large for its batch cap -- with no intervening successful catch-up or unrelated // reconnect state (see MAX_CATCHUP_CAP_GAP_ATTEMPTS for the full budget accounting). @@ -750,18 +756,28 @@ public CursorWebSocketSendLoop(WebSocketClient client, CursorSendEngine engine, // catch-up-free. if (engine.recoveredMaxSymbolDeltaStart() > 0L) { // The prefix seed above may already have malloc'd the mirror, and - // appendSymbolToMirror -> ensureSentDictCapacity can still throw here (a - // native realloc OOM, or the MAX_SENT_DICT_BYTES ceiling). Such a throw + // ensureSentDictCapacity can still throw here (a native realloc OOM, or + // the MAX_SENT_DICT_BYTES ceiling). Such a throw // propagates out of the constructor, so the half-built loop is never // assigned to a reference -- neither ensureConnected's catch nor // BackgroundDrainer's finally can close() it -- and the mirror would leak. // Free it on any throw so the constructor leaves nothing behind, mirroring // the loopNeverRan free in close() / ioLoop's exit. try { - ObjList fromFrames = new ObjList<>(); - if (engine.collectReplaySymbolsAbove(sentDictCount, fromFrames) >= 0) { - for (int i = 0, n = fromFrames.size(); i < n; i++) { - appendSymbolToMirror(fromFrames.getQuick(i)); + int baseline = sentDictCount; + if (engine.recoveredSymbolCoverage(baseline) >= 0L) { + int suffixLen = engine.recoveredSymbolSuffixLen(baseline); + int suffixCount = engine.recoveredSymbolSuffixCount(baseline); + if (suffixLen > 0) { + if (forceMirrorSeedFailureForTest) { + throw new LineSenderException( + "simulated mirror seed allocation failure (test only)"); + } + ensureSentDictCapacity((long) sentDictBytesLen + suffixLen); + engine.copyRecoveredSymbolSuffix( + baseline, sentDictBytesAddr + sentDictBytesLen); + sentDictBytesLen += suffixLen; + sentDictCount += suffixCount; } } } catch (Throwable t) { @@ -1120,6 +1136,9 @@ public synchronized void close() { // buffer and drive setWireBaselineWithCatchUp into a null-mirror catch-up. sentDictCount = 0; } + if (loopNeverRan) { + freeCatchUpFrameBuffer(); + } } /** @@ -2030,6 +2049,7 @@ private void ioLoop() { sentDictBytesLen = 0; sentDictCount = 0; // keep the mirror all-or-nothing (see close()) } + freeCatchUpFrameBuffer(); shutdownLatch.countDown(); // Failed-stop hand-off (see delegateEngineClose): the owner could // not free the engine safely while this thread was alive, so the @@ -2613,7 +2633,8 @@ private void sendCatchUpChunk(int deltaStart, int deltaCount, long symbolsAddr, } int payloadLen = (int) payloadLenL; int frameLen = (int) frameLenL; - long frame = Unsafe.malloc(frameLen, MemoryTag.NATIVE_DEFAULT); + ensureCatchUpFrameCapacity(frameLen); + long frame = catchUpFrameAddr; try { Unsafe.getUnsafe().putByte(frame, (byte) 'Q'); Unsafe.getUnsafe().putByte(frame + 1, (byte) 'W'); @@ -2647,14 +2668,42 @@ private void sendCatchUpChunk(int deltaStart, int deltaCount, long symbolsAddr, throw (Error) t; } throw new CatchUpSendException(t); - } finally { - Unsafe.free(frame, frameLen, MemoryTag.NATIVE_DEFAULT); } nextWireSeq++; // this catch-up chunk consumed a wire sequence lastFrameOrPingNanos = System.nanoTime(); totalFramesSent.incrementAndGet(); } + @TestOnly + public int catchUpFrameGrowthCount() { + return catchUpFrameGrowthCount; + } + + private void ensureCatchUpFrameCapacity(int required) { + if (catchUpFrameCapacity >= required) { + return; + } + long newCapacity = Math.max(required, Math.max(4096L, (long) catchUpFrameCapacity * 2L)); + if (newCapacity > MAX_SENT_DICT_BYTES) { + newCapacity = MAX_SENT_DICT_BYTES; + } + catchUpFrameAddr = Unsafe.realloc( + catchUpFrameAddr, + catchUpFrameCapacity, + (int) newCapacity, + MemoryTag.NATIVE_DEFAULT); + catchUpFrameCapacity = (int) newCapacity; + catchUpFrameGrowthCount++; + } + + private void freeCatchUpFrameBuffer() { + if (catchUpFrameAddr != 0L) { + Unsafe.free(catchUpFrameAddr, catchUpFrameCapacity, MemoryTag.NATIVE_DEFAULT); + catchUpFrameAddr = 0L; + catchUpFrameCapacity = 0; + } + } + private boolean tryReceiveAcks() { boolean any = false; try { diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java index b2ee674b..8942cdfb 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java @@ -639,6 +639,20 @@ public long collectReplaySymbolsAbove( return coverage; } + /** + * Feeds every recovered frame in this segment to the engine's single-pass + * recovery fold. Segments are visited oldest-first by {@link SegmentRing}. + */ + void scanRecovery(RecoveredFrameAnalysis analysis) { + long off = HEADER_SIZE; + long frames = frameCount; + for (long i = 0; i < frames; i++) { + int payloadLen = Unsafe.getUnsafe().getInt(mmapAddress + off + 4); + analysis.accept(baseSeq + i, mmapAddress + off + FRAME_HEADER_SIZE, payloadLen); + off += FRAME_HEADER_SIZE + payloadLen; + } + } + /** * Highest {@code deltaStart + deltaCount} (one past the highest symbol id) any * delta-flagged frame in this segment carries, or {@code 0} only when NO diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java index 835d365a..982f93c2 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java @@ -27,6 +27,7 @@ import io.questdb.client.cutlass.qwp.client.GlobalSymbolDictionary; import io.questdb.client.cutlass.qwp.client.NativeBufferWriter; import io.questdb.client.std.Crc32c; +import io.questdb.client.std.Files; import io.questdb.client.std.FilesFacade; import io.questdb.client.std.MemoryTag; import io.questdb.client.std.ObjList; @@ -130,9 +131,9 @@ * owner (the engine) closes it, and {@code close()} is callable from any thread * (a shutdown hook, test cleanup). {@code close()} and the append methods are * therefore {@code synchronized}: without that, a close racing an in-flight append - * could free the scratch buffer or close the fd mid-write and let the write land - * on a descriptor the OS has reused for another file (silent cross-file - * corruption). Not thread-safe for concurrent writers. + * could unmap the production append region (or free the fault-test scratch buffer), + * close the fd, and let an in-flight write corrupt memory or land on a descriptor + * the OS has reused for another file. Not thread-safe for concurrent writers. */ public final class PersistedSymbolDict implements QuietCloseable { @@ -145,6 +146,7 @@ public final class PersistedSymbolDict implements QuietCloseable { static final int CRC_SIZE = 4; // u32 CRC-32C trailing every chunk static final int FILE_MAGIC = 0x31445953; // 'SYD1' little-endian static final int HEADER_SIZE = 8; + static final int INITIAL_APPEND_MAP_CAPACITY = 64 * 1024; /** * Upper bound on a chunk's two header varints ({@code entryCount} and * {@code entryBytes}): each is at most 5 bytes for a 32-bit value. The @@ -167,7 +169,15 @@ public final class PersistedSymbolDict implements QuietCloseable { // tests inject a fault facade to exercise recovery I/O failures (a truncate // that cannot drop a torn tail, a short write) without a real broken disk. private final FilesFacade ff; + // Production writes directly into segmented append mappings. Custom facades retain the + // positioned-write path so fault tests can inject short writes through ff.write. + private final boolean mappedAppend; + private long appendMapAddr; + private long appendMapCapacity; + private long appendMapOffset; + private int appendMapGrowthCount; private long appendOffset; + private long appendWriteCount; private boolean closed; // In-memory copy of the WIRE entry region [len][utf8]... -- chunk headers and // CRCs stripped -- populated only when open() recovered existing chunks @@ -186,6 +196,7 @@ public final class PersistedSymbolDict implements QuietCloseable { private PersistedSymbolDict(FilesFacade ff, int fd, long appendOffset, int size, long loadedEntriesAddr, int loadedEntriesLen) { this.ff = ff; this.fd = fd; + this.mappedAppend = ff == FilesFacade.INSTANCE; this.appendOffset = appendOffset; this.size = size; this.loadedEntriesAddr = loadedEntriesAddr; @@ -332,6 +343,16 @@ public synchronized void appendRawEntries(long addr, int len, int count) { // production. assert validateRawEntries(addr, len, count); int hdrLen = NativeBufferWriter.varintSize(count) + NativeBufferWriter.varintSize(len); + if (mappedAppend) { + long recLen = (long) hdrLen + len + CRC_SIZE; + ensureAppendMap(checkedRequiredOffset(recLen)); + long recStart = appendMapAddr + appendOffset - appendMapOffset; + long p = NativeBufferWriter.writeVarint(recStart, count); + NativeBufferWriter.writeVarint(p, len); + Unsafe.getUnsafe().copyMemory(addr, recStart + hdrLen, len); + commitMappedChunk(recStart, hdrLen, len, count); + return; + } ensureScratch((long) hdrLen + len + CRC_SIZE); long p = NativeBufferWriter.writeVarint(scratchAddr, count); NativeBufferWriter.writeVarint(p, len); @@ -357,6 +378,20 @@ public synchronized void appendSymbol(CharSequence symbol) { } int utf8Len = Utf8s.utf8Bytes(symbol); int wireLen = NativeBufferWriter.varintSize(utf8Len) + utf8Len; // [len][utf8] + if (mappedAppend) { + int hdrLen = NativeBufferWriter.varintSize(1) + NativeBufferWriter.varintSize(wireLen); + long recLen = (long) hdrLen + wireLen + CRC_SIZE; + ensureAppendMap(checkedRequiredOffset(recLen)); + long recStart = appendMapAddr + appendOffset - appendMapOffset; + long p = NativeBufferWriter.writeVarint(recStart, 1); + p = NativeBufferWriter.writeVarint(p, wireLen); + p = NativeBufferWriter.writeVarint(p, utf8Len); + if (utf8Len > 0) { + Utf8s.strCpyUtf8(symbol, p, utf8Len); + } + commitMappedChunk(recStart, hdrLen, wireLen, 1); + return; + } ensureScratch((long) MAX_CHUNK_HEADER_SIZE + wireLen + CRC_SIZE); long entryStart = scratchAddr + MAX_CHUNK_HEADER_SIZE; long p = NativeBufferWriter.writeVarint(entryStart, utf8Len); @@ -367,13 +402,12 @@ public synchronized void appendSymbol(CharSequence symbol) { } /** - * Appends the dense id range {@code [from .. to]} as ONE chunk, in a SINGLE - * write with a SINGLE checksum. Encodes the whole entry region into scratch - * first -- versus one {@code pwrite(2)} and one native CRC call per symbol. - * That per-symbol syscall and JNI count is the hot-path cost on a - * high-cardinality batch (one new symbol per row), which is exactly the - * store-and-forward workload delta encoding targets. Callers pass the dictionary - * and the range so the ids resolve to their symbol strings. + * Appends the dense id range {@code [from .. to]} as one chunk with one + * checksum. Production encodes directly into a segmented append mapping, avoiding + * both the staging copy and a positioned-write syscall on every flush. The + * scratch/positioned-write fallback exists only behind an injected filesystem + * facade so short-write recovery tests retain their fault seam. Callers pass the + * dictionary and the range so the ids resolve to their symbol strings. *

      * Same durability and idempotency contract as {@link #appendSymbol}: no fsync, * and a short write throws WITHOUT advancing {@code size}/{@code appendOffset}, @@ -385,6 +419,37 @@ public synchronized void appendSymbols(GlobalSymbolDictionary dict, int from, in return; } int count = to - from + 1; + if (mappedAppend) { + long entriesLenLong = 0L; + for (int id = from; id <= to; id++) { + int utf8Len = Utf8s.utf8Bytes(dict.getSymbol(id)); + entriesLenLong += NativeBufferWriter.varintSize(utf8Len) + (long) utf8Len; + if (entriesLenLong > MAX_SCRATCH_BYTES) { + throw new IllegalStateException("symbol dict chunk exceeds the maximum size to " + + FILE_NAME + " [required=" + entriesLenLong + ", max=" + + MAX_SCRATCH_BYTES + ']'); + } + } + int entriesLen = (int) entriesLenLong; + int hdrLen = NativeBufferWriter.varintSize(count) + + NativeBufferWriter.varintSize(entriesLen); + long recLen = (long) hdrLen + entriesLen + CRC_SIZE; + ensureAppendMap(checkedRequiredOffset(recLen)); + long recStart = appendMapAddr + appendOffset - appendMapOffset; + long p = NativeBufferWriter.writeVarint(recStart, count); + p = NativeBufferWriter.writeVarint(p, entriesLen); + for (int id = from; id <= to; id++) { + CharSequence symbol = dict.getSymbol(id); + int utf8Len = Utf8s.utf8Bytes(symbol); + p = NativeBufferWriter.writeVarint(p, utf8Len); + if (utf8Len > 0) { + Utf8s.strCpyUtf8(symbol, p, utf8Len); + p += utf8Len; + } + } + commitMappedChunk(recStart, hdrLen, entriesLen, count); + return; + } int entriesLen = 0; for (int id = from; id <= to; id++) { CharSequence symbol = dict.getSymbol(id); @@ -415,6 +480,20 @@ public synchronized void close() { loadedEntriesAddr = 0L; loadedEntriesLen = 0; } + if (appendMapAddr != 0L) { + Files.munmap(appendMapAddr, appendMapCapacity, MemoryTag.MMAP_DEFAULT); + appendMapAddr = 0L; + appendMapCapacity = 0L; + appendMapOffset = 0L; + // The active window reserves space past the logical end. Return that tail on + // orderly close; after a crash open() treats the zero-filled reserve as + // a torn trailing chunk and truncates it to the same appendOffset. + if (!ff.truncate(fd, appendOffset)) { + LOG.warn("symbol dict {} could not trim mmap reserve to {}; recovery will " + + "discard the zero-filled tail on the next open", + FILE_NAME, appendOffset); + } + } if (scratchAddr != 0L) { Unsafe.free(scratchAddr, scratchCap, MemoryTag.NATIVE_DEFAULT); scratchAddr = 0L; @@ -425,6 +504,16 @@ public synchronized void close() { } } + @TestOnly + public synchronized int appendMapGrowthCount() { + return appendMapGrowthCount; + } + + @TestOnly + public synchronized long appendWriteCount() { + return appendWriteCount; + } + /** * Base address of the loaded entry region -- the concatenated * {@code [len][utf8]} bytes of every recovered symbol in id order, exactly as a @@ -726,6 +815,85 @@ private static boolean validateRawEntries(long addr, int len, int count) { return true; } + private long checkedRequiredOffset(long recordLen) { + long required = appendOffset + recordLen; + if (recordLen < 0L || required < appendOffset) { + throw new IllegalStateException("symbol dict append offset overflow to " + + FILE_NAME + " [offset=" + appendOffset + ", recordLen=" + recordLen + ']'); + } + return required; + } + + /** + * Commits a chunk already assembled directly in the append mmap. The logical + * offset and symbol count advance last, after the CRC and a store fence, so an + * interrupted process leaves either a complete chunk or a tail that open() + * rejects and truncates. + */ + private void commitMappedChunk(long recStart, int hdrLen, int entriesLen, int count) { + long bodyLen = (long) hdrLen + entriesLen; + long recLen = bodyLen + CRC_SIZE; + Unsafe.getUnsafe().putInt( + recStart + bodyLen, + Crc32c.update(Crc32c.INIT, recStart, bodyLen)); + Unsafe.getUnsafe().storeFence(); + appendOffset += recLen; + size += count; + } + + /** + * Ensures the production append mmap covers the absolute file offset + * {@code required}. The log is mapped in 64 KiB-aligned windows: small flushes + * share a window, while a large existing dictionary does not force the process + * to reserve and remap its whole prefix (or geometrically over-allocate it). + */ + private void ensureAppendMap(long required) { + if (appendMapAddr != 0L + && required >= appendMapOffset + && required <= appendMapOffset + appendMapCapacity) { + return; + } + long newOffset = appendOffset - appendOffset % INITIAL_APPEND_MAP_CAPACITY; + long needed = required - newOffset; + long newCapacity = Math.max(INITIAL_APPEND_MAP_CAPACITY, needed); + long remainder = newCapacity % INITIAL_APPEND_MAP_CAPACITY; + if (remainder != 0L) { + long padding = INITIAL_APPEND_MAP_CAPACITY - remainder; + if (newCapacity > Long.MAX_VALUE - padding) { + throw new IllegalStateException("symbol dict mmap capacity overflow to " + + FILE_NAME + " [required=" + required + ']'); + } + newCapacity += padding; + } + if (newOffset > Long.MAX_VALUE - newCapacity) { + throw new IllegalStateException("symbol dict mmap file offset overflow to " + + FILE_NAME + " [offset=" + newOffset + ", capacity=" + newCapacity + ']'); + } + long newFileSize = newOffset + newCapacity; + if (!ff.allocate(fd, newFileSize)) { + throw new IllegalStateException("could not grow mmap append region for " + + FILE_NAME + " [required=" + required + ", fileSize=" + newFileSize + ']'); + } + if (appendMapAddr != 0L) { + long oldAddr = appendMapAddr; + long oldCapacity = appendMapCapacity; + appendMapAddr = 0L; + appendMapCapacity = 0L; + appendMapOffset = 0L; + Files.munmap(oldAddr, oldCapacity, MemoryTag.MMAP_DEFAULT); + } + long newAddr = Files.mmap( + fd, newCapacity, newOffset, Files.MAP_RW, MemoryTag.MMAP_DEFAULT); + if (newAddr == Files.FAILED_MMAP_ADDRESS) { + throw new IllegalStateException("could not mmap append region for " + + FILE_NAME + " [offset=" + newOffset + ", capacity=" + newCapacity + ']'); + } + appendMapAddr = newAddr; + appendMapCapacity = newCapacity; + appendMapOffset = newOffset; + appendMapGrowthCount++; + } + /** * Grows the append scratch buffer to hold at least {@code required} bytes. *

      @@ -769,6 +937,7 @@ private void flushChunk(long recStart, int hdrLen, int entriesLen, int count) { int bodyLen = hdrLen + entriesLen; int recLen = bodyLen + CRC_SIZE; Unsafe.getUnsafe().putInt(recStart + bodyLen, Crc32c.update(Crc32c.INIT, recStart, bodyLen)); + appendWriteCount++; long written = ff.write(fd, recStart, recLen, appendOffset); if (written != recLen) { throw new IllegalStateException("short write to " + FILE_NAME diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/RecoveredFrameAnalysis.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/RecoveredFrameAnalysis.java new file mode 100644 index 00000000..c9650cfd --- /dev/null +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/RecoveredFrameAnalysis.java @@ -0,0 +1,261 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.cutlass.qwp.client.sf.cursor; + +import io.questdb.client.cutlass.qwp.protocol.QwpConstants; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.ObjList; +import io.questdb.client.std.QuietCloseable; +import io.questdb.client.std.Unsafe; +import io.questdb.client.std.str.Utf8s; + +/** + * Engine-owned result of the one ordered frame walk performed during recovery. + * It folds commit-boundary detection, delta extrema, gap detection and the raw + * symbol suffix into the same pass. Running state is checkpointed only at a + * commit-bearing frame, so a deferred orphan tail can be scanned without ever + * leaking its metadata or symbols into the committed result. + */ +final class RecoveredFrameAnalysis implements QuietCloseable { + + private static final int MAX_RAW_BYTES = Integer.MAX_VALUE - 8; + private final int baseline; + private long committedBoundaryFsn = -1L; + private long committedCoverage; + private boolean committedGap; + private long committedMaxDeltaEnd; + private long committedMaxDeltaStart; + private int committedRawCount; + private int committedRawLen; + private long framesVisited; + private boolean runningGap; + private long runningMaxDeltaEnd; + private long runningMaxDeltaStart; + private long rawAddr; + private int rawCapacity; + private int runningRawCount; + private int runningRawLen; + private long runningCoverage; + + RecoveredFrameAnalysis(int baseline) { + this.baseline = baseline; + this.runningCoverage = baseline; + this.committedCoverage = baseline; + } + + void accept(long fsn, long payload, int payloadLen) { + framesVisited++; + boolean isQwp = payloadLen >= QwpConstants.HEADER_SIZE + && payloadLen > QwpConstants.HEADER_OFFSET_FLAGS + && Unsafe.getUnsafe().getInt(payload) == QwpConstants.MAGIC_MESSAGE; + byte flags = isQwp + ? Unsafe.getUnsafe().getByte(payload + QwpConstants.HEADER_OFFSET_FLAGS) + : 0; + if (isQwp && (flags & QwpConstants.FLAG_DELTA_SYMBOL_DICT) != 0) { + foldDelta(payload + QwpConstants.HEADER_SIZE, payload + payloadLen); + } + + // Only a positively identified deferred QWP frame can belong to an + // uncommitted tail. Short, foreign or otherwise non-QWP payloads remain + // retirement barriers, matching findLastFrameFsnWithoutPayloadFlag. + if (!isQwp || (flags & QwpConstants.FLAG_DEFER_COMMIT) == 0) { + committedBoundaryFsn = fsn; + committedCoverage = runningCoverage; + committedGap = runningGap; + committedMaxDeltaEnd = runningMaxDeltaEnd; + committedMaxDeltaStart = runningMaxDeltaStart; + committedRawLen = runningRawLen; + committedRawCount = runningRawCount; + } + } + + void appendDecodedSymbols(ObjList out) { + long p = rawAddr; + long limit = rawAddr + committedRawLen; + for (int i = 0; i < committedRawCount; i++) { + long encoded = readVarint(p, limit); + if (encoded < 0L) { + throw new IllegalStateException("malformed cached symbol dictionary suffix"); + } + int varintLen = (int) (encoded & 7L); + long symbolLen = encoded >>> 3; + p += varintLen; + if (symbolLen > limit - p) { + throw new IllegalStateException("truncated cached symbol dictionary suffix"); + } + out.add(Utf8s.stringFromUtf8Bytes(p, p + symbolLen)); + p += symbolLen; + } + if (p != limit) { + throw new IllegalStateException("overfilled cached symbol dictionary suffix"); + } + } + + int baseline() { + return baseline; + } + + long commitBoundaryFsn() { + return committedBoundaryFsn; + } + + long coverage() { + return committedGap ? -1L : committedCoverage; + } + + long framesVisited() { + return framesVisited; + } + + long maxDeltaEnd() { + return committedMaxDeltaEnd; + } + + long maxDeltaStart() { + return committedMaxDeltaStart; + } + + long rawAddr() { + return rawAddr; + } + + int rawCount() { + return committedRawCount; + } + + int rawLen() { + return committedRawLen; + } + + @Override + public void close() { + if (rawAddr != 0L) { + Unsafe.free(rawAddr, rawCapacity, MemoryTag.NATIVE_DEFAULT); + rawAddr = 0L; + rawCapacity = 0; + runningRawLen = 0; + committedRawLen = 0; + } + } + + private void appendRaw(long entryStart, int entryLen) { + long required = (long) runningRawLen + entryLen; + if (required > MAX_RAW_BYTES) { + throw new IllegalStateException("recovered symbol dictionary suffix exceeds maximum size " + + "[required=" + required + ", max=" + MAX_RAW_BYTES + ']'); + } + if (required > rawCapacity) { + long newCapacity = Math.max(required, Math.max(4_096L, (long) rawCapacity * 2L)); + if (newCapacity > MAX_RAW_BYTES) { + newCapacity = MAX_RAW_BYTES; + } + rawAddr = Unsafe.realloc(rawAddr, rawCapacity, (int) newCapacity, MemoryTag.NATIVE_DEFAULT); + rawCapacity = (int) newCapacity; + } + Unsafe.getUnsafe().copyMemory(entryStart, rawAddr + runningRawLen, entryLen); + runningRawLen += entryLen; + runningRawCount++; + } + + private void foldDelta(long p, long limit) { + long encodedStart = readVarint(p, limit); + if (encodedStart < 0L) { + runningGap = true; + return; + } + int startLen = (int) (encodedStart & 7L); + long deltaStart = encodedStart >>> 3; + p += startLen; + if (deltaStart > runningMaxDeltaStart) { + runningMaxDeltaStart = deltaStart; + } + + long encodedCount = readVarint(p, limit); + if (encodedCount < 0L) { + runningGap = true; + return; + } + int countLen = (int) (encodedCount & 7L); + long deltaCount = encodedCount >>> 3; + p += countLen; + long deltaEnd = deltaCount > Long.MAX_VALUE - deltaStart + ? Long.MAX_VALUE + : deltaStart + deltaCount; + if (deltaEnd > runningMaxDeltaEnd) { + runningMaxDeltaEnd = deltaEnd; + } + if (runningGap) { + return; + } + if (deltaStart > runningCoverage) { + runningGap = true; + return; + } + + long id = deltaStart; + for (long i = 0; i < deltaCount; i++, id++) { + long entryStart = p; + long encodedLen = readVarint(p, limit); + if (encodedLen < 0L) { + runningGap = true; + return; + } + int varintLen = (int) (encodedLen & 7L); + long symbolLen = encodedLen >>> 3; + p += varintLen; + if (symbolLen > limit - p) { + runningGap = true; + return; + } + p += symbolLen; + if (id >= runningCoverage) { + appendRaw(entryStart, (int) (p - entryStart)); + } + } + if (deltaEnd > runningCoverage) { + runningCoverage = deltaEnd; + } + } + + /** + * Returns {@code (value << 3) | encodedByteCount}, or {@code -1} for an + * unterminated/oversized 32-bit protocol varint. + */ + private static long readVarint(long p, long limit) { + long value = 0L; + int shift = 0; + int bytes = 0; + while (p < limit && bytes < 5) { + byte b = Unsafe.getUnsafe().getByte(p++); + value |= (long) (b & 0x7F) << shift; + bytes++; + if ((b & 0x80) == 0) { + return (value << 3) | bytes; + } + shift += 7; + } + return -1L; + } +} diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java index 06d1e64d..8e596a01 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java @@ -581,6 +581,24 @@ public synchronized long collectReplaySymbolsAbove( minFsnInclusive, maxFsnInclusive, coverage, out); } + /** + * Performs the one ordered recovery fold across sealed segments and the + * active segment. The returned native suffix remains owned by the caller. + */ + synchronized RecoveredFrameAnalysis analyzeRecovery(int symbolBaseline) { + RecoveredFrameAnalysis analysis = new RecoveredFrameAnalysis(symbolBaseline); + try { + for (int i = 0, n = sealedSegments.size(); i < n; i++) { + sealedSegments.get(i).scanRecovery(analysis); + } + active.scanRecovery(analysis); + return analysis; + } catch (Throwable t) { + analysis.close(); + throw t; + } + } + /** * Highest {@code deltaStart + deltaCount} any symbol-dict delta frame at or below * {@code maxFsnInclusive} references (0 when none). See diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java index 43bd9972..34ef3c4a 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java @@ -162,6 +162,38 @@ public void testSplitCatchUpFramesAcksDoNotAdvanceTrimWatermark() throws Excepti }); } + @Test + public void testSplitCatchUpReusesOneFrameBufferAcrossReconnects() throws Exception { + TestUtils.assertMemoryLeak(() -> { + CatchUpCapturingClient client = new CatchUpCapturingClient(3_100); + try (CursorSendEngine engine = newEngine()) { + CursorWebSocketSendLoop loop = newLoop(engine, client); + try { + seedMirror(loop, TestUtils.repeat("x", 3_000), TestUtils.repeat("y", 3_000)); + + invokeSetWireBaselineWithCatchUp(loop, 0L); + assertEquals("the small cap must split the dictionary", 2, client.framesSent); + assertEquals("the split chunks fit the initial native buffer", + 1, loop.catchUpFrameGrowthCount()); + + client.cap = 7_000; + invokeSetWireBaselineWithCatchUp(loop, 0L); + assertEquals("the larger cap must combine the dictionary", 3, client.framesSent); + assertEquals("the combined frame must grow the retained native buffer once", + 2, loop.catchUpFrameGrowthCount()); + + invokeSetWireBaselineWithCatchUp(loop, 0L); + assertEquals("the next reconnect sends one combined frame", 4, client.framesSent); + assertEquals("the grown native frame buffer must be reused across reconnects", + 2, loop.catchUpFrameGrowthCount()); + } finally { + // assertMemoryLeak verifies that close releases the retained buffer. + loop.close(); + } + } + }); + } + @Test public void testTransientCatchUpSendFailureIsRetriableNotTerminal() throws Exception { // A transient wire failure WHILE shipping the catch-up (the fresh diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopOrphanTailTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopOrphanTailTest.java index d19f0e12..52235637 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopOrphanTailTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopOrphanTailTest.java @@ -33,6 +33,7 @@ import io.questdb.client.network.PlainSocketFactory; import io.questdb.client.std.Files; import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.ObjList; import io.questdb.client.std.Unsafe; import io.questdb.client.test.tools.TestUtils; import org.junit.After; @@ -365,6 +366,38 @@ public void testZeroCountDeltaFrameAnchorsRecoveredMaxSymbolIdAtItsBaseline() th }); } + @Test + public void testRecoveryScansFramesOnceAndReusesCachedSymbolSuffix() throws Exception { + TestUtils.assertMemoryLeak(() -> { + try (CursorSendEngine engine = newEngine()) { + appendDeltaSymbolFrame(engine, 0, 'a'); + appendDeltaSymbolFrame(engine, 1, 'b'); + appendDeltaSymbolFrame(engine, 2, 'c'); + } + + try (CursorSendEngine engine = newEngine()) { + assertEquals("one recovery visit per frame", 3L, engine.recoveryFramesVisited()); + + ObjList first = new ObjList<>(); + ObjList second = new ObjList<>(); + assertEquals(3L, engine.collectReplaySymbolsAbove(0, first)); + assertEquals(3L, engine.collectReplaySymbolsAbove(0, second)); + assertEquals("a", first.getQuick(0)); + assertEquals("c", second.getQuick(2)); + assertEquals("producer seed reads must reuse the recovery result", + 3L, engine.recoveryFramesVisited()); + + CursorWebSocketSendLoop loop = newLoop(engine, new ArrayList<>()); + try { + assertEquals("send-loop construction must copy the cached native suffix", + 3L, engine.recoveryFramesVisited()); + } finally { + loop.close(); + } + } + }); + } + // --------------------------------------------------------------------- // harness // --------------------------------------------------------------------- @@ -456,6 +489,25 @@ private static void appendDeltaFrame(CursorSendEngine engine, boolean defer, int } } + private static void appendDeltaSymbolFrame(CursorSendEngine engine, int deltaStart, char symbol) { + int size = HEADER_SIZE + 4; // start, count=1, symbolLen=1, symbol byte + long buf = Unsafe.malloc(size, MemoryTag.NATIVE_DEFAULT); + try { + for (int i = 0; i < size; i++) { + Unsafe.getUnsafe().putByte(buf + i, (byte) 0); + } + Unsafe.getUnsafe().putInt(buf, MAGIC_MESSAGE); + Unsafe.getUnsafe().putByte(buf + HEADER_OFFSET_FLAGS, (byte) FLAG_DELTA_SYMBOL_DICT); + Unsafe.getUnsafe().putByte(buf + HEADER_SIZE, (byte) deltaStart); + Unsafe.getUnsafe().putByte(buf + HEADER_SIZE + 1, (byte) 1); + Unsafe.getUnsafe().putByte(buf + HEADER_SIZE + 2, (byte) 1); + Unsafe.getUnsafe().putByte(buf + HEADER_SIZE + 3, (byte) symbol); + engine.appendBlocking(buf, size); + } finally { + Unsafe.free(buf, size, MemoryTag.NATIVE_DEFAULT); + } + } + private static void awaitAckedFsn(CursorSendEngine engine, long target) throws InterruptedException { long deadline = System.nanoTime() + 10_000_000_000L; while (engine.ackedFsn() < target) { diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java index 6c6dcf91..280155a7 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java @@ -446,6 +446,40 @@ public void testBatchAppendWritesOneChunkWithOneChecksum() throws Exception { }); } + @Test + public void testMappedAppendAmortizesFlushesWithoutPositionedWrites() throws Exception { + assertMemoryLeak(() -> { + Path dir = Files.createTempDirectory("qwp-symdict"); + try { + PersistedSymbolDict d = PersistedSymbolDict.open(dir.toString()); + Assert.assertNotNull(d); + try { + for (int i = 0; i < 10_000; i++) { + d.appendSymbol("sym-" + i); + } + Assert.assertEquals(10_000, d.size()); + Assert.assertEquals("production appends must write directly into the mmap", + 0L, d.appendWriteCount()); + Assert.assertEquals("ten thousand flushes must require only three mapped windows", + 3, d.appendMapGrowthCount()); + } finally { + d.close(); + } + + PersistedSymbolDict re = PersistedSymbolDict.open(dir.toString()); + Assert.assertNotNull(re); + try { + Assert.assertEquals(10_000, re.size()); + Assert.assertEquals("sym-9999", re.readLoadedSymbols().getQuick(9_999)); + } finally { + re.close(); + } + } finally { + rmDir(dir); + } + }); + } + private static int varintSize(int v) { int n = 1; while ((v >>>= 7) != 0) { From f3282a49563c4ec479a07f16c6b34495638e146d Mon Sep 17 00:00:00 2001 From: glasstiger Date: Wed, 15 Jul 2026 22:30:57 +0100 Subject: [PATCH 71/96] Harden delta dictionary test correctness --- .../qwp/client/DeltaDictCatchUpTest.java | 51 ++++++++++++++++-- .../qwp/client/DeltaDictRecoveryTest.java | 54 ++++++++++++++++--- 2 files changed, 94 insertions(+), 11 deletions(-) diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictCatchUpTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictCatchUpTest.java index a7b2fb5f..21fdcd58 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictCatchUpTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictCatchUpTest.java @@ -97,6 +97,7 @@ public void testReconnectCatchUpRebuildsDictionary() throws Exception { Assert.assertEquals("2nd connection dictionary size", 2, conn2.size()); Assert.assertEquals("alpha", conn2.get(0)); Assert.assertEquals("beta", conn2.get(1)); + assertAckSequencesStartAtZero(handler.ackSequenceStarts()); } }); } @@ -143,6 +144,7 @@ public void testReconnectPreservesMonotonicDeltaBaseline() throws Exception { + "baseline (deltaStart >= 1); a reset baseline re-ships the whole " + "dictionary from deltaStart 0", handler.conn2SawDeltaAboveBaseline); + assertAckSequencesStartAtZero(handler.ackSequenceStarts()); } }); } @@ -204,6 +206,7 @@ public void testFixedCapNearBoundarySymbolCatchesUpWithoutTerminal() throws Exce Assert.assertEquals("2nd connection dictionary size", 2, conn2.size()); Assert.assertEquals(nearCapSymbol, conn2.get(0)); Assert.assertEquals("beta", conn2.get(1)); + assertAckSequencesStartAtZero(handler.ackSequenceStarts()); } }); } @@ -251,6 +254,7 @@ public void testForegroundCatchUpCapGapRetriesPastOrphanBudgetAndRecovers() thro Assert.assertTrue("foreground sender must recover and drain after the cap is restored", sender.awaitAckedFsn(targetFsn, 5_000)); } + assertAckSequencesStartAtZero(handler.ackSequenceStarts()); } }); } @@ -307,10 +311,19 @@ public void testReconnectCatchUpSplitsLargeDictionaryAcrossFrames() throws Excep Assert.assertTrue("catch-up split into multiple frames (saw " + handler.zeroTableFramesOnConn2 + ")", handler.zeroTableFramesOnConn2 >= 2); + assertAckSequencesStartAtZero(handler.ackSequenceStarts()); } }); } + private static void assertAckSequencesStartAtZero(List ackSequenceStarts) { + Assert.assertFalse("the fixture must ACK at least one connection", ackSequenceStarts.isEmpty()); + for (int i = 0; i < ackSequenceStarts.size(); i++) { + Assert.assertEquals("connection " + (i + 1) + " must begin ACK sequencing at zero", + 0L, ackSequenceStarts.get(i).longValue()); + } + } + private static String symbolName(int i) { // 10-char symbols so 40 of them clearly exceed the advertised 160-byte cap. return "symbol" + (1000 + i); @@ -371,10 +384,15 @@ private static class CatchUpHandler implements TestWebSocketServer.WebSocketServ // post-reconnect frame trips it. volatile boolean conn2SawDeltaAboveBaseline; volatile boolean sawZeroTableFrameOnConn2; + private final List ackSequenceStarts = new CopyOnWriteArrayList<>(); private final List> dictsByConn = new CopyOnWriteArrayList<>(); private TestWebSocketServer.ClientHandler currentClient; private final AtomicLong nextSeq = new AtomicLong(0); + List ackSequenceStarts() { + return new ArrayList<>(ackSequenceStarts); + } + synchronized List dictFor(int connNumber) { return connNumber <= dictsByConn.size() // Copy under the lock: the caller iterates it unlocked while the @@ -390,6 +408,7 @@ public synchronized void onBinaryMessage(TestWebSocketServer.ClientHandler clien currentClient = client; connectionsAccepted.incrementAndGet(); dictsByConn.add(new ArrayList<>()); // fresh dictionary per connection + nextSeq.set(0); } int connNumber = dictsByConn.size(); List dict = dictsByConn.get(connNumber - 1); @@ -414,7 +433,11 @@ public synchronized void onBinaryMessage(TestWebSocketServer.ClientHandler clien } } try { - client.sendBinary(buildAck(nextSeq.getAndIncrement())); + long ackSequence = nextSeq.getAndIncrement(); + if (newConnection) { + ackSequenceStarts.add(ackSequence); + } + client.sendBinary(buildAck(ackSequence)); // Drop the first connection right after ACKing its only frame, // forcing the sender to reconnect onto a fresh dictionary. if (connNumber == 1) { @@ -474,23 +497,33 @@ private static int tableCount(byte[] frame) { */ private static class CapShrinkHandler implements TestWebSocketServer.WebSocketServerHandler { final AtomicInteger connectionsAccepted = new AtomicInteger(); + private final List ackSequenceStarts = new CopyOnWriteArrayList<>(); private final AtomicLong nextSeq = new AtomicLong(0); private TestWebSocketServer.ClientHandler currentClient; private volatile TestWebSocketServer server; + List ackSequenceStarts() { + return new ArrayList<>(ackSequenceStarts); + } + void setServer(TestWebSocketServer server) { this.server = server; } @Override public synchronized void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] data) { - if (currentClient != client) { + boolean newConnection = currentClient != client; + if (newConnection) { currentClient = client; connectionsAccepted.incrementAndGet(); nextSeq.set(0); } try { - client.sendBinary(CatchUpHandler.buildAck(nextSeq.getAndIncrement())); + long ackSequence = nextSeq.getAndIncrement(); + if (newConnection) { + ackSequenceStarts.add(ackSequence); + } + client.sendBinary(CatchUpHandler.buildAck(ackSequence)); if (connectionsAccepted.get() == 1) { // Connection 1 registered the big symbol. Shrink the cap so the // reconnect's catch-up budget can't fit it, then drop to force @@ -518,6 +551,7 @@ private static class SplitCatchUpHandler implements TestWebSocketServer.WebSocke // Set once the server has closed connection 1 (see CatchUpHandler.conn1Closed). volatile boolean conn1Closed; volatile int zeroTableFramesOnConn2; + private final List ackSequenceStarts = new CopyOnWriteArrayList<>(); private final List> dictsByConn = new CopyOnWriteArrayList<>(); private final int dropConn1AtDictSize; private final AtomicLong nextSeq = new AtomicLong(0); @@ -528,6 +562,10 @@ private static class SplitCatchUpHandler implements TestWebSocketServer.WebSocke this.dropConn1AtDictSize = dropConn1AtDictSize; } + List ackSequenceStarts() { + return new ArrayList<>(ackSequenceStarts); + } + synchronized List dictFor(int connNumber) { return connNumber <= dictsByConn.size() // Copy under the lock: the caller iterates it unlocked while the @@ -543,6 +581,7 @@ public synchronized void onBinaryMessage(TestWebSocketServer.ClientHandler clien currentClient = client; connectionsAccepted.incrementAndGet(); dictsByConn.add(new ArrayList<>()); // fresh dictionary per connection + nextSeq.set(0); } int connNumber = dictsByConn.size(); List dict = dictsByConn.get(connNumber - 1); @@ -551,7 +590,11 @@ public synchronized void onBinaryMessage(TestWebSocketServer.ClientHandler clien zeroTableFramesOnConn2++; } try { - client.sendBinary(CatchUpHandler.buildAck(nextSeq.getAndIncrement())); + long ackSequence = nextSeq.getAndIncrement(); + if (newConnection) { + ackSequenceStarts.add(ackSequence); + } + client.sendBinary(CatchUpHandler.buildAck(ackSequence)); // Drop connection 1 only once it has learned the entire batch, so // the sender's sent-dictionary mirror is complete and the reconnect // catch-up must replay a dictionary larger than the batch cap. diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java index f9f0e968..b14b0708 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java @@ -789,8 +789,9 @@ public void testPersistFailureSurfacesAsLineSenderException() throws Exception { CursorSendEngine engine = new CursorSendEngine( slot, 4L * 1024 * 1024, CursorSendEngine.DEFAULT_APPEND_DEADLINE_NANOS, CursorSendEngine.DEFAULT_APPEND_DEADLINE_NANOS, ff); - try (Sender sender = QwpWebSocketSender.connect( - "localhost", port, null, 0, 0, 0L, null, false, engine)) { + Sender sender = QwpWebSocketSender.connect( + "localhost", port, null, 0, 0, 0L, null, false, engine); + try { ff.armed = true; // the next dictionary append short-writes sender.table("m").symbol("s", "boom").longColumn("v", 1L).atNow(); try { @@ -801,14 +802,42 @@ public void testPersistFailureSurfacesAsLineSenderException() throws Exception { + "not leak a raw IllegalStateException: " + expected.getMessage(), expected.getMessage().contains("failed to persist symbol dictionary before publish")); } - } catch (LineSenderException ignored) { - // close() re-flushes the still-buffered row; the facade has disarmed, so - // this normally succeeds. Either way it is not what we assert. + } finally { + try { + sender.close(); + } catch (LineSenderException ignored) { + // close() re-flushes the still-buffered row; the facade has disarmed, so + // this normally succeeds. Either way it is not what we assert. + } } } }); } + @Test + public void testReconstructingFixtureResetsAckSequencePerConnection() throws Exception { + assertMemoryLeak(() -> { + DictReconstructingHandler handler = new DictReconstructingHandler(); + try (TestWebSocketServer server = new TestWebSocketServer(handler)) { + int port = server.getPort(); + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + + for (int i = 0; i < 2; i++) { + try (Sender sender = Sender.fromConfig("ws::addr=localhost:" + port + ";")) { + sender.table("m").symbol("s", "sym-" + i).longColumn("v", i).atNow(); + long fsn = sender.flushAndGetSequence(); + Assert.assertTrue("connection " + (i + 1) + " must receive ACK zero", + sender.awaitAckedFsn(fsn, 5_000)); + } + } + + Assert.assertEquals("each fresh connection must restart wire ACK sequencing", + Arrays.asList(0L, 0L), handler.ackSequenceStarts()); + } + }); + } + @Test public void testFailedPublishDoesNotDuplicatePersistedSymbols() throws Exception { // Regression: persistNewSymbolsBeforePublish is a write-ahead -- it runs @@ -1385,10 +1414,15 @@ private static int readVarint(byte[] buf, int[] pos) { */ private static class DictReconstructingHandler implements TestWebSocketServer.WebSocketServerHandler { volatile boolean sawCatchUpFrame; + private final List ackSequenceStarts = new ArrayList<>(); private final List dict = new ArrayList<>(); private final AtomicLong nextSeq = new AtomicLong(0); private TestWebSocketServer.ClientHandler currentClient; + synchronized List ackSequenceStarts() { + return new ArrayList<>(ackSequenceStarts); + } + synchronized List dictSnapshot() { return new ArrayList<>(dict); } @@ -1399,16 +1433,22 @@ synchronized int maxDictSize() { @Override public synchronized void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] data) { - if (currentClient != client) { + boolean newConnection = currentClient != client; + if (newConnection) { currentClient = client; dict.clear(); // fresh server dictionary per connection + nextSeq.set(0); } accumulate(data); if (tableCount(data) == 0 && hasDelta(data)) { sawCatchUpFrame = true; } try { - client.sendBinary(buildAck(nextSeq.getAndIncrement())); + long ackSequence = nextSeq.getAndIncrement(); + if (newConnection) { + ackSequenceStarts.add(ackSequence); + } + client.sendBinary(buildAck(ackSequence)); } catch (IOException e) { throw new RuntimeException(e); } From 675d036733ed9d86b988c6569b275d76e44bbdba Mon Sep 17 00:00:00 2001 From: glasstiger Date: Wed, 15 Jul 2026 23:08:32 +0100 Subject: [PATCH 72/96] Avoid re-encoding split QWP frames --- .../qwp/client/QwpWebSocketEncoder.java | 108 +++++++++++++++++- .../qwp/client/QwpWebSocketSender.java | 96 ++++++++-------- .../qwp/client/QwpWebSocketEncoderTest.java | 96 +++++++++------- 3 files changed, 205 insertions(+), 95 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java index 8013330a..f38d9e93 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java @@ -26,6 +26,8 @@ import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; import io.questdb.client.std.QuietCloseable; +import io.questdb.client.std.Unsafe; +import io.questdb.client.std.Vect; import static io.questdb.client.cutlass.qwp.protocol.QwpConstants.*; @@ -45,8 +47,10 @@ public class QwpWebSocketEncoder implements QuietCloseable { // .symbol-dict instead of re-encoding the same symbols (see // QwpWebSocketSender.persistNewSymbolsBeforePublish). Valid until the next // beginMessage; stored as offsets so they survive a buffer realloc. + private int deltaCount; private int deltaEntriesEnd; private int deltaEntriesStart; + private int deltaStart; // QWP ingress always advertises Gorilla timestamp encoding. The column // writer still emits a per-column encoding byte and falls back to raw // values when delta-of-delta overflows int32. @@ -73,8 +77,8 @@ public void beginMessage( int batchMaxId ) { buffer.reset(); - int deltaStart = confirmedMaxId + 1; - int deltaCount = Math.max(0, batchMaxId - confirmedMaxId); + deltaStart = confirmedMaxId + 1; + deltaCount = Math.max(0, batchMaxId - confirmedMaxId); byte headerFlags = (byte) (flags | FLAG_DELTA_SYMBOL_DICT); byte origFlags = flags; flags = headerFlags; @@ -100,6 +104,64 @@ public void close() { } } + /** + * Copies one single-table split message from the combined message currently + * staged in this encoder. The table body is copied byte-for-byte from its + * recorded offset; columns and rows are not encoded again. + */ + public int copySplitMessage( + MicrobatchBuffer target, + int tableBodyOffset, + int tableBodyLength, + boolean deferCommit, + int confirmedMaxId, + int batchMaxId + ) { + if (target.getBufferPos() != 0) { + throw new IllegalStateException("split message target is not empty"); + } + if (tableBodyOffset < deltaEntriesEnd + || tableBodyLength < 0 + || (long) tableBodyOffset + tableBodyLength > buffer.getPosition()) { + throw new IllegalArgumentException("table body slice is outside the staged message"); + } + + int splitDeltaStart = confirmedMaxId + 1; + int splitDeltaCount = Math.max(0, batchMaxId - confirmedMaxId); + int deltaEntriesLength = splitDeltaEntriesLength(splitDeltaStart, splitDeltaCount); + int messageSize = splitMessageSize( + tableBodyLength, splitDeltaStart, splitDeltaCount, deltaEntriesLength); + target.ensureCapacity(messageSize); + + long source = buffer.getBufferPtr(); + long destination = target.getBufferPtr(); + Vect.memcpy(destination, source, HEADER_SIZE); + + byte splitFlags = Unsafe.getUnsafe().getByte(source + HEADER_OFFSET_FLAGS); + if (deferCommit) { + splitFlags |= FLAG_DEFER_COMMIT; + } else { + splitFlags &= ~FLAG_DEFER_COMMIT; + } + Unsafe.getUnsafe().putByte(destination + HEADER_OFFSET_FLAGS, splitFlags); + Unsafe.getUnsafe().putShort(destination + 6, (short) 1); + Unsafe.getUnsafe().putInt(destination + 8, messageSize - HEADER_SIZE); + + long writeAddress = destination + HEADER_SIZE; + writeAddress = NativeBufferWriter.writeVarint(writeAddress, splitDeltaStart); + writeAddress = NativeBufferWriter.writeVarint(writeAddress, splitDeltaCount); + if (deltaEntriesLength > 0) { + Vect.memcpy(writeAddress, source + deltaEntriesStart, deltaEntriesLength); + writeAddress += deltaEntriesLength; + } + Vect.memcpy(writeAddress, source + tableBodyOffset, tableBodyLength); + writeAddress += tableBodyLength; + assert writeAddress == destination + messageSize; + + target.setBufferPos(messageSize); + return messageSize; + } + public int encode(QwpTableBuffer tableBuffer) { buffer.reset(); writeHeader(1, 0); @@ -148,6 +210,17 @@ public int getDeltaEntriesStart() { return deltaEntriesStart; } + public int getSplitMessageSize(int tableBodyLength, int confirmedMaxId, int batchMaxId) { + if (tableBodyLength < 0) { + throw new IllegalArgumentException("tableBodyLength must be non-negative"); + } + int splitDeltaStart = confirmedMaxId + 1; + int splitDeltaCount = Math.max(0, batchMaxId - confirmedMaxId); + int deltaEntriesLength = splitDeltaEntriesLength(splitDeltaStart, splitDeltaCount); + return splitMessageSize( + tableBodyLength, splitDeltaStart, splitDeltaCount, deltaEntriesLength); + } + public void setDeferCommit(boolean defer) { if (defer) { flags |= FLAG_DEFER_COMMIT; @@ -170,4 +243,35 @@ public void writeHeader(int tableCount, int payloadLength) { buffer.putShort((short) tableCount); buffer.putInt(payloadLength); } + + private int splitDeltaEntriesLength(int splitDeltaStart, int splitDeltaCount) { + if (splitDeltaCount == 0) { + return 0; + } + if (splitDeltaStart != deltaStart || splitDeltaCount != deltaCount) { + throw new IllegalStateException("split delta does not match the staged message" + + " [stagedStart=" + deltaStart + + ", stagedCount=" + deltaCount + + ", splitStart=" + splitDeltaStart + + ", splitCount=" + splitDeltaCount + ']'); + } + return deltaEntriesEnd - deltaEntriesStart; + } + + private int splitMessageSize( + int tableBodyLength, + int splitDeltaStart, + int splitDeltaCount, + int deltaEntriesLength + ) { + long messageSize = (long) HEADER_SIZE + + NativeBufferWriter.varintSize(splitDeltaStart) + + NativeBufferWriter.varintSize(splitDeltaCount) + + deltaEntriesLength + + tableBodyLength; + if (messageSize > Integer.MAX_VALUE) { + throw new OutOfMemoryError("split QWP message size overflow: " + messageSize); + } + return (int) messageSize; + } } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index c22f01ec..696e168e 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -172,9 +172,9 @@ public class QwpWebSocketSender implements Sender { private final ReentrantLock connectWalkLock = new ReentrantLock(); private final QwpHostHealthTracker hostTracker; // Per-table encoded body byte counts captured during flushPendingRows' combined - // encode, reused by flushPendingRowsSplit to size each split frame arithmetically - // instead of re-encoding the batch a second time. Cleared and repopulated on every - // flush; only consumed on the (exceptional) split path. Reused to stay zero-GC. + // encode. flushPendingRowsSplit uses them both for preflight sizing and to walk + // the staged body slices without encoding the batch a second time. Cleared and + // repopulated on every flush; reused to stay zero-GC. private final IntList splitFrameBodyBytes = new IntList(); private final CharSequenceObjHashMap tableBuffers; // null means plain text (no TLS) @@ -3497,15 +3497,13 @@ private void flushPendingRows(boolean deferCommit) { encoder.setDeferCommit(deferCommit); encoder.beginMessage(tableCount, globalSymbolDictionary, symbolDeltaBaseline(), currentBatchMaxSymbolId); - // Record each table's encoded body size (position delta across addTable) so - // the split path can size its per-table frames arithmetically rather than - // re-encoding the whole batch. Body encoding is context-free (a table's bytes - // don't depend on its siblings or the delta section), so the size a table - // takes here equals the size it takes in its own split frame. Only consumed - // when the batch overflows the cap; the capture is a couple of int ops per - // table on the common path. + // Record each table's encoded body size (position delta across addTable). + // When the batch needs splitting, these lengths delimit immutable body + // slices in the combined encoder buffer for direct frame assembly. The + // capture is a couple of int ops per table on the common path. splitFrameBodyBytes.clear(); - int bodyStart = encoder.getBuffer().getPosition(); + int combinedBodyStart = encoder.getBuffer().getPosition(); + int bodyStart = combinedBodyStart; for (int i = 0, n = keys.size(); i < n; i++) { CharSequence tableName = keys.getQuick(i); if (tableName == null) { @@ -3538,10 +3536,9 @@ private void flushPendingRows(boolean deferCommit) { // the next flush picks up the new cap. int cap = serverMaxBatchSize; if (cap > 0 && messageSize > cap) { - // The combined frame's delta-entry bytes are byte-identical to the first - // split frame's (same baseline + batch max), so capture the length now - // for the arithmetic frame-sizing in flushPendingRowsSplit. - flushPendingRowsSplit(keys, deferCommit, encoder.getDeltaEntriesLen(), cap); + // Keep the completed combined frame staged in the encoder while the + // split path copies its delta entries and table-body slices. + flushPendingRowsSplit(keys, deferCommit, combinedBodyStart, cap); return; } @@ -3593,7 +3590,12 @@ private void flushPendingRows(boolean deferCommit) { * carry FLAG_DEFER_COMMIT. When false, only the * last message omits the flag. */ - private void flushPendingRowsSplit(ObjList keys, boolean deferCommit, int combinedDeltaEntriesLen, int cap) { + private void flushPendingRowsSplit( + ObjList keys, + boolean deferCommit, + int combinedBodyStart, + int cap + ) { if (LOG.isDebugEnabled()) { LOG.debug("Splitting flush across multiple messages [serverMaxBatchSize={}, defer={}]", cap, deferCommit); } @@ -3609,14 +3611,10 @@ private void flushPendingRowsSplit(ObjList keys, boolean deferComm // front makes the split all-or-nothing: either every frame fits and all // publish, or none publish and we throw with nothing stranded. // - // Each split frame's size is derived ARITHMETICALLY from the combined encode - // flushPendingRows already performed -- header + the two delta-section varints - // + the delta entries (byte-identical to the combined frame's when this frame - // carries them) + the table's own body bytes (captured in splitFrameBodyBytes, - // context-free so identical here and in its solo frame) -- rather than - // re-encoding every table a second time. simBaseline mirrors the publish loop's - // baseline advance (advanceSentMaxSymbolId), so each size equals the frame the - // publish loop will build; this pass mutates no delta/persist state. + // Each split frame's size is derived from the combined encode flushPendingRows + // already performed. simBaseline mirrors the publish loop's baseline advance + // (advanceSentMaxSymbolId), so each size equals the frame the staged-slice + // assembler will build; this pass mutates no delta/persist state. int nonEmptyCount = 0; int simBaseline = symbolDeltaBaseline(); int bodyIdx = 0; @@ -3630,13 +3628,8 @@ private void flushPendingRowsSplit(ObjList keys, boolean deferComm continue; } nonEmptyCount++; - int deltaStart = simBaseline + 1; - int deltaCount = Math.max(0, currentBatchMaxSymbolId - simBaseline); - int messageSize = QwpConstants.HEADER_SIZE - + NativeBufferWriter.varintSize(deltaStart) - + NativeBufferWriter.varintSize(deltaCount) - + (deltaCount > 0 ? combinedDeltaEntriesLen : 0) - + splitFrameBodyBytes.getQuick(bodyIdx); + int messageSize = encoder.getSplitMessageSize( + splitFrameBodyBytes.getQuick(bodyIdx), simBaseline, currentBatchMaxSymbolId); bodyIdx++; if (messageSize > cap) { resetTableBuffersAfterFlush(keys); @@ -3653,6 +3646,8 @@ private void flushPendingRowsSplit(ObjList keys, boolean deferComm } int sent = 0; + bodyIdx = 0; + int tableBodyOffset = combinedBodyStart; for (int i = 0, n = keys.size(); i < n; i++) { CharSequence tableName = keys.getQuick(i); if (tableName == null) { @@ -3667,35 +3662,38 @@ private void flushPendingRowsSplit(ObjList keys, boolean deferComm boolean isLast = (sent == nonEmptyCount); boolean deferThis = deferCommit || !isLast; - encoder.setDeferCommit(deferThis); - // Each split frame emits the delta above sentMaxSymbolId; the first - // frame ships the whole batch's new ids and advances the baseline, so - // the remaining frames carry an empty delta and just reference ids the - // first frame already registered. - encoder.beginMessage(1, globalSymbolDictionary, - symbolDeltaBaseline(), currentBatchMaxSymbolId); - encoder.addTable(tableBuffer); - int messageSize = encoder.finishMessage(); - QwpBufferWriter buffer = encoder.getBuffer(); + int tableBodyLength = splitFrameBodyBytes.getQuick(bodyIdx++); + // Persist before touching activeBuffer. If the write-ahead fails, the + // caller can retry with both the source rows and active microbatch + // unchanged. The first frame carries the batch's new symbols; later + // frames are no-ops once the baseline has advanced. + persistNewSymbolsBeforePublish(); + ensureActiveBufferReady(); + // The combined encoder buffer remains immutable for the whole split. + // Assemble this frame directly into the active microbatch: patched + // header + staged delta bytes + the staged table-body slice. No row or + // column is encoded a second time. + int messageSize = encoder.copySplitMessage( + activeBuffer, + tableBodyOffset, + tableBodyLength, + deferThis, + symbolDeltaBaseline(), + currentBatchMaxSymbolId + ); + tableBodyOffset += tableBodyLength; // The pre-flight pass above already verified every split frame fits the // cap, so none can be found oversized here -- which is what keeps this // loop from publishing (and stranding) a deferred prefix before an // oversized table. Both passes size against the SAME snapshot cap, so a // mid-flush failover cannot make them disagree; the assert therefore only // catches a genuine divergence between the pre-flight arithmetic and the - // real encode (a future bug), not a cap race. It deliberately does NOT + // staged assembler (a future bug), not a cap race. It deliberately does NOT // reset+throw here, because by this point a prefix may already be on the ring. assert messageSize <= cap : "split frame exceeded serverMaxBatchSize after pre-flight [table=" + tableName + ", messageSize=" + messageSize + ", serverMaxBatchSize=" + cap + ']'; - // Write-ahead persist before publish (see flushPendingRows). The - // first split frame carries the batch's new symbols; the rest are - // no-ops once the baseline has advanced past them. - persistNewSymbolsBeforePublish(); - ensureActiveBufferReady(); - activeBuffer.ensureCapacity(messageSize); - activeBuffer.write(buffer.getBufferPtr(), messageSize); activeBuffer.incrementRowCount(); sealAndSwapBuffer(); // Frame queued: advance so the next split frame's delta starts above diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketEncoderTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketEncoderTest.java index b21d2e25..5acc0cf9 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketEncoderTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketEncoderTest.java @@ -25,6 +25,7 @@ package io.questdb.client.test.cutlass.qwp.client; import io.questdb.client.cutlass.qwp.client.GlobalSymbolDictionary; +import io.questdb.client.cutlass.qwp.client.MicrobatchBuffer; import io.questdb.client.cutlass.qwp.client.QwpBufferWriter; import io.questdb.client.cutlass.qwp.client.QwpWebSocketEncoder; import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; @@ -1285,30 +1286,16 @@ public void testReset() throws Exception { } @Test - public void testTableBodyEncodingIsContextFree() throws Exception { - // The split-flush path (QwpWebSocketSender.flushPendingRowsSplit) sizes each - // per-table frame ARITHMETICALLY from splitFrameBodyBytes -- the body byte - // count captured during the COMBINED encode in flushPendingRows -- instead of - // re-encoding to measure. That is sound only while a table's body bytes are - // context-free: identical whether the table is encoded solo or as the k-th - // table after other tables and the delta section. Today every column encoder - // is stateless and symbol cells carry absolute global ids, so the property - // holds; this pins it so a future column encoder that ever carried - // cross-table state (which would break the arithmetic sizing and could strand - // a deferred prefix mid-split) fails loudly here rather than silently in - // production. + public void testSplitMessageCopiesStagedTableBodies() throws Exception { assertMemoryLeak(() -> { try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + MicrobatchBuffer target = new MicrobatchBuffer(64); QwpTableBuffer t0 = new QwpTableBuffer("alpha"); - QwpTableBuffer t1 = new QwpTableBuffer("bravo"); - QwpTableBuffer t2 = new QwpTableBuffer("charlie")) { + QwpTableBuffer t1 = new QwpTableBuffer("bravo")) { GlobalSymbolDictionary dict = new GlobalSymbolDictionary(); int aapl = dict.getOrAddSymbol("AAPL"); // 0 int goog = dict.getOrAddSymbol("GOOG"); // 1 - int msft = dict.getOrAddSymbol("MSFT"); // 2 - // Distinct schemas + a symbol column (absolute global ids) so a - // positional / cross-table encoder bug would shift the body bytes. t0.getOrCreateColumn("sym", TYPE_SYMBOL, false).addSymbolWithGlobalId("AAPL", aapl); t0.getOrCreateColumn("v", TYPE_LONG, false).addLong(1L); t0.nextRow(); @@ -1318,35 +1305,56 @@ public void testTableBodyEncodingIsContextFree() throws Exception { t1.getOrCreateColumn("s", TYPE_VARCHAR, true).addString("hello"); t1.nextRow(); - t2.getOrCreateColumn("sym", TYPE_SYMBOL, false).addSymbolWithGlobalId("MSFT", msft); - t2.getOrCreateDesignatedTimestampColumn(TYPE_TIMESTAMP).addLong(1_000_000L); - t2.nextRow(); - - QwpTableBuffer[] tables = {t0, t1, t2}; - int confirmedMaxId = -1; - int batchMaxId = 2; - - // Combined encode: capture each table's body bytes exactly as - // flushPendingRows' splitFrameBodyBytes does (position delta per addTable). - int[] combinedBody = new int[tables.length]; - encoder.beginMessage(tables.length, dict, confirmedMaxId, batchMaxId); - int bodyStart = encoder.getBuffer().getPosition(); - for (int i = 0; i < tables.length; i++) { - encoder.addTable(tables[i]); - int bodyEnd = encoder.getBuffer().getPosition(); - combinedBody[i] = bodyEnd - bodyStart; - bodyStart = bodyEnd; + encoder.beginMessage(2, dict, -1, 1); + int t0BodyOffset = encoder.getBuffer().getPosition(); + encoder.addTable(t0); + int t0BodyLength = encoder.getBuffer().getPosition() - t0BodyOffset; + int t1BodyOffset = encoder.getBuffer().getPosition(); + encoder.addTable(t1); + int t1BodyLength = encoder.getBuffer().getPosition() - t1BodyOffset; + encoder.finishMessage(); + long staged = encoder.getBuffer().getBufferPtr(); + + // Erase the source tables after their one combined encode. Split + // assembly must still reproduce both bodies from staged bytes. + t0.clear(); + t1.clear(); + + int firstSize = encoder.copySplitMessage(target, t0BodyOffset, t0BodyLength, + true, -1, 1); + long first = target.getBufferPtr(); + Assert.assertEquals(firstSize, target.getBufferPos()); + Assert.assertEquals(1, Unsafe.getUnsafe().getShort(first + 6)); + Assert.assertEquals(firstSize - HEADER_SIZE, Unsafe.getUnsafe().getInt(first + 8)); + Assert.assertEquals(FLAG_DEFER_COMMIT, + (byte) (Unsafe.getUnsafe().getByte(first + HEADER_OFFSET_FLAGS) & FLAG_DEFER_COMMIT)); + Cursor cursor = new Cursor(first + HEADER_SIZE); + Assert.assertEquals(0, cursor.readVarint()); + Assert.assertEquals(2, cursor.readVarint()); + Assert.assertEquals("AAPL", cursor.readString()); + Assert.assertEquals("GOOG", cursor.readString()); + for (int i = 0; i < t0BodyLength; i++) { + Assert.assertEquals("first staged table body byte " + i, + Unsafe.getUnsafe().getByte(staged + t0BodyOffset + i), + Unsafe.getUnsafe().getByte(cursor.address + i)); } - // Solo encode each table under the SAME baseline/batch max, as the - // split publish loop does, and assert the body bytes match the capture. - for (int i = 0; i < tables.length; i++) { - encoder.beginMessage(1, dict, confirmedMaxId, batchMaxId); - int soloBodyStart = encoder.getBuffer().getPosition(); - encoder.addTable(tables[i]); - int soloBody = encoder.getBuffer().getPosition() - soloBodyStart; - Assert.assertEquals("table " + i + " body must encode identically solo vs combined " - + "(splitFrameBodyBytes relies on a context-free body)", combinedBody[i], soloBody); + target.reset(); + int secondSize = encoder.copySplitMessage(target, t1BodyOffset, t1BodyLength, + false, 1, 1); + long second = target.getBufferPtr(); + Assert.assertEquals(secondSize, target.getBufferPos()); + Assert.assertEquals(1, Unsafe.getUnsafe().getShort(second + 6)); + Assert.assertEquals(secondSize - HEADER_SIZE, Unsafe.getUnsafe().getInt(second + 8)); + Assert.assertEquals(0, + Unsafe.getUnsafe().getByte(second + HEADER_OFFSET_FLAGS) & FLAG_DEFER_COMMIT); + cursor = new Cursor(second + HEADER_SIZE); + Assert.assertEquals(2, cursor.readVarint()); + Assert.assertEquals(0, cursor.readVarint()); + for (int i = 0; i < t1BodyLength; i++) { + Assert.assertEquals("second staged table body byte " + i, + Unsafe.getUnsafe().getByte(staged + t1BodyOffset + i), + Unsafe.getUnsafe().getByte(cursor.address + i)); } } }); From f12c934e8c58134328cc357db5ec260c279d65aa Mon Sep 17 00:00:00 2001 From: glasstiger Date: Wed, 15 Jul 2026 23:27:52 +0100 Subject: [PATCH 73/96] Clean up QWP delta dictionary tests --- .../qwp/client/DeltaDictCatchUpTest.java | 67 +- .../qwp/client/DeltaDictRecoveryTest.java | 113 +- .../cutlass/qwp/client/QwpWireTestUtils.java | 95 ++ .../sf/cursor/PersistedSymbolDictTest.java | 1004 +++++++---------- 4 files changed, 540 insertions(+), 739 deletions(-) create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWireTestUtils.java diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictCatchUpTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictCatchUpTest.java index 21fdcd58..400ea1d3 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictCatchUpTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictCatchUpTest.java @@ -31,9 +31,6 @@ import org.junit.Test; import java.io.IOException; -import java.nio.ByteBuffer; -import java.nio.ByteOrder; -import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.List; import java.util.concurrent.CopyOnWriteArrayList; @@ -329,19 +326,6 @@ private static String symbolName(int i) { return "symbol" + (1000 + i); } - private static int readVarint(byte[] buf, int[] pos) { - int result = 0; - int shift = 0; - while (pos[0] < buf.length) { - int b = buf[pos[0]++] & 0xFF; - result |= (b & 0x7F) << shift; - if ((b & 0x80) == 0) return result; - shift += 7; - if (shift > 28) throw new IllegalStateException("varint too long"); - } - throw new IllegalStateException("varint truncated"); - } - private static void waitFor(BoolCondition cond, long timeoutMillis) { long deadline = System.currentTimeMillis() + timeoutMillis; while (System.currentTimeMillis() < deadline) { @@ -412,9 +396,9 @@ public synchronized void onBinaryMessage(TestWebSocketServer.ClientHandler clien } int connNumber = dictsByConn.size(); List dict = dictsByConn.get(connNumber - 1); - accumulate(data, dict); + QwpWireTestUtils.accumulateDeltaDictionary(data, dict); if (connNumber == 2) { - if (tableCount(data) == 0) { + if (QwpWireTestUtils.tableCount(data) == 0) { sawZeroTableFrameOnConn2 = true; // FLAG_DEFER_COMMIT is bit 0x01 of the flags byte (offset 5). catchUpDeferredOnConn2 = (data[5] & 0x01) != 0; @@ -427,7 +411,7 @@ public synchronized void onBinaryMessage(TestWebSocketServer.ClientHandler clien // frame arriving ahead of the new one -- that replay carries its // original deltaStart 0 and does not trip the flag. int[] pos = {12}; - if (readVarint(data, pos) >= 1) { + if (QwpWireTestUtils.readVarint(data, pos) >= 1) { conn2SawDeltaAboveBaseline = true; } } @@ -437,7 +421,7 @@ public synchronized void onBinaryMessage(TestWebSocketServer.ClientHandler clien if (newConnection) { ackSequenceStarts.add(ackSequence); } - client.sendBinary(buildAck(ackSequence)); + client.sendBinary(QwpWireTestUtils.buildAck(ackSequence)); // Drop the first connection right after ACKing its only frame, // forcing the sender to reconnect onto a fresh dictionary. if (connNumber == 1) { @@ -451,41 +435,6 @@ public synchronized void onBinaryMessage(TestWebSocketServer.ClientHandler clien } } - private static void accumulate(byte[] frame, List dict) { - final byte FLAG_DELTA_SYMBOL_DICT = 0x08; - if (frame.length < 12 || (frame[5] & FLAG_DELTA_SYMBOL_DICT) == 0) { - return; - } - int[] pos = {12}; // just past the 12-byte QWP header - int deltaStart = readVarint(frame, pos); - int deltaCount = readVarint(frame, pos); - while (dict.size() < deltaStart) { - dict.add(null); // null-pad, mirroring the server - } - for (int i = 0; i < deltaCount; i++) { - int len = readVarint(frame, pos); - String symbol = new String(frame, pos[0], len, StandardCharsets.UTF_8); - pos[0] += len; - int idx = deltaStart + i; - while (dict.size() <= idx) { - dict.add(null); - } - dict.set(idx, symbol); - } - } - - private static byte[] buildAck(long seq) { - byte[] buf = new byte[1 + 8 + 2]; - ByteBuffer bb = ByteBuffer.wrap(buf).order(ByteOrder.LITTLE_ENDIAN); - bb.put((byte) 0x00); - bb.putLong(seq); - bb.putShort((short) 0); - return buf; - } - - private static int tableCount(byte[] frame) { - return (frame[6] & 0xFF) | ((frame[7] & 0xFF) << 8); - } } /** @@ -523,7 +472,7 @@ public synchronized void onBinaryMessage(TestWebSocketServer.ClientHandler clien if (newConnection) { ackSequenceStarts.add(ackSequence); } - client.sendBinary(CatchUpHandler.buildAck(ackSequence)); + client.sendBinary(QwpWireTestUtils.buildAck(ackSequence)); if (connectionsAccepted.get() == 1) { // Connection 1 registered the big symbol. Shrink the cap so the // reconnect's catch-up budget can't fit it, then drop to force @@ -585,8 +534,8 @@ public synchronized void onBinaryMessage(TestWebSocketServer.ClientHandler clien } int connNumber = dictsByConn.size(); List dict = dictsByConn.get(connNumber - 1); - CatchUpHandler.accumulate(data, dict); - if (connNumber == 2 && CatchUpHandler.tableCount(data) == 0) { + QwpWireTestUtils.accumulateDeltaDictionary(data, dict); + if (connNumber == 2 && QwpWireTestUtils.tableCount(data) == 0) { zeroTableFramesOnConn2++; } try { @@ -594,7 +543,7 @@ public synchronized void onBinaryMessage(TestWebSocketServer.ClientHandler clien if (newConnection) { ackSequenceStarts.add(ackSequence); } - client.sendBinary(CatchUpHandler.buildAck(ackSequence)); + client.sendBinary(QwpWireTestUtils.buildAck(ackSequence)); // Drop connection 1 only once it has learned the entire batch, so // the sender's sent-dictionary mirror is complete and the reconnect // catch-up must replay a dictionary larger than the batch cap. diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java index b14b0708..885a3699 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java @@ -41,10 +41,8 @@ import org.junit.Test; import java.io.IOException; -import java.lang.reflect.Field; import java.nio.ByteBuffer; import java.nio.ByteOrder; -import java.nio.charset.StandardCharsets; import java.nio.file.Paths; import java.util.ArrayList; import java.util.Arrays; @@ -1099,7 +1097,8 @@ public void testRecoverySeedKeepsUtf8CollidingSymbolsInLockstep() throws Excepti } Assert.assertEquals("two colliding symbols must persist as two entries", 2, persistedSize); - // Phase 2: recover. The seeded producer id space must match pd.size(). + // Phase 2: recover, then introduce a third symbol. The wire must extend + // the two recovered ids at id 2 rather than overwrite id 1. DictReconstructingHandler handler = new DictReconstructingHandler(); try (TestWebSocketServer good = new TestWebSocketServer(handler)) { int port = good.getPort(); @@ -1107,12 +1106,16 @@ public void testRecoverySeedKeepsUtf8CollidingSymbolsInLockstep() throws Excepti Assert.assertTrue(good.awaitStart(5, TimeUnit.SECONDS)); String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";"; try (Sender s2 = Sender.fromConfig(cfg)) { - Assert.assertEquals("recovered producer dictionary must match the persisted " - + "entry count (not de-duped), else the delta baseline desyncs from the mirror", - persistedSize, globalDictSize(s2)); - Assert.assertEquals("delta baseline must resume at the persisted tip", - persistedSize - 1, intField(s2, "sentMaxSymbolId")); + s2.table("m").symbol("s", "tail").longColumn("v", 3L).atNow(); + long fsn = s2.flushAndGetSequence(); + Assert.assertTrue("the post-recovery frame must be acknowledged", + s2.awaitAckedFsn(fsn, 5_000)); } + Assert.assertEquals("the new symbol delta must continue after both recovered entries", + persistedSize, handler.lastDataDeltaStart()); + Assert.assertEquals("the reconstructed dictionary must retain both colliding entries " + + "and append the new symbol at id 2", + Arrays.asList("?", "?", "tail"), handler.dictSnapshot()); } }); } @@ -1369,19 +1372,6 @@ public void testFullDictFramesRecoverInFullDictModeInsteadOfBricking() throws Ex }); } - private static int globalDictSize(Sender sender) throws Exception { - Field f = sender.getClass().getDeclaredField("globalSymbolDictionary"); - f.setAccessible(true); - Object dict = f.get(sender); - return (int) dict.getClass().getMethod("size").invoke(dict); - } - - private static int intField(Sender sender, String name) throws Exception { - Field f = sender.getClass().getDeclaredField(name); - f.setAccessible(true); - return f.getInt(sender); - } - private static void writeAckWatermark(java.nio.file.Path path, long fsn) throws IOException { byte[] buf = new byte[16]; ByteBuffer bb = ByteBuffer.wrap(buf).order(ByteOrder.LITTLE_ENDIAN); @@ -1391,23 +1381,6 @@ private static void writeAckWatermark(java.nio.file.Path path, long fsn) throws java.nio.file.Files.write(path, buf); } - private static int readVarint(byte[] buf, int[] pos) { - int result = 0; - int shift = 0; - while (pos[0] < buf.length) { - int b = buf[pos[0]++] & 0xFF; - result |= (b & 0x7F) << shift; - if ((b & 0x80) == 0) { - return result; - } - shift += 7; - if (shift > 28) { - throw new IllegalStateException("varint too long"); - } - } - throw new IllegalStateException("varint truncated"); - } - /** * Reconstructs the per-connection symbol dictionary from delta sections, * mirroring the server's {@code setQuick(deltaStart + i)} + null-padding. @@ -1418,6 +1391,7 @@ private static class DictReconstructingHandler implements TestWebSocketServer.We private final List dict = new ArrayList<>(); private final AtomicLong nextSeq = new AtomicLong(0); private TestWebSocketServer.ClientHandler currentClient; + private volatile int lastDataDeltaStart = -1; synchronized List ackSequenceStarts() { return new ArrayList<>(ackSequenceStarts); @@ -1431,6 +1405,10 @@ synchronized int maxDictSize() { return dict.size(); } + int lastDataDeltaStart() { + return lastDataDeltaStart; + } + @Override public synchronized void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] data) { boolean newConnection = currentClient != client; @@ -1439,59 +1417,23 @@ public synchronized void onBinaryMessage(TestWebSocketServer.ClientHandler clien dict.clear(); // fresh server dictionary per connection nextSeq.set(0); } - accumulate(data); - if (tableCount(data) == 0 && hasDelta(data)) { + QwpWireTestUtils.accumulateDeltaDictionary(data, dict); + int tableCount = QwpWireTestUtils.tableCount(data); + if (tableCount == 0 && QwpWireTestUtils.hasDelta(data)) { sawCatchUpFrame = true; + } else if (tableCount > 0 && QwpWireTestUtils.hasDelta(data)) { + lastDataDeltaStart = QwpWireTestUtils.readVarint(data, new int[]{12}); } try { long ackSequence = nextSeq.getAndIncrement(); if (newConnection) { ackSequenceStarts.add(ackSequence); } - client.sendBinary(buildAck(ackSequence)); + client.sendBinary(QwpWireTestUtils.buildAck(ackSequence)); } catch (IOException e) { throw new RuntimeException(e); } } - - private static byte[] buildAck(long seq) { - byte[] buf = new byte[1 + 8 + 2]; - ByteBuffer bb = ByteBuffer.wrap(buf).order(ByteOrder.LITTLE_ENDIAN); - bb.put((byte) 0x00); - bb.putLong(seq); - bb.putShort((short) 0); - return buf; - } - - private static boolean hasDelta(byte[] frame) { - return frame.length >= 12 && (frame[5] & 0x08) != 0; - } - - private static int tableCount(byte[] frame) { - return (frame[6] & 0xFF) | ((frame[7] & 0xFF) << 8); - } - - private void accumulate(byte[] frame) { - if (!hasDelta(frame)) { - return; - } - int[] pos = {12}; - int deltaStart = readVarint(frame, pos); - int deltaCount = readVarint(frame, pos); - while (dict.size() < deltaStart) { - dict.add(null); - } - for (int i = 0; i < deltaCount; i++) { - int len = readVarint(frame, pos); - String sym = new String(frame, pos[0], len, StandardCharsets.UTF_8); - pos[0] += len; - int idx = deltaStart + i; - while (dict.size() <= idx) { - dict.add(null); - } - dict.set(idx, sym); - } - } } private static class SilentHandler implements TestWebSocketServer.WebSocketServerHandler { @@ -1510,20 +1452,11 @@ private static class CountingHandler implements TestWebSocketServer.WebSocketSer public void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] data) { frames.incrementAndGet(); try { - client.sendBinary(buildAck(nextSeq.getAndIncrement())); + client.sendBinary(QwpWireTestUtils.buildAck(nextSeq.getAndIncrement())); } catch (IOException e) { throw new RuntimeException(e); } } - - private static byte[] buildAck(long seq) { - byte[] buf = new byte[1 + 8 + 2]; - ByteBuffer bb = ByteBuffer.wrap(buf).order(ByteOrder.LITTLE_ENDIAN); - bb.put((byte) 0x00); - bb.putLong(seq); - bb.putShort((short) 0); - return buf; - } } /** * Short-writes the FIRST dictionary append after it is armed, modelling a disk that diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWireTestUtils.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWireTestUtils.java new file mode 100644 index 00000000..2d2cc0b1 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWireTestUtils.java @@ -0,0 +1,95 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client; + +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.charset.StandardCharsets; +import java.util.List; + +import static io.questdb.client.cutlass.qwp.protocol.QwpConstants.FLAG_DELTA_SYMBOL_DICT; +import static io.questdb.client.cutlass.qwp.protocol.QwpConstants.HEADER_SIZE; + +final class QwpWireTestUtils { + + static void accumulateDeltaDictionary(byte[] frame, List dictionary) { + if (!hasDelta(frame)) { + return; + } + int[] position = {HEADER_SIZE}; + int deltaStart = readVarint(frame, position); + int deltaCount = readVarint(frame, position); + while (dictionary.size() < deltaStart) { + dictionary.add(null); + } + for (int i = 0; i < deltaCount; i++) { + int length = readVarint(frame, position); + String symbol = new String(frame, position[0], length, StandardCharsets.UTF_8); + position[0] += length; + int symbolId = deltaStart + i; + while (dictionary.size() <= symbolId) { + dictionary.add(null); + } + dictionary.set(symbolId, symbol); + } + } + + static byte[] buildAck(long sequence) { + byte[] buffer = new byte[1 + Long.BYTES + Short.BYTES]; + ByteBuffer byteBuffer = ByteBuffer.wrap(buffer).order(ByteOrder.LITTLE_ENDIAN); + byteBuffer.put((byte) 0x00); + byteBuffer.putLong(sequence); + byteBuffer.putShort((short) 0); + return buffer; + } + + static boolean hasDelta(byte[] frame) { + return frame.length >= HEADER_SIZE && (frame[5] & FLAG_DELTA_SYMBOL_DICT) != 0; + } + + static int readVarint(byte[] buffer, int[] position) { + int result = 0; + int shift = 0; + while (position[0] < buffer.length) { + int b = buffer[position[0]++] & 0xFF; + result |= (b & 0x7F) << shift; + if ((b & 0x80) == 0) { + return result; + } + shift += 7; + if (shift > 28) { + throw new IllegalStateException("varint too long"); + } + } + throw new IllegalStateException("varint truncated"); + } + + static int tableCount(byte[] frame) { + return (frame[6] & 0xFF) | ((frame[7] & 0xFF) << 8); + } + + private QwpWireTestUtils() { + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java index 280155a7..aea6d1af 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java @@ -32,8 +32,11 @@ import io.questdb.client.test.tools.TestUtils; import org.junit.Assert; import org.junit.Assume; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import java.io.IOException; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; @@ -48,51 +51,41 @@ public class PersistedSymbolDictTest { private static final int CRC_SIZE = 4; private static final int HEADER_SIZE = 8; + @Rule + public final TemporaryFolder temporaryFolder = TemporaryFolder.builder().assureDeletion().build(); + @Test public void testAppendPersistsAcrossReopen() throws Exception { assertMemoryLeak(() -> { - Path dir = Files.createTempDirectory("qwp-symdict"); - try { - PersistedSymbolDict d = PersistedSymbolDict.open(dir.toString()); + Path dir = newFolder("qwp-symdict"); + try (PersistedSymbolDict d = PersistedSymbolDict.open(dir.toString())) { Assert.assertNotNull(d); - try { - Assert.assertEquals(0, d.size()); - d.appendSymbol("AAPL"); - d.appendSymbol("GOOG"); - d.appendSymbol("MSFT"); - Assert.assertEquals(3, d.size()); - } finally { - d.close(); - } + Assert.assertEquals(0, d.size()); + d.appendSymbol("AAPL"); + d.appendSymbol("GOOG"); + d.appendSymbol("MSFT"); + Assert.assertEquals(3, d.size()); + } - // Reopen: entries recovered in id order. - PersistedSymbolDict reopened = PersistedSymbolDict.open(dir.toString()); + // Reopen: entries recovered in id order. + try (PersistedSymbolDict reopened = PersistedSymbolDict.open(dir.toString())) { Assert.assertNotNull(reopened); - try { - Assert.assertEquals(3, reopened.size()); - ObjList symbols = reopened.readLoadedSymbols(); - Assert.assertEquals(3, symbols.size()); - Assert.assertEquals("AAPL", symbols.getQuick(0)); - Assert.assertEquals("GOOG", symbols.getQuick(1)); - Assert.assertEquals("MSFT", symbols.getQuick(2)); - Assert.assertTrue(reopened.loadedEntriesLen() > 0); - - // Appending after recovery continues from the recovered tip. - reopened.appendSymbol("TSLA"); - Assert.assertEquals(4, reopened.size()); - } finally { - reopened.close(); - } + Assert.assertEquals(3, reopened.size()); + ObjList symbols = reopened.readLoadedSymbols(); + Assert.assertEquals(3, symbols.size()); + Assert.assertEquals("AAPL", symbols.getQuick(0)); + Assert.assertEquals("GOOG", symbols.getQuick(1)); + Assert.assertEquals("MSFT", symbols.getQuick(2)); + Assert.assertTrue(reopened.loadedEntriesLen() > 0); + + // Appending after recovery continues from the recovered tip. + reopened.appendSymbol("TSLA"); + Assert.assertEquals(4, reopened.size()); + } - PersistedSymbolDict third = PersistedSymbolDict.open(dir.toString()); - try { - Assert.assertEquals(4, third.size()); - Assert.assertEquals("TSLA", third.readLoadedSymbols().getQuick(3)); - } finally { - third.close(); - } - } finally { - rmDir(dir); + try (PersistedSymbolDict third = PersistedSymbolDict.open(dir.toString())) { + Assert.assertEquals(4, third.size()); + Assert.assertEquals("TSLA", third.readLoadedSymbols().getQuick(3)); } }); } @@ -107,48 +100,33 @@ public void testAppendRawEntriesMatchesAppendSymbols() throws Exception { // appendRawEntries into a fresh dict, and assert the recovered symbols // match -- including an empty entry mid-range. assertMemoryLeak(() -> { - Path src = Files.createTempDirectory("qwp-symdict-src"); - Path dst = Files.createTempDirectory("qwp-symdict-dst"); - try { - GlobalSymbolDictionary dict = new GlobalSymbolDictionary(); - dict.getOrAddSymbol("AAPL"); // id 0 - dict.getOrAddSymbol(""); // id 1 -- empty entry mid-range - dict.getOrAddSymbol("MSFT"); // id 2 - - PersistedSymbolDict encoded = PersistedSymbolDict.open(src.toString()); + Path src = newFolder("qwp-symdict-src"); + Path dst = newFolder("qwp-symdict-dst"); + GlobalSymbolDictionary dict = new GlobalSymbolDictionary(); + dict.getOrAddSymbol("AAPL"); // id 0 + dict.getOrAddSymbol(""); // id 1 -- empty entry mid-range + dict.getOrAddSymbol("MSFT"); // id 2 + + try (PersistedSymbolDict encoded = PersistedSymbolDict.open(src.toString())) { encoded.appendSymbols(dict, 0, 2); - encoded.close(); + } - // Reopen to obtain the on-disk entry region [len][utf8]... verbatim, - // then replay it byte-for-byte into a fresh dict via appendRawEntries. - PersistedSymbolDict reopened = PersistedSymbolDict.open(src.toString()); - try { - PersistedSymbolDict raw = PersistedSymbolDict.open(dst.toString()); - try { - raw.appendRawEntries(reopened.loadedEntriesAddr(), - reopened.loadedEntriesLen(), reopened.size()); - Assert.assertEquals(3, raw.size()); - } finally { - raw.close(); - } - } finally { - reopened.close(); - } + // Reopen to obtain the on-disk entry region [len][utf8]... verbatim, + // then replay it byte-for-byte into a fresh dict via appendRawEntries. + try (PersistedSymbolDict reopened = PersistedSymbolDict.open(src.toString()); + PersistedSymbolDict raw = PersistedSymbolDict.open(dst.toString())) { + raw.appendRawEntries(reopened.loadedEntriesAddr(), + reopened.loadedEntriesLen(), reopened.size()); + Assert.assertEquals(3, raw.size()); + } - // The raw-appended dict must recover the same dense symbols. - PersistedSymbolDict recovered = PersistedSymbolDict.open(dst.toString()); - try { - Assert.assertEquals(3, recovered.size()); - ObjList symbols = recovered.readLoadedSymbols(); - Assert.assertEquals("AAPL", symbols.getQuick(0)); - Assert.assertEquals("", symbols.getQuick(1)); - Assert.assertEquals("MSFT", symbols.getQuick(2)); - } finally { - recovered.close(); - } - } finally { - rmDir(src); - rmDir(dst); + // The raw-appended dict must recover the same dense symbols. + try (PersistedSymbolDict recovered = PersistedSymbolDict.open(dst.toString())) { + Assert.assertEquals(3, recovered.size()); + ObjList symbols = recovered.readLoadedSymbols(); + Assert.assertEquals("AAPL", symbols.getQuick(0)); + Assert.assertEquals("", symbols.getQuick(1)); + Assert.assertEquals("MSFT", symbols.getQuick(2)); } }); } @@ -166,39 +144,29 @@ public void testAppendRawEntriesRejectsInconsistentTripleUnderAssertions() throw assert assertionsEnabled = true; Assume.assumeTrue("the guard is assert-gated; only observable under -ea", assertionsEnabled); assertMemoryLeak(() -> { - Path src = Files.createTempDirectory("qwp-symdict-src"); - Path dst = Files.createTempDirectory("qwp-symdict-dst"); - try { - GlobalSymbolDictionary dict = new GlobalSymbolDictionary(); - dict.getOrAddSymbol("AAPL"); // id 0 - dict.getOrAddSymbol("MSFT"); // id 1 - dict.getOrAddSymbol("GOOG"); // id 2 - PersistedSymbolDict encoded = PersistedSymbolDict.open(src.toString()); + Path src = newFolder("qwp-symdict-src"); + Path dst = newFolder("qwp-symdict-dst"); + GlobalSymbolDictionary dict = new GlobalSymbolDictionary(); + dict.getOrAddSymbol("AAPL"); // id 0 + dict.getOrAddSymbol("MSFT"); // id 1 + dict.getOrAddSymbol("GOOG"); // id 2 + try (PersistedSymbolDict encoded = PersistedSymbolDict.open(src.toString())) { encoded.appendSymbols(dict, 0, 2); - encoded.close(); + } - // Grab the on-disk entry region (3 entries) verbatim, then feed it back - // with a count of 1: validateRawEntries stops one entry in with - // src < srcLimit and throws "under-filled", before any bytes are written. - PersistedSymbolDict reopened = PersistedSymbolDict.open(src.toString()); + // Grab the on-disk entry region (3 entries) verbatim, then feed it back + // with a count of 1: validateRawEntries stops one entry in with + // src < srcLimit and throws "under-filled", before any bytes are written. + try (PersistedSymbolDict reopened = PersistedSymbolDict.open(src.toString()); + PersistedSymbolDict raw = PersistedSymbolDict.open(dst.toString())) { try { - PersistedSymbolDict raw = PersistedSymbolDict.open(dst.toString()); - try { - raw.appendRawEntries(reopened.loadedEntriesAddr(), reopened.loadedEntriesLen(), 1); - Assert.fail("an inconsistent (len,count) triple must be rejected"); - } catch (IllegalStateException expected) { - Assert.assertTrue("message names the under-filled buffer: " + expected.getMessage(), - expected.getMessage().contains("under-filled")); - Assert.assertEquals("a rejected triple must write nothing", 0, raw.size()); - } finally { - raw.close(); - } - } finally { - reopened.close(); + raw.appendRawEntries(reopened.loadedEntriesAddr(), reopened.loadedEntriesLen(), 1); + Assert.fail("an inconsistent (len,count) triple must be rejected"); + } catch (IllegalStateException expected) { + Assert.assertTrue("message names the under-filled buffer: " + expected.getMessage(), + expected.getMessage().contains("under-filled")); + Assert.assertEquals("a rejected triple must write nothing", 0, raw.size()); } - } finally { - rmDir(src); - rmDir(dst); } }); } @@ -214,63 +182,49 @@ public void testAppendRawEntriesShortWriteThrowsWithoutAdvancingIsIdempotentOnRe // the persist-FAILURE trigger, whose idempotency the write-ahead // (persistNewSymbolsBeforePublish, resuming from pd.size()) relies on. assertMemoryLeak(() -> { - Path src = Files.createTempDirectory("qwp-symdict-src"); - Path dst = Files.createTempDirectory("qwp-symdict-dst"); - try { - GlobalSymbolDictionary dict = new GlobalSymbolDictionary(); - dict.getOrAddSymbol("AAPL"); // id 0 - dict.getOrAddSymbol("MSFT"); // id 1 - - // Encode the range once to obtain its on-disk [len][utf8]... bytes. - PersistedSymbolDict encoded = PersistedSymbolDict.open(src.toString()); + Path src = newFolder("qwp-symdict-src"); + Path dst = newFolder("qwp-symdict-dst"); + GlobalSymbolDictionary dict = new GlobalSymbolDictionary(); + dict.getOrAddSymbol("AAPL"); // id 0 + dict.getOrAddSymbol("MSFT"); // id 1 + + // Encode the range once to obtain its on-disk [len][utf8]... bytes. + try (PersistedSymbolDict encoded = PersistedSymbolDict.open(src.toString())) { encoded.appendSymbols(dict, 0, 1); - encoded.close(); + } - PersistedSymbolDict source = PersistedSymbolDict.open(src.toString()); - try { - long addr = source.loadedEntriesAddr(); - int rawLen = source.loadedEntriesLen(); - int count = source.size(); + try (PersistedSymbolDict source = PersistedSymbolDict.open(src.toString())) { + long addr = source.loadedEntriesAddr(); + int rawLen = source.loadedEntriesLen(); + int count = source.size(); - ShortWriteOnceFacade ff = new ShortWriteOnceFacade(); - PersistedSymbolDict d = PersistedSymbolDict.open(ff, dst.toString()); + ShortWriteOnceFacade ff = new ShortWriteOnceFacade(); + try (PersistedSymbolDict d = PersistedSymbolDict.open(ff, dst.toString())) { Assert.assertNotNull(d); + ff.armed = true; // the next entry append lands short try { - ff.armed = true; // the next entry append lands short - try { - d.appendRawEntries(addr, rawLen, count); - Assert.fail("a short write must throw"); - } catch (IllegalStateException expected) { - Assert.assertTrue("short-write error: " + expected.getMessage(), - expected.getMessage().contains("short write")); - } - // The throw preceded the size/offset advance: nothing persisted. - Assert.assertEquals("short write must NOT advance size", 0, d.size()); - - // Retry the SAME bytes (the facade auto-disarmed): the write - // lands at the unchanged offset, overwriting the torn prefix. d.appendRawEntries(addr, rawLen, count); - Assert.assertEquals(2, d.size()); - } finally { - d.close(); + Assert.fail("a short write must throw"); + } catch (IllegalStateException expected) { + Assert.assertTrue("short-write error: " + expected.getMessage(), + expected.getMessage().contains("short write")); } - } finally { - source.close(); - } + // The throw preceded the size/offset advance: nothing persisted. + Assert.assertEquals("short write must NOT advance size", 0, d.size()); - // Recovery sees a gap-free, duplicate-free dictionary (size 2, not 3). - PersistedSymbolDict reopened = PersistedSymbolDict.open(dst.toString()); - try { - Assert.assertEquals("retry must not duplicate or gap the dictionary", 2, reopened.size()); - ObjList symbols = reopened.readLoadedSymbols(); - Assert.assertEquals("AAPL", symbols.getQuick(0)); - Assert.assertEquals("MSFT", symbols.getQuick(1)); - } finally { - reopened.close(); + // Retry the SAME bytes (the facade auto-disarmed): the write + // lands at the unchanged offset, overwriting the torn prefix. + d.appendRawEntries(addr, rawLen, count); + Assert.assertEquals(2, d.size()); } - } finally { - rmDir(src); - rmDir(dst); + } + + // Recovery sees a gap-free, duplicate-free dictionary (size 2, not 3). + try (PersistedSymbolDict reopened = PersistedSymbolDict.open(dst.toString())) { + Assert.assertEquals("retry must not duplicate or gap the dictionary", 2, reopened.size()); + ObjList symbols = reopened.readLoadedSymbols(); + Assert.assertEquals("AAPL", symbols.getQuick(0)); + Assert.assertEquals("MSFT", symbols.getQuick(1)); } }); } @@ -283,52 +237,39 @@ public void testAppendSymbolsBatchWritesDenseRange() throws Exception { // batched call keyed off size() must continue densely (the resume-from- // durable-size contract the producer relies on). assertMemoryLeak(() -> { - Path dir = Files.createTempDirectory("qwp-symdict"); - try { - GlobalSymbolDictionary dict = new GlobalSymbolDictionary(); - dict.getOrAddSymbol("AAPL"); // id 0 - dict.getOrAddSymbol(""); // id 1 -- empty symbol mid-range - dict.getOrAddSymbol("MSFT"); // id 2 - dict.getOrAddSymbol("TSLA"); // id 3 - - PersistedSymbolDict d = PersistedSymbolDict.open(dir.toString()); - try { - d.appendSymbols(dict, 0, 3); // one write for all four ids - Assert.assertEquals(4, d.size()); - d.appendSymbols(dict, 4, 3); // empty range (to < from) is a no-op - Assert.assertEquals(4, d.size()); - } finally { - d.close(); - } + Path dir = newFolder("qwp-symdict"); + GlobalSymbolDictionary dict = new GlobalSymbolDictionary(); + dict.getOrAddSymbol("AAPL"); // id 0 + dict.getOrAddSymbol(""); // id 1 -- empty symbol mid-range + dict.getOrAddSymbol("MSFT"); // id 2 + dict.getOrAddSymbol("TSLA"); // id 3 + + try (PersistedSymbolDict d = PersistedSymbolDict.open(dir.toString())) { + d.appendSymbols(dict, 0, 3); // one write for all four ids + Assert.assertEquals(4, d.size()); + d.appendSymbols(dict, 4, 3); // empty range (to < from) is a no-op + Assert.assertEquals(4, d.size()); + } - PersistedSymbolDict reopened = PersistedSymbolDict.open(dir.toString()); - try { - Assert.assertEquals(4, reopened.size()); - ObjList symbols = reopened.readLoadedSymbols(); - Assert.assertEquals(4, symbols.size()); - Assert.assertEquals("AAPL", symbols.getQuick(0)); - Assert.assertEquals("", symbols.getQuick(1)); - Assert.assertEquals("MSFT", symbols.getQuick(2)); - Assert.assertEquals("TSLA", symbols.getQuick(3)); - - // A follow-on batch keyed off the recovered size continues - // the dense sequence without a gap or duplicate. - dict.getOrAddSymbol("NVDA"); // id 4 - reopened.appendSymbols(dict, reopened.size(), 4); - Assert.assertEquals(5, reopened.size()); - } finally { - reopened.close(); - } + try (PersistedSymbolDict reopened = PersistedSymbolDict.open(dir.toString())) { + Assert.assertEquals(4, reopened.size()); + ObjList symbols = reopened.readLoadedSymbols(); + Assert.assertEquals(4, symbols.size()); + Assert.assertEquals("AAPL", symbols.getQuick(0)); + Assert.assertEquals("", symbols.getQuick(1)); + Assert.assertEquals("MSFT", symbols.getQuick(2)); + Assert.assertEquals("TSLA", symbols.getQuick(3)); + + // A follow-on batch keyed off the recovered size continues + // the dense sequence without a gap or duplicate. + dict.getOrAddSymbol("NVDA"); // id 4 + reopened.appendSymbols(dict, reopened.size(), 4); + Assert.assertEquals(5, reopened.size()); + } - PersistedSymbolDict third = PersistedSymbolDict.open(dir.toString()); - try { - Assert.assertEquals(5, third.size()); - Assert.assertEquals("NVDA", third.readLoadedSymbols().getQuick(4)); - } finally { - third.close(); - } - } finally { - rmDir(dir); + try (PersistedSymbolDict third = PersistedSymbolDict.open(dir.toString())) { + Assert.assertEquals(5, third.size()); + Assert.assertEquals("NVDA", third.readLoadedSymbols().getQuick(4)); } }); } @@ -342,47 +283,37 @@ public void testAppendSymbolsShortWriteThrowsWithoutAdvancingIsIdempotentOnRetry // gap-free, duplicate-free dictionary. A regression that advanced size before // the written==len check would strand a torn/duplicated dictionary here. assertMemoryLeak(() -> { - Path dir = Files.createTempDirectory("qwp-symdict"); - try { - GlobalSymbolDictionary dict = new GlobalSymbolDictionary(); - dict.getOrAddSymbol("AAPL"); // id 0 - dict.getOrAddSymbol("GOOG"); // id 1 + Path dir = newFolder("qwp-symdict"); + GlobalSymbolDictionary dict = new GlobalSymbolDictionary(); + dict.getOrAddSymbol("AAPL"); // id 0 + dict.getOrAddSymbol("GOOG"); // id 1 - ShortWriteOnceFacade ff = new ShortWriteOnceFacade(); - PersistedSymbolDict d = PersistedSymbolDict.open(ff, dir.toString()); + ShortWriteOnceFacade ff = new ShortWriteOnceFacade(); + try (PersistedSymbolDict d = PersistedSymbolDict.open(ff, dir.toString())) { Assert.assertNotNull(d); + ff.armed = true; // the next entry append lands short try { - ff.armed = true; // the next entry append lands short - try { - d.appendSymbols(dict, 0, 1); - Assert.fail("a short write must throw"); - } catch (IllegalStateException expected) { - Assert.assertTrue("short-write error: " + expected.getMessage(), - expected.getMessage().contains("short write")); - } - // The throw preceded the size/offset advance: nothing persisted. - Assert.assertEquals("short write must NOT advance size", 0, d.size()); - - // Retry the SAME range (the facade auto-disarmed): re-writes at - // the unchanged offset, overwriting the torn bytes. d.appendSymbols(dict, 0, 1); - Assert.assertEquals(2, d.size()); - } finally { - d.close(); - } + Assert.fail("a short write must throw"); + } catch (IllegalStateException expected) { + Assert.assertTrue("short-write error: " + expected.getMessage(), + expected.getMessage().contains("short write")); + } + // The throw preceded the size/offset advance: nothing persisted. + Assert.assertEquals("short write must NOT advance size", 0, d.size()); + + // Retry the SAME range (the facade auto-disarmed): re-writes at + // the unchanged offset, overwriting the torn bytes. + d.appendSymbols(dict, 0, 1); + Assert.assertEquals(2, d.size()); + } - // Recovery sees a gap-free, duplicate-free dictionary (size 2, not 3). - PersistedSymbolDict reopened = PersistedSymbolDict.open(dir.toString()); - try { - Assert.assertEquals("retry must not duplicate or gap the dictionary", 2, reopened.size()); - ObjList symbols = reopened.readLoadedSymbols(); - Assert.assertEquals("AAPL", symbols.getQuick(0)); - Assert.assertEquals("GOOG", symbols.getQuick(1)); - } finally { - reopened.close(); - } - } finally { - rmDir(dir); + // Recovery sees a gap-free, duplicate-free dictionary (size 2, not 3). + try (PersistedSymbolDict reopened = PersistedSymbolDict.open(dir.toString())) { + Assert.assertEquals("retry must not duplicate or gap the dictionary", 2, reopened.size()); + ObjList symbols = reopened.readLoadedSymbols(); + Assert.assertEquals("AAPL", symbols.getQuick(0)); + Assert.assertEquals("GOOG", symbols.getQuick(1)); } }); } @@ -404,44 +335,34 @@ public void testBatchAppendWritesOneChunkWithOneChecksum() throws Exception { // symbols cost N * ([len varint] + 1 utf8 byte) of entries, wrapped in one chunk. // Per-entry CRCs would add 4 * N bytes instead of 4. assertMemoryLeak(() -> { - Path dir = Files.createTempDirectory("qwp-symdict"); - try { - final int n = 26; // 'a'..'z' -- distinct, and one UTF-8 byte each - GlobalSymbolDictionary dict = new GlobalSymbolDictionary(); - for (int i = 0; i < n; i++) { - dict.getOrAddSymbol(String.valueOf((char) ('a' + i))); - } - Assert.assertEquals(n, dict.size()); - PersistedSymbolDict d = PersistedSymbolDict.open(dir.toString()); + Path dir = newFolder("qwp-symdict"); + final int n = 26; // 'a'..'z' -- distinct, and one UTF-8 byte each + GlobalSymbolDictionary dict = new GlobalSymbolDictionary(); + for (int i = 0; i < n; i++) { + dict.getOrAddSymbol(String.valueOf((char) ('a' + i))); + } + Assert.assertEquals(n, dict.size()); + try (PersistedSymbolDict d = PersistedSymbolDict.open(dir.toString())) { Assert.assertNotNull(d); - try { - d.appendSymbols(dict, 0, n - 1); - Assert.assertEquals(n, d.size()); - } finally { - d.close(); - } + d.appendSymbols(dict, 0, n - 1); + Assert.assertEquals(n, d.size()); + } - int entryBytes = n * 2; // [len=1 varint][1 utf8 byte] - int chunkHeader = 1 + varintSize(entryBytes); // entryCount=50 fits one byte - long expected = HEADER_SIZE + chunkHeader + entryBytes + CRC_SIZE; - Assert.assertEquals( - "a batch append must cost ONE chunk checksum, not one per symbol", - expected, Files.size(dir.resolve(".symbol-dict"))); + int entryBytes = n * 2; // [len=1 varint][1 utf8 byte] + int chunkHeader = 1 + varintSize(entryBytes); // entryCount=50 fits one byte + long expected = HEADER_SIZE + chunkHeader + entryBytes + CRC_SIZE; + Assert.assertEquals( + "a batch append must cost ONE chunk checksum, not one per symbol", + expected, Files.size(dir.resolve(".symbol-dict"))); - // And it must still read back symbol-for-symbol. - PersistedSymbolDict re = PersistedSymbolDict.open(dir.toString()); + // And it must still read back symbol-for-symbol. + try (PersistedSymbolDict re = PersistedSymbolDict.open(dir.toString())) { Assert.assertNotNull(re); - try { - Assert.assertEquals(n, re.size()); - ObjList got = re.readLoadedSymbols(); - for (int i = 0; i < n; i++) { - Assert.assertEquals(dict.getSymbol(i), got.getQuick(i)); - } - } finally { - re.close(); + Assert.assertEquals(n, re.size()); + ObjList got = re.readLoadedSymbols(); + for (int i = 0; i < n; i++) { + Assert.assertEquals(dict.getSymbol(i), got.getQuick(i)); } - } finally { - rmDir(dir); } }); } @@ -449,33 +370,23 @@ public void testBatchAppendWritesOneChunkWithOneChecksum() throws Exception { @Test public void testMappedAppendAmortizesFlushesWithoutPositionedWrites() throws Exception { assertMemoryLeak(() -> { - Path dir = Files.createTempDirectory("qwp-symdict"); - try { - PersistedSymbolDict d = PersistedSymbolDict.open(dir.toString()); + Path dir = newFolder("qwp-symdict"); + try (PersistedSymbolDict d = PersistedSymbolDict.open(dir.toString())) { Assert.assertNotNull(d); - try { - for (int i = 0; i < 10_000; i++) { - d.appendSymbol("sym-" + i); - } - Assert.assertEquals(10_000, d.size()); - Assert.assertEquals("production appends must write directly into the mmap", - 0L, d.appendWriteCount()); - Assert.assertEquals("ten thousand flushes must require only three mapped windows", - 3, d.appendMapGrowthCount()); - } finally { - d.close(); - } + for (int i = 0; i < 10_000; i++) { + d.appendSymbol("sym-" + i); + } + Assert.assertEquals(10_000, d.size()); + Assert.assertEquals("production appends must write directly into the mmap", + 0L, d.appendWriteCount()); + Assert.assertEquals("ten thousand flushes must require only three mapped windows", + 3, d.appendMapGrowthCount()); + } - PersistedSymbolDict re = PersistedSymbolDict.open(dir.toString()); + try (PersistedSymbolDict re = PersistedSymbolDict.open(dir.toString())) { Assert.assertNotNull(re); - try { - Assert.assertEquals(10_000, re.size()); - Assert.assertEquals("sym-9999", re.readLoadedSymbols().getQuick(9_999)); - } finally { - re.close(); - } - } finally { - rmDir(dir); + Assert.assertEquals(10_000, re.size()); + Assert.assertEquals("sym-9999", re.readLoadedSymbols().getQuick(9_999)); } }); } @@ -495,18 +406,14 @@ public void testBadMagicDegradesWithoutDestroyingTheFile() throws Exception { // fall through to openFresh(), which is O_TRUNC: a single unreadable byte // destroyed the only copy of load-bearing state. assertMemoryLeak(() -> { - Path dir = Files.createTempDirectory("qwp-symdict"); - try { - Path f = dir.resolve(".symbol-dict"); - byte[] original = new byte[]{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; - Files.write(f, original); - Assert.assertNull("an unparseable existing dict must degrade to null", - PersistedSymbolDict.open(dir.toString())); - Assert.assertArrayEquals("open() must NOT destroy an existing dictionary", - original, Files.readAllBytes(f)); - } finally { - rmDir(dir); - } + Path dir = newFolder("qwp-symdict"); + Path f = dir.resolve(".symbol-dict"); + byte[] original = new byte[]{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; + Files.write(f, original); + Assert.assertNull("an unparseable existing dict must degrade to null", + PersistedSymbolDict.open(dir.toString())); + Assert.assertArrayEquals("open() must NOT destroy an existing dictionary", + original, Files.readAllBytes(f)); }); } @@ -524,25 +431,21 @@ public void testBadVersionDegradesWithoutDestroyingTheFile() throws Exception { // unreplayable. Degrading to null instead leaves the bytes for the client that // does understand them. assertMemoryLeak(() -> { - Path dir = Files.createTempDirectory("qwp-symdict"); - try { - Path f = dir.resolve(".symbol-dict"); - PersistedSymbolDict seed = PersistedSymbolDict.open(dir.toString()); + Path dir = newFolder("qwp-symdict"); + Path f = dir.resolve(".symbol-dict"); + try (PersistedSymbolDict seed = PersistedSymbolDict.open(dir.toString())) { Assert.assertNotNull(seed); seed.appendSymbol("a"); - seed.close(); - // Corrupt ONLY the version byte (offset 4) to an unknown value. - byte[] bytes = Files.readAllBytes(f); - bytes[4] = (byte) 99; - Files.write(f, bytes); - - Assert.assertNull("an unknown-version dict must degrade to null", - PersistedSymbolDict.open(dir.toString())); - Assert.assertArrayEquals("open() must NOT destroy a future-version dictionary", - bytes, Files.readAllBytes(f)); - } finally { - rmDir(dir); } + // Corrupt ONLY the version byte (offset 4) to an unknown value. + byte[] bytes = Files.readAllBytes(f); + bytes[4] = (byte) 99; + Files.write(f, bytes); + + Assert.assertNull("an unknown-version dict must degrade to null", + PersistedSymbolDict.open(dir.toString())); + Assert.assertArrayEquals("open() must NOT destroy a future-version dictionary", + bytes, Files.readAllBytes(f)); }); } @@ -553,48 +456,35 @@ public void testCloseNullsLoadedEntries() throws Exception { // dereference freed native memory. Pre-fix the pointer survived close() // non-zero. assertMemoryLeak(() -> { - Path dir = Files.createTempDirectory("qwp-symdict"); - try { - PersistedSymbolDict d = PersistedSymbolDict.open(dir.toString()); + Path dir = newFolder("qwp-symdict"); + try (PersistedSymbolDict d = PersistedSymbolDict.open(dir.toString())) { d.appendSymbol("AAPL"); - d.close(); - - // Reopen so recovery loads the entries into native memory. - PersistedSymbolDict re = PersistedSymbolDict.open(dir.toString()); - Assert.assertTrue("recovery must load entries into native memory", - re.loadedEntriesAddr() != 0L && re.loadedEntriesLen() > 0); - re.close(); - Assert.assertEquals("close() must null loadedEntriesAddr", 0L, re.loadedEntriesAddr()); - Assert.assertEquals("close() must null loadedEntriesLen", 0, re.loadedEntriesLen()); - } finally { - rmDir(dir); } + + // Reopen so recovery loads the entries into native memory. This test + // closes explicitly because the post-close state is the assertion target. + PersistedSymbolDict re = PersistedSymbolDict.open(dir.toString()); + Assert.assertTrue("recovery must load entries into native memory", + re.loadedEntriesAddr() != 0L && re.loadedEntriesLen() > 0); + re.close(); + Assert.assertEquals("close() must null loadedEntriesAddr", 0L, re.loadedEntriesAddr()); + Assert.assertEquals("close() must null loadedEntriesLen", 0, re.loadedEntriesLen()); }); } @Test public void testEmptySymbolRoundTrips() throws Exception { assertMemoryLeak(() -> { - Path dir = Files.createTempDirectory("qwp-symdict"); - try { - PersistedSymbolDict d = PersistedSymbolDict.open(dir.toString()); - try { - d.appendSymbol(""); - d.appendSymbol("nonempty"); - } finally { - d.close(); - } - PersistedSymbolDict re = PersistedSymbolDict.open(dir.toString()); - try { - Assert.assertEquals(2, re.size()); - ObjList s = re.readLoadedSymbols(); - Assert.assertEquals("", s.getQuick(0)); - Assert.assertEquals("nonempty", s.getQuick(1)); - } finally { - re.close(); - } - } finally { - rmDir(dir); + Path dir = newFolder("qwp-symdict"); + try (PersistedSymbolDict d = PersistedSymbolDict.open(dir.toString())) { + d.appendSymbol(""); + d.appendSymbol("nonempty"); + } + try (PersistedSymbolDict re = PersistedSymbolDict.open(dir.toString())) { + Assert.assertEquals(2, re.size()); + ObjList s = re.readLoadedSymbols(); + Assert.assertEquals("", s.getQuick(0)); + Assert.assertEquals("nonempty", s.getQuick(1)); } }); } @@ -610,49 +500,39 @@ public void testInteriorCorruptionIsCaughtNotSilentlyMisattributed() throws Exce // only the intact prefix (fail-clean: the send loop's torn-dict guard then // forces a resend of the rest). assertMemoryLeak(() -> { - Path dir = Files.createTempDirectory("qwp-symdict"); - try { - PersistedSymbolDict d = PersistedSymbolDict.open(dir.toString()); - try { - d.appendSymbol("s0"); - d.appendSymbol("s1"); - d.appendSymbol("s2"); - d.appendSymbol("s3"); - d.appendSymbol("s4"); - Assert.assertEquals(5, d.size()); - } finally { - d.close(); - } + Path dir = newFolder("qwp-symdict"); + try (PersistedSymbolDict d = PersistedSymbolDict.open(dir.toString())) { + d.appendSymbol("s0"); + d.appendSymbol("s1"); + d.appendSymbol("s2"); + d.appendSymbol("s3"); + d.appendSymbol("s4"); + Assert.assertEquals(5, d.size()); + } - // Corrupt one byte inside the 3rd chunk's entry region (id 2). Each - // appendSymbol writes one single-entry chunk - // ([entryCount][entryBytes][len][utf8][crc32c]), and all five symbols - // are the same width, so the five chunks are equal-sized -- derive the - // stride from the file rather than hard-coding it. The byte flipped is - // the last one before the chunk's trailing CRC, so that CRC goes stale. - Path f = dir.resolve(".symbol-dict"); - byte[] bytes = Files.readAllBytes(f); - int chunkLen = (bytes.length - HEADER_SIZE) / 5; - int chunk2 = HEADER_SIZE + 2 * chunkLen; - bytes[chunk2 + chunkLen - CRC_SIZE - 1] ^= 0x7F; - Files.write(f, bytes); - - PersistedSymbolDict re = PersistedSymbolDict.open(dir.toString()); + // Corrupt one byte inside the 3rd chunk's entry region (id 2). Each + // appendSymbol writes one single-entry chunk + // ([entryCount][entryBytes][len][utf8][crc32c]), and all five symbols + // are the same width, so the five chunks are equal-sized -- derive the + // stride from the file rather than hard-coding it. The byte flipped is + // the last one before the chunk's trailing CRC, so that CRC goes stale. + Path f = dir.resolve(".symbol-dict"); + byte[] bytes = Files.readAllBytes(f); + int chunkLen = (bytes.length - HEADER_SIZE) / 5; + int chunk2 = HEADER_SIZE + 2 * chunkLen; + bytes[chunk2 + chunkLen - CRC_SIZE - 1] ^= 0x7F; + Files.write(f, bytes); + + try (PersistedSymbolDict re = PersistedSymbolDict.open(dir.toString())) { Assert.assertNotNull(re); - try { - // Only the intact prefix [s0, s1] is trusted; the corrupt chunk - // and everything after it are dropped. No recovered symbol is - // the corrupted string -- the tear is DETECTED, never silently - // misattributed. - Assert.assertEquals("parse must stop at the corrupt interior chunk", 2, re.size()); - ObjList s = re.readLoadedSymbols(); - Assert.assertEquals("s0", s.getQuick(0)); - Assert.assertEquals("s1", s.getQuick(1)); - } finally { - re.close(); - } - } finally { - rmDir(dir); + // Only the intact prefix [s0, s1] is trusted; the corrupt chunk + // and everything after it are dropped. No recovered symbol is + // the corrupted string -- the tear is DETECTED, never silently + // misattributed. + Assert.assertEquals("parse must stop at the corrupt interior chunk", 2, re.size()); + ObjList s = re.readLoadedSymbols(); + Assert.assertEquals("s0", s.getQuick(0)); + Assert.assertEquals("s1", s.getQuick(1)); } }); } @@ -668,35 +548,25 @@ public void testLargeSymbolRoundTripsAcrossReopen() throws Exception { // forward's process-crash durability for large symbols. The file length is // now the only bound, so the write and read paths agree. assertMemoryLeak(() -> { - Path dir = Files.createTempDirectory("qwp-symdict"); - try { - // Just over the old 1 << 20 (1 MB) ceiling. - String big = TestUtils.repeat("x", (1 << 20) + 17); - PersistedSymbolDict d = PersistedSymbolDict.open(dir.toString()); - try { - d.appendSymbol("before"); - d.appendSymbol(big); - d.appendSymbol("after"); - Assert.assertEquals(3, d.size()); - } finally { - d.close(); - } + Path dir = newFolder("qwp-symdict"); + // Just over the old 1 << 20 (1 MB) ceiling. + String big = TestUtils.repeat("x", (1 << 20) + 17); + try (PersistedSymbolDict d = PersistedSymbolDict.open(dir.toString())) { + d.appendSymbol("before"); + d.appendSymbol(big); + d.appendSymbol("after"); + Assert.assertEquals(3, d.size()); + } - // Recovery must load ALL three; pre-fix the reopen truncated at the - // big entry and came back with size 1 (only "before" survived). - PersistedSymbolDict re = PersistedSymbolDict.open(dir.toString()); - try { - Assert.assertEquals("large entry must survive recovery, not be truncated", - 3, re.size()); - ObjList s = re.readLoadedSymbols(); - Assert.assertEquals("before", s.getQuick(0)); - Assert.assertEquals(big, s.getQuick(1)); - Assert.assertEquals("after", s.getQuick(2)); - } finally { - re.close(); - } - } finally { - rmDir(dir); + // Recovery must load ALL three; pre-fix the reopen truncated at the + // big entry and came back with size 1 (only "before" survived). + try (PersistedSymbolDict re = PersistedSymbolDict.open(dir.toString())) { + Assert.assertEquals("large entry must survive recovery, not be truncated", + 3, re.size()); + ObjList s = re.readLoadedSymbols(); + Assert.assertEquals("before", s.getQuick(0)); + Assert.assertEquals(big, s.getQuick(1)); + Assert.assertEquals("after", s.getQuick(2)); } }); } @@ -709,43 +579,30 @@ public void testOpenCleanDiscardsSurvivingDictionary() throws Exception { // fresh-start producer is not seeded from it -- shifts the dense id->symbol // mapping and misattributes symbols on the next reconnect. assertMemoryLeak(() -> { - Path dir = Files.createTempDirectory("qwp-symdict-clean"); - try { - PersistedSymbolDict stale = PersistedSymbolDict.open(dir.toString()); + Path dir = newFolder("qwp-symdict-clean"); + try (PersistedSymbolDict stale = PersistedSymbolDict.open(dir.toString())) { Assert.assertNotNull(stale); - try { - stale.appendSymbol("staleX"); - stale.appendSymbol("staleY"); - Assert.assertEquals(2, stale.size()); - } finally { - stale.close(); - } + stale.appendSymbol("staleX"); + stale.appendSymbol("staleY"); + Assert.assertEquals(2, stale.size()); + } - // Fresh start: openClean yields an EMPTY dictionary regardless of - // the survivor, and appends from id 0 again. - PersistedSymbolDict fresh = PersistedSymbolDict.openClean(dir.toString()); + // Fresh start: openClean yields an EMPTY dictionary regardless of + // the survivor, and appends from id 0 again. + try (PersistedSymbolDict fresh = PersistedSymbolDict.openClean(dir.toString())) { Assert.assertNotNull(fresh); - try { - Assert.assertEquals(0, fresh.size()); - Assert.assertEquals(0, fresh.readLoadedSymbols().size()); - fresh.appendSymbol("freshA"); - Assert.assertEquals(1, fresh.size()); - } finally { - fresh.close(); - } + Assert.assertEquals(0, fresh.size()); + Assert.assertEquals(0, fresh.readLoadedSymbols().size()); + fresh.appendSymbol("freshA"); + Assert.assertEquals(1, fresh.size()); + } - // The survivor's bytes are physically gone, not just hidden: a - // subsequent recovery open() sees only the post-clean content. - PersistedSymbolDict reopened = PersistedSymbolDict.open(dir.toString()); + // The survivor's bytes are physically gone, not just hidden: a + // subsequent recovery open() sees only the post-clean content. + try (PersistedSymbolDict reopened = PersistedSymbolDict.open(dir.toString())) { Assert.assertNotNull(reopened); - try { - Assert.assertEquals(1, reopened.size()); - Assert.assertEquals("freshA", reopened.readLoadedSymbols().getQuick(0)); - } finally { - reopened.close(); - } - } finally { - rmDir(dir); + Assert.assertEquals(1, reopened.size()); + Assert.assertEquals("freshA", reopened.readLoadedSymbols().getQuick(0)); } }); } @@ -753,18 +610,14 @@ public void testOpenCleanDiscardsSurvivingDictionary() throws Exception { @Test public void testRemoveOrphanDeletesFile() throws Exception { assertMemoryLeak(() -> { - Path dir = Files.createTempDirectory("qwp-symdict"); - try { - PersistedSymbolDict d = PersistedSymbolDict.open(dir.toString()); + Path dir = newFolder("qwp-symdict"); + try (PersistedSymbolDict d = PersistedSymbolDict.open(dir.toString())) { d.appendSymbol("A"); - d.close(); - Path f = dir.resolve(".symbol-dict"); - Assert.assertTrue(Files.exists(f)); - PersistedSymbolDict.removeOrphan(dir.toString()); - Assert.assertFalse(Files.exists(f)); - } finally { - rmDir(dir); } + Path f = dir.resolve(".symbol-dict"); + Assert.assertTrue(Files.exists(f)); + PersistedSymbolDict.removeOrphan(dir.toString()); + Assert.assertFalse(Files.exists(f)); }); } @@ -785,76 +638,62 @@ public void testTooLargeToReopenDegradesWithoutOpeningTheFile() throws Exception // null) but that the file is never even OPENED: with the guard, open() returns // before openRW; without it, openExisting opens the file and parses garbage. assertMemoryLeak(() -> { - Path dir = Files.createTempDirectory("qwp-symdict"); - try { - PersistedSymbolDict seed = PersistedSymbolDict.open(dir.toString()); + Path dir = newFolder("qwp-symdict"); + try (PersistedSymbolDict seed = PersistedSymbolDict.open(dir.toString())) { Assert.assertNotNull(seed); for (int i = 0; i < 40; i++) { seed.appendSymbol("sym" + i); } - seed.close(); - Path f = dir.resolve(".symbol-dict"); - byte[] before = Files.readAllBytes(f); - - HugeLengthFacade ff = new HugeLengthFacade(); - Assert.assertNull("a >=2GB dictionary must degrade to null", - PersistedSymbolDict.open(ff, dir.toString())); - Assert.assertEquals("the guard must short-circuit BEFORE opening the file", - 0, ff.openRwCalls); - Assert.assertArrayEquals("open() must NOT destroy or trim an oversized dictionary", - before, Files.readAllBytes(f)); - } finally { - rmDir(dir); } + Path f = dir.resolve(".symbol-dict"); + byte[] before = Files.readAllBytes(f); + + HugeLengthFacade ff = new HugeLengthFacade(); + Assert.assertNull("a >=2GB dictionary must degrade to null", + PersistedSymbolDict.open(ff, dir.toString())); + Assert.assertEquals("the guard must short-circuit BEFORE opening the file", + 0, ff.openRwCalls); + Assert.assertArrayEquals("open() must NOT destroy or trim an oversized dictionary", + before, Files.readAllBytes(f)); }); } @Test public void testTornTrailingEntrySelfHeals() throws Exception { assertMemoryLeak(() -> { - Path dir = Files.createTempDirectory("qwp-symdict"); - try { - // Write two complete entries, then a torn trailing record: a - // length prefix of 5 followed by only 2 bytes (crash mid-append). - PersistedSymbolDict d = PersistedSymbolDict.open(dir.toString()); + Path dir = newFolder("qwp-symdict"); + // Write two complete entries, then a torn trailing record: a + // length prefix of 5 followed by only 2 bytes (crash mid-append). + try (PersistedSymbolDict d = PersistedSymbolDict.open(dir.toString())) { d.appendSymbol("one"); d.appendSymbol("two"); - d.close(); - - Path f = dir.resolve(".symbol-dict"); - long cleanLen = Files.size(f); // header + "one" + "two", no tail - Files.write(f, new byte[]{(byte) 5, (byte) 'x', (byte) 'y'}, - StandardOpenOption.APPEND); - Assert.assertEquals("torn tail present before reopen", cleanLen + 3, Files.size(f)); + } - // Reopen: the torn tail is ignored; only the two complete entries load. - PersistedSymbolDict re = PersistedSymbolDict.open(dir.toString()); - try { - // open() physically truncates the torn tail: the file returns - // to its clean length, so a later SHORTER append can never - // leave residue past its end that a future recovery mis-parses - // as a ghost symbol. - Assert.assertEquals("torn tail physically dropped by open", cleanLen, Files.size(f)); - Assert.assertEquals(2, re.size()); - ObjList s = re.readLoadedSymbols(); - Assert.assertEquals("one", s.getQuick(0)); - Assert.assertEquals("two", s.getQuick(1)); - // The next append continues from the truncated tail cleanly. - re.appendSymbol("three"); - Assert.assertEquals(3, re.size()); - } finally { - re.close(); - } + Path f = dir.resolve(".symbol-dict"); + long cleanLen = Files.size(f); // header + "one" + "two", no tail + Files.write(f, new byte[]{(byte) 5, (byte) 'x', (byte) 'y'}, + StandardOpenOption.APPEND); + Assert.assertEquals("torn tail present before reopen", cleanLen + 3, Files.size(f)); + + // Reopen: the torn tail is ignored; only the two complete entries load. + try (PersistedSymbolDict re = PersistedSymbolDict.open(dir.toString())) { + // open() physically truncates the torn tail: the file returns + // to its clean length, so a later SHORTER append can never + // leave residue past its end that a future recovery mis-parses + // as a ghost symbol. + Assert.assertEquals("torn tail physically dropped by open", cleanLen, Files.size(f)); + Assert.assertEquals(2, re.size()); + ObjList s = re.readLoadedSymbols(); + Assert.assertEquals("one", s.getQuick(0)); + Assert.assertEquals("two", s.getQuick(1)); + // The next append continues from the truncated tail cleanly. + re.appendSymbol("three"); + Assert.assertEquals(3, re.size()); + } - PersistedSymbolDict re2 = PersistedSymbolDict.open(dir.toString()); - try { - Assert.assertEquals(3, re2.size()); - Assert.assertEquals("three", re2.readLoadedSymbols().getQuick(2)); - } finally { - re2.close(); - } - } finally { - rmDir(dir); + try (PersistedSymbolDict re2 = PersistedSymbolDict.open(dir.toString())) { + Assert.assertEquals(3, re2.size()); + Assert.assertEquals("three", re2.readLoadedSymbols().getQuick(2)); } }); } @@ -871,44 +710,37 @@ public void testTransientLengthFaultDegradesWithoutDestroyingTheFile() throws Ex // null (fall back to full self-sufficient frames) and leave every byte on disk // for a later attempt, once the transient clears. assertMemoryLeak(() -> { - Path dir = Files.createTempDirectory("qwp-symdict"); - try { - PersistedSymbolDict d = PersistedSymbolDict.open(dir.toString()); + Path dir = newFolder("qwp-symdict"); + try (PersistedSymbolDict d = PersistedSymbolDict.open(dir.toString())) { d.appendSymbol("one"); d.appendSymbol("two"); - d.close(); - - Path f = dir.resolve(".symbol-dict"); - byte[] before = Files.readAllBytes(f); - Assert.assertTrue("a populated dict must exceed the header", before.length > HEADER_SIZE); - - // Reopen through a facade whose length() reports the -1 stat-error - // sentinel for the (present) file -- the branch under test. - PersistedSymbolDict reopened = PersistedSymbolDict.open(new StatFailsLengthFacade(), dir.toString()); - if (reopened != null) { - // Pre-fix: open() fell through to openFresh() and handed back a fresh - // empty dict over the now-truncated file. Close its fd so only the - // assertion below reports the failure, not a leaked descriptor. - reopened.close(); - } - Assert.assertNull("a populated dict whose length cannot be stat'd must degrade to null, not truncate", - reopened); - Assert.assertArrayEquals("a transient length-stat fault must NOT destroy the dictionary", - before, Files.readAllBytes(f)); + } + + Path f = dir.resolve(".symbol-dict"); + byte[] before = Files.readAllBytes(f); + Assert.assertTrue("a populated dict must exceed the header", before.length > HEADER_SIZE); + + // Reopen through a facade whose length() reports the -1 stat-error + // sentinel for the (present) file -- the branch under test. + PersistedSymbolDict reopened = PersistedSymbolDict.open(new StatFailsLengthFacade(), dir.toString()); + if (reopened != null) { + // Pre-fix: open() fell through to openFresh() and handed back a fresh + // empty dict over the now-truncated file. Close its fd so only the + // assertion below reports the failure, not a leaked descriptor. + reopened.close(); + } + Assert.assertNull("a populated dict whose length cannot be stat'd must degrade to null, not truncate", + reopened); + Assert.assertArrayEquals("a transient length-stat fault must NOT destroy the dictionary", + before, Files.readAllBytes(f)); - // Once the filesystem recovers, the SAME file reopens its intact content. - PersistedSymbolDict re = PersistedSymbolDict.open(dir.toString()); + // Once the filesystem recovers, the SAME file reopens its intact content. + try (PersistedSymbolDict re = PersistedSymbolDict.open(dir.toString())) { Assert.assertNotNull(re); - try { - Assert.assertEquals("the intact content survives the transient", 2, re.size()); - ObjList s = re.readLoadedSymbols(); - Assert.assertEquals("one", s.getQuick(0)); - Assert.assertEquals("two", s.getQuick(1)); - } finally { - re.close(); - } - } finally { - rmDir(dir); + Assert.assertEquals("the intact content survives the transient", 2, re.size()); + ObjList s = re.readLoadedSymbols(); + Assert.assertEquals("one", s.getQuick(0)); + Assert.assertEquals("two", s.getQuick(1)); } }); } @@ -928,52 +760,44 @@ public void testTruncateFailureDegradesWithoutDestroyingTheFile() throws Excepti // makes every surviving delta frame permanently unreplayable. Degrading leaves // the bytes for the next attempt, once the mount is writable again. assertMemoryLeak(() -> { - Path dir = Files.createTempDirectory("qwp-symdict"); - try { - PersistedSymbolDict d = PersistedSymbolDict.open(dir.toString()); + Path dir = newFolder("qwp-symdict"); + try (PersistedSymbolDict d = PersistedSymbolDict.open(dir.toString())) { d.appendSymbol("one"); d.appendSymbol("two"); - d.close(); - - // Append a torn trailing record so the reopen parses [one, two], then - // finds validLen < len and tries to truncate the tail -- the branch - // under test. - Path f = dir.resolve(".symbol-dict"); - long cleanLen = Files.size(f); // header + "one" + "two", no tail - Files.write(f, new byte[]{(byte) 5, (byte) 'x', (byte) 'y'}, StandardOpenOption.APPEND); - Assert.assertEquals("torn tail present before reopen", cleanLen + 3, Files.size(f)); - byte[] before = Files.readAllBytes(f); - - // Reopen through a facade whose truncate() fails. - Assert.assertNull("a dict whose torn tail cannot be trimmed must degrade to null", - PersistedSymbolDict.open(new FailingTruncateFacade(), dir.toString())); - Assert.assertArrayEquals("a failed truncate must NOT destroy the dictionary", - before, Files.readAllBytes(f)); - - // And once the filesystem is writable again, the SAME file recovers its - // intact prefix in full -- which is the whole point of not destroying it. - PersistedSymbolDict re = PersistedSymbolDict.open(dir.toString()); + } + + // Append a torn trailing record so the reopen parses [one, two], then + // finds validLen < len and tries to truncate the tail -- the branch + // under test. + Path f = dir.resolve(".symbol-dict"); + long cleanLen = Files.size(f); // header + "one" + "two", no tail + Files.write(f, new byte[]{(byte) 5, (byte) 'x', (byte) 'y'}, StandardOpenOption.APPEND); + Assert.assertEquals("torn tail present before reopen", cleanLen + 3, Files.size(f)); + byte[] before = Files.readAllBytes(f); + + // Reopen through a facade whose truncate() fails. + Assert.assertNull("a dict whose torn tail cannot be trimmed must degrade to null", + PersistedSymbolDict.open(new FailingTruncateFacade(), dir.toString())); + Assert.assertArrayEquals("a failed truncate must NOT destroy the dictionary", + before, Files.readAllBytes(f)); + + // And once the filesystem is writable again, the SAME file recovers its + // intact prefix in full -- which is the whole point of not destroying it. + try (PersistedSymbolDict re = PersistedSymbolDict.open(dir.toString())) { Assert.assertNotNull(re); - try { - Assert.assertEquals("the intact prefix survives the transient", 2, re.size()); - ObjList s = re.readLoadedSymbols(); - Assert.assertEquals("one", s.getQuick(0)); - Assert.assertEquals("two", s.getQuick(1)); - Assert.assertEquals("the torn tail is trimmed once truncate works", cleanLen, Files.size(f)); - } finally { - re.close(); - } - } finally { - rmDir(dir); + Assert.assertEquals("the intact prefix survives the transient", 2, re.size()); + ObjList s = re.readLoadedSymbols(); + Assert.assertEquals("one", s.getQuick(0)); + Assert.assertEquals("two", s.getQuick(1)); + Assert.assertEquals("the torn tail is trimmed once truncate works", cleanLen, Files.size(f)); } }); } - private static void rmDir(Path dir) { - TestUtils.removeTmpDirRec(dir == null ? null : dir.toString()); + private Path newFolder(String name) throws IOException { + return temporaryFolder.newFolder(name).toPath(); } - /** * Fails every {@link #truncate(int, long)} -- reproducing a host where the * torn/stale-tail truncate cannot succeed (read-only remount, Windows share From 78a1d3206499e7828d185c885f4baae860323359 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Thu, 16 Jul 2026 12:31:16 +0100 Subject: [PATCH 74/96] Fix Windows QWP client test failures --- .../qwp/client/QwpWebSocketSender.java | 32 +++++++++++++++++++ .../qwp/client/sf/cursor/SlotLock.java | 12 ++++--- .../qwp/client/DeltaDictRecoveryTest.java | 30 +++++++---------- .../qwp/client/SelfSufficientFramesTest.java | 21 +++++------- .../qwp/client/sf/cursor/SlotLockTest.java | 6 ++-- 5 files changed, 63 insertions(+), 38 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index 696e168e..85cf1148 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -1796,6 +1796,38 @@ public long getPendingBytes() { return pendingBytes; } + /** + * Snapshot of the producer's symbol prefix whose persisted-dictionary chunks + * have committed. The persisted size advances only after the chunk CRC and + * payload have been written, so this observes the write-ahead boundary without + * reopening the live mmap-backed dictionary (which would attempt recovery-tail + * truncation and is not supported while the producer owns the file on Windows). + * Returns {@code null} in memory mode or when the persisted dictionary is + * unavailable. + */ + @TestOnly + public ObjList getPersistedSymbolsForTest() { + CursorSendEngine engine = cursorEngine; + if (engine == null) { + return null; + } + PersistedSymbolDict persisted = engine.getPersistedSymbolDict(); + if (persisted == null) { + return null; + } + int persistedSize = persisted.size(); + int globalSize = globalSymbolDictionary.size(); + if (persistedSize > globalSize) { + throw new IllegalStateException("persisted symbol dictionary exceeds producer dictionary" + + " [persisted=" + persistedSize + ", producer=" + globalSize + ']'); + } + ObjList snapshot = new ObjList<>(persistedSize); + for (int i = 0; i < persistedSize; i++) { + snapshot.add(globalSymbolDictionary.getSymbol(i)); + } + return snapshot; + } + /** * Server-advertised cap on the per-batch raw byte size. Zero before the * first connect; updated by every successful reconnect via diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SlotLock.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SlotLock.java index eb6281a7..0ec9f9f5 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SlotLock.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SlotLock.java @@ -31,6 +31,8 @@ import io.questdb.client.std.Unsafe; import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.nio.file.Paths; /** * Advisory exclusive locks for a single SF slot. @@ -101,13 +103,15 @@ public static SlotLock acquire(String slotDir) { */ public static SlotLock acquireLogical(String slotDir) { validateSlotDir(slotDir); - int separator = slotDir.lastIndexOf('/'); - if (separator < 1 || separator == slotDir.length() - 1) { + Path slotPath = Paths.get(slotDir); + Path parentPath = slotPath.getParent(); + Path slotNamePath = slotPath.getFileName(); + if (parentPath == null || slotNamePath == null || slotNamePath.toString().isEmpty()) { throw new IllegalArgumentException( "slotDir must contain a parent and slot name: " + slotDir); } - String parentDir = slotDir.substring(0, separator); - String slotName = slotDir.substring(separator + 1); + String parentDir = parentPath.toString(); + String slotName = slotNamePath.toString(); String logicalLockDir = parentDir + "/" + LOGICAL_LOCK_DIR_NAME; ensureDirectory(logicalLockDir, "logical slot lock dir"); String lockPath = logicalLockDir + "/" + slotName + ".lock"; diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java index 885a3699..14d6c846 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java @@ -877,15 +877,11 @@ public void testFailedPublishDoesNotDuplicatePersistedSymbols() throws Exception // The persisted dictionary must hold "s0" EXACTLY ONCE. // Pre-fix, the retry duplicated it (size == 2). - PersistedSymbolDict pd = PersistedSymbolDict.open(Paths.get(sfDir, "default").toString()); - Assert.assertNotNull(pd); - try { - Assert.assertEquals("failed-publish retry must not duplicate the persisted symbol", - 1, pd.size()); - Assert.assertEquals("s0", pd.readLoadedSymbols().getQuick(0)); - } finally { - pd.close(); - } + ObjList persisted = ((QwpWebSocketSender) sender).getPersistedSymbolsForTest(); + Assert.assertNotNull(persisted); + Assert.assertEquals("failed-publish retry must not duplicate the persisted symbol", + 1, persisted.size()); + Assert.assertEquals("s0", persisted.getQuick(0)); } finally { try { sender.close(); @@ -942,16 +938,12 @@ public void testFailedPublishThenNewSymbolPersistsSuffixWithoutDuplicating() thr // The else branch persisted ONLY s1 (the suffix). The dictionary holds // s0, s1 exactly once each. Pre-fix (appendSymbols from // sentMaxSymbolId+1) re-appended s0, giving size 3. - PersistedSymbolDict pd = PersistedSymbolDict.open(Paths.get(sfDir, "default").toString()); - Assert.assertNotNull(pd); - try { - Assert.assertEquals("re-encode suffix must not duplicate the persisted prefix", - 2, pd.size()); - Assert.assertEquals("s0", pd.readLoadedSymbols().getQuick(0)); - Assert.assertEquals("s1", pd.readLoadedSymbols().getQuick(1)); - } finally { - pd.close(); - } + ObjList persisted = ((QwpWebSocketSender) sender).getPersistedSymbolsForTest(); + Assert.assertNotNull(persisted); + Assert.assertEquals("re-encode suffix must not duplicate the persisted prefix", + 2, persisted.size()); + Assert.assertEquals("s0", persisted.getQuick(0)); + Assert.assertEquals("s1", persisted.getQuick(1)); } finally { try { sender.close(); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java index c0d27e09..b534d53a 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java @@ -26,8 +26,8 @@ import io.questdb.client.Sender; import io.questdb.client.cutlass.line.LineSenderException; +import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender; import io.questdb.client.std.ObjList; -import io.questdb.client.cutlass.qwp.client.sf.cursor.PersistedSymbolDict; import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer; import io.questdb.client.test.tools.TestUtils; import org.junit.Assert; @@ -389,18 +389,13 @@ public void testFileModeSplitPersistsDictBeforeAFAILEDPublish() throws Exception // The write-ahead already ran: both of the batch's new symbols are // durable even though the frame that references them never // published. Move the persist after sealAndSwapBuffer and this is 0. - PersistedSymbolDict pd = PersistedSymbolDict.open( - sfDir.resolve("default").toString()); - Assert.assertNotNull(pd); - try { - Assert.assertEquals("the split path must persist its new symbols BEFORE " - + "publishing the frame that references them", 2, pd.size()); - ObjList symbols = pd.readLoadedSymbols(); - Assert.assertEquals("alpha", symbols.getQuick(0)); - Assert.assertEquals("bravo", symbols.getQuick(1)); - } finally { - pd.close(); - } + ObjList persisted = + ((QwpWebSocketSender) sender).getPersistedSymbolsForTest(); + Assert.assertNotNull(persisted); + Assert.assertEquals("the split path must persist its new symbols BEFORE " + + "publishing the frame that references them", 2, persisted.size()); + Assert.assertEquals("alpha", persisted.getQuick(0)); + Assert.assertEquals("bravo", persisted.getQuick(1)); } finally { try { sender.close(); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SlotLockTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SlotLockTest.java index 6643bff8..d38ef9e5 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SlotLockTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SlotLockTest.java @@ -106,8 +106,10 @@ public void testCloseReleasesLock() throws Exception { @Test public void testLogicalLockRemainsContendedAcrossSlotRenameAndRecreate() throws Exception { TestUtils.assertMemoryLeak(() -> { - String slot = parentDir + "/rename"; - String moved = parentDir + "/rename.quarantined"; + // Use the platform-native separator. In particular, this exercises + // acquireLogical with a backslash-only path on Windows. + String slot = Paths.get(parentDir, "rename").toString(); + String moved = Paths.get(parentDir, "rename.quarantined").toString(); assertEquals(0, Files.mkdir(slot, Files.DIR_MODE_DEFAULT)); try (SlotLock ignored = SlotLock.acquireLogical(slot)) { From 12a3aec8a70dd7b04903e709a964fe099063eafa Mon Sep 17 00:00:00 2001 From: glasstiger Date: Thu, 16 Jul 2026 13:51:55 +0100 Subject: [PATCH 75/96] Fix QWP store-and-forward recovery safety Retain source rows when split preflight rejects an oversized table. Allow full dictionary frames to re-anchor ACKed recovery gaps while preserving unacked-gap safety. Bump the segment format and install a legacy-reader rollback barrier. --- .../qwp/client/QwpWebSocketSender.java | 1 - .../client/sf/cursor/CursorSendEngine.java | 8 +++ .../qwp/client/sf/cursor/MmapSegment.java | 71 ++++++++++++++++--- .../sf/cursor/RecoveredFrameAnalysis.java | 39 +++++++--- .../qwp/client/sf/cursor/SegmentRing.java | 29 +++++++- .../qwp/client/DeltaDictRecoveryTest.java | 7 +- .../qwp/client/RecoveryReplayTest.java | 4 +- .../qwp/client/SelfSufficientFramesTest.java | 15 ++++ .../sf/cursor/CursorSendEngineTest.java | 51 ++++++++++++- ...CursorWebSocketSendLoopOrphanTailTest.java | 50 +++++++++++++ ...sorWebSocketSendLoopTornDictGuardTest.java | 3 +- .../qwp/client/sf/cursor/MmapSegmentTest.java | 27 +++++++ .../client/test/impl/SenderPoolSfTest.java | 4 +- 13 files changed, 283 insertions(+), 26 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index 85cf1148..e0dd4da8 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -3664,7 +3664,6 @@ private void flushPendingRowsSplit( splitFrameBodyBytes.getQuick(bodyIdx), simBaseline, currentBatchMaxSymbolId); bodyIdx++; if (messageSize > cap) { - resetTableBuffersAfterFlush(keys); throw new LineSenderException("single table batch too large for server batch cap") .put(" [table=").put(tableName) .put(", messageSize=").put(messageSize) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java index 5fdc129c..38bca269 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java @@ -254,6 +254,14 @@ private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager man PersistedSymbolDict persistedDictInProgress = null; RecoveredFrameAnalysis recoveredFrameAnalysisInProgress = null; try { + // v2 segment payloads may depend on a persisted symbol-dictionary + // prefix. Install the rollback barrier before recovery or any new + // append so a v1-only client can never skip the v2 files, treat the + // slot as empty, and silently restart at FSN 0. Current recovery + // recognizes and skips the reserved guard filenames. + if (!memoryMode) { + SegmentRing.installLegacyReaderBarrier(sfDir); + } // Disk mode: try to recover any *.sfa files left behind by a prior // session before deciding to start fresh. Without this the engine // would create a new sf-initial.sfa at baseSeq=0, overlapping FSNs diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java index 8942cdfb..f74c8655 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java @@ -46,7 +46,7 @@ *

      * On-disk layout — header and frame format: * 

      - *   [u32 magic 'SF01'] [u8 ver=1] [u8 flags=0] [u16 reserved=0]
+ *   [u32 magic 'SF01'] [u8 ver]   [u8 flags=0] [u16 reserved=0]
  *   [u64 baseSeq]       [u64 createdMicros]                       24-byte header
  *   frame, frame, ...                                              each frame:
  *                                                                  [u32 crc32c]
@@ -65,7 +65,8 @@ public final class MmapSegment implements QuietCloseable {
     public static final int FILE_MAGIC = 0x31304653; // 'SF01' little-endian
     public static final int FRAME_HEADER_SIZE = 8;   // u32 crc + u32 payloadLen
     public static final int HEADER_SIZE = 24;
-    public static final byte VERSION = 1;
+    public static final byte LEGACY_VERSION = 1;
+    public static final byte VERSION = 2;
     private static final int[] CRC32C_TABLE = buildCrc32cTable();
     private static final Logger LOG = LoggerFactory.getLogger(MmapSegment.class);
 
@@ -76,6 +77,7 @@ public final class MmapSegment implements QuietCloseable {
     // memory-backed segments — same cursor architecture, no disk involvement.
     // close() and msync() branch on this flag.
     private final boolean memoryBacked;
+    private final byte version;
     // appendCursor: written only by the producer thread, never read by anyone else
     // — it's the reservation cursor. Plain field is fine.
     private long appendCursor;
@@ -106,7 +108,7 @@ public final class MmapSegment implements QuietCloseable {
 
     private MmapSegment(String path, int fd, long mmapAddress, long sizeBytes,
                         long baseSeq, long initialCursor, long frameCount,
-                        boolean memoryBacked, long tornTailBytes) {
+                        boolean memoryBacked, long tornTailBytes, byte version) {
         this.path = path;
         this.fd = fd;
         this.mmapAddress = mmapAddress;
@@ -117,6 +119,7 @@ private MmapSegment(String path, int fd, long mmapAddress, long sizeBytes,
         this.frameCount = frameCount;
         this.memoryBacked = memoryBacked;
         this.tornTailBytes = tornTailBytes;
+        this.version = version;
     }
 
     /**
@@ -164,6 +167,50 @@ public static MmapSegment create(FilesFacade ff, String path, long baseSeq, long
      * per-call {@code byte[]} + native-malloc the way the String overload does.
      */
     public static MmapSegment create(FilesFacade ff, long pathPtr, String displayPath, long baseSeq, long sizeBytes) {
+        return create(ff, pathPtr, displayPath, baseSeq, sizeBytes, VERSION);
+    }
+
+    /**
+     * Creates a tiny v1 segment containing one deliberately non-QWP payload.
+     * Two such files with non-contiguous base sequences form the rollback
+     * barrier installed by {@link SegmentRing}: a v1-only reader opens both
+     * and fails its mandatory FSN-contiguity check before replay. If a process
+     * dies after creating only one guard, that reader still adopts a non-empty
+     * log and sends the invalid one-byte head frame first, so it cannot silently
+     * discard the v2 files and start a fresh slot.
+     */
+    static void createLegacyReaderGuard(String path, long baseSeq) {
+        long pathPtr = FilesFacade.INSTANCE.allocNativePath(path);
+        long payload = 0L;
+        try (MmapSegment segment = create(
+                FilesFacade.INSTANCE,
+                pathPtr,
+                path,
+                baseSeq,
+                HEADER_SIZE + FRAME_HEADER_SIZE + 1L,
+                LEGACY_VERSION
+        )) {
+            payload = Unsafe.malloc(1, MemoryTag.NATIVE_DEFAULT);
+            Unsafe.getUnsafe().putByte(payload, (byte) 0);
+            if (segment.tryAppend(payload, 1) != HEADER_SIZE) {
+                throw new MmapSegmentException("could not append legacy-reader guard frame: " + path);
+            }
+        } finally {
+            if (payload != 0L) {
+                Unsafe.free(payload, 1, MemoryTag.NATIVE_DEFAULT);
+            }
+            FilesFacade.INSTANCE.freeNativePath(pathPtr);
+        }
+    }
+
+    private static MmapSegment create(
+            FilesFacade ff,
+            long pathPtr,
+            String displayPath,
+            long baseSeq,
+            long sizeBytes,
+            byte version
+    ) {
         if (sizeBytes < HEADER_SIZE + FRAME_HEADER_SIZE + 1) {
             throw new IllegalArgumentException(
                     "sizeBytes too small for header + one minimal frame: " + sizeBytes);
@@ -194,12 +241,13 @@ public static MmapSegment create(FilesFacade ff, long pathPtr, String displayPat
             }
             // Header goes straight into the mapping — no separate write syscall.
             Unsafe.getUnsafe().putInt(addr, FILE_MAGIC);
-            Unsafe.getUnsafe().putByte(addr + 4, VERSION);
+            Unsafe.getUnsafe().putByte(addr + 4, version);
             Unsafe.getUnsafe().putByte(addr + 5, (byte) 0); // flags
             Unsafe.getUnsafe().putShort(addr + 6, (short) 0); // reserved
             Unsafe.getUnsafe().putLong(addr + 8, baseSeq);
             Unsafe.getUnsafe().putLong(addr + 16, Os.currentTimeMicros());
-            return new MmapSegment(displayPath, fd, addr, sizeBytes, baseSeq, HEADER_SIZE, 0, false, 0L);
+            return new MmapSegment(displayPath, fd, addr, sizeBytes, baseSeq,
+                    HEADER_SIZE, 0, false, 0L, version);
         } catch (Throwable t) {
             if (addr != Files.FAILED_MMAP_ADDRESS) {
                 Files.munmap(addr, sizeBytes, MemoryTag.MMAP_DEFAULT);
@@ -236,7 +284,8 @@ public static MmapSegment createInMemory(long baseSeq, long sizeBytes) {
             Unsafe.getUnsafe().putShort(addr + 6, (short) 0);
             Unsafe.getUnsafe().putLong(addr + 8, baseSeq);
             Unsafe.getUnsafe().putLong(addr + 16, Os.currentTimeMicros());
-            return new MmapSegment(null, -1, addr, sizeBytes, baseSeq, HEADER_SIZE, 0, true, 0L);
+            return new MmapSegment(null, -1, addr, sizeBytes, baseSeq,
+                    HEADER_SIZE, 0, true, 0L, VERSION);
         } catch (Throwable t) {
             Unsafe.free(addr, sizeBytes, MemoryTag.NATIVE_DEFAULT);
             throw t;
@@ -300,7 +349,7 @@ public static MmapSegment openExisting(FilesFacade ff, String path) {
                         "bad magic in " + path + ": 0x" + Integer.toHexString(magic));
             }
             byte version = Unsafe.getUnsafe().getByte(addr + 4);
-            if (version != VERSION) {
+            if (version != LEGACY_VERSION && version != VERSION) {
                 throw new MmapSegmentException("unsupported version in " + path + ": " + version);
             }
             long baseSeq = Unsafe.getUnsafe().getLong(addr + 8);
@@ -326,7 +375,8 @@ public static MmapSegment openExisting(FilesFacade ff, String path) {
                                 + "Investigate disk health or unexpected writer crash.",
                         path, tornTail, lastGood, fileSize, count);
             }
-            return new MmapSegment(path, fd, addr, fileSize, baseSeq, lastGood, count, false, tornTail);
+            return new MmapSegment(path, fd, addr, fileSize, baseSeq,
+                    lastGood, count, false, tornTail, version);
         } catch (Throwable t) {
             if (addr != Files.FAILED_MMAP_ADDRESS) {
                 Files.munmap(addr, fileSize, MemoryTag.MMAP_DEFAULT);
@@ -440,6 +490,11 @@ public long sizeBytes() {
         return sizeBytes;
     }
 
+    /** On-disk format version read from or written to this segment. */
+    public byte version() {
+        return version;
+    }
+
     /**
      * Appends one frame: writes {@code [crc32c | u32 payloadLen | payload]}
      * starting at the current append cursor, then advances both cursors
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/RecoveredFrameAnalysis.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/RecoveredFrameAnalysis.java
index c9650cfd..386d0f1e 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/RecoveredFrameAnalysis.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/RecoveredFrameAnalysis.java
@@ -41,6 +41,7 @@
 final class RecoveredFrameAnalysis implements QuietCloseable {
 
     private static final int MAX_RAW_BYTES = Integer.MAX_VALUE - 8;
+    private final long ackedFsn;
     private final int baseline;
     private long committedBoundaryFsn = -1L;
     private long committedCoverage;
@@ -51,6 +52,7 @@ final class RecoveredFrameAnalysis implements QuietCloseable {
     private int committedRawLen;
     private long framesVisited;
     private boolean runningGap;
+    private boolean runningUnackedGap;
     private long runningMaxDeltaEnd;
     private long runningMaxDeltaStart;
     private long rawAddr;
@@ -59,8 +61,9 @@ final class RecoveredFrameAnalysis implements QuietCloseable {
     private int runningRawLen;
     private long runningCoverage;
 
-    RecoveredFrameAnalysis(int baseline) {
+    RecoveredFrameAnalysis(int baseline, long ackedFsn) {
         this.baseline = baseline;
+        this.ackedFsn = ackedFsn;
         this.runningCoverage = baseline;
         this.committedCoverage = baseline;
     }
@@ -74,7 +77,7 @@ void accept(long fsn, long payload, int payloadLen) {
                 ? Unsafe.getUnsafe().getByte(payload + QwpConstants.HEADER_OFFSET_FLAGS)
                 : 0;
         if (isQwp && (flags & QwpConstants.FLAG_DELTA_SYMBOL_DICT) != 0) {
-            foldDelta(payload + QwpConstants.HEADER_SIZE, payload + payloadLen);
+            foldDelta(fsn, payload + QwpConstants.HEADER_SIZE, payload + payloadLen);
         }
 
         // Only a positively identified deferred QWP frame can belong to an
@@ -179,10 +182,10 @@ private void appendRaw(long entryStart, int entryLen) {
         runningRawCount++;
     }
 
-    private void foldDelta(long p, long limit) {
+    private void foldDelta(long fsn, long p, long limit) {
         long encodedStart = readVarint(p, limit);
         if (encodedStart < 0L) {
-            runningGap = true;
+            markGap(fsn);
             return;
         }
         int startLen = (int) (encodedStart & 7L);
@@ -194,7 +197,7 @@ private void foldDelta(long p, long limit) {
 
         long encodedCount = readVarint(p, limit);
         if (encodedCount < 0L) {
-            runningGap = true;
+            markGap(fsn);
             return;
         }
         int countLen = (int) (encodedCount & 7L);
@@ -207,10 +210,21 @@ private void foldDelta(long p, long limit) {
             runningMaxDeltaEnd = deltaEnd;
         }
         if (runningGap) {
-            return;
+            // A full dictionary is a new self-sufficient epoch, but it may only
+            // repair a gap that is entirely behind the durable ACK watermark.
+            // If any gapped frame will replay first, accepting this reset would
+            // hide the unsafe wire-order gap and let the server observe missing
+            // ids before it reaches the full frame.
+            if (deltaStart != 0L || runningUnackedGap) {
+                return;
+            }
+            runningGap = false;
+            runningCoverage = baseline;
+            runningRawLen = 0;
+            runningRawCount = 0;
         }
         if (deltaStart > runningCoverage) {
-            runningGap = true;
+            markGap(fsn);
             return;
         }
 
@@ -219,14 +233,14 @@ private void foldDelta(long p, long limit) {
             long entryStart = p;
             long encodedLen = readVarint(p, limit);
             if (encodedLen < 0L) {
-                runningGap = true;
+                markGap(fsn);
                 return;
             }
             int varintLen = (int) (encodedLen & 7L);
             long symbolLen = encodedLen >>> 3;
             p += varintLen;
             if (symbolLen > limit - p) {
-                runningGap = true;
+                markGap(fsn);
                 return;
             }
             p += symbolLen;
@@ -239,6 +253,13 @@ private void foldDelta(long p, long limit) {
         }
     }
 
+    private void markGap(long fsn) {
+        runningGap = true;
+        if (fsn > ackedFsn) {
+            runningUnackedGap = true;
+        }
+    }
+
     /**
      * Returns {@code (value << 3) | encodedByteCount}, or {@code -1} for an
      * unterminated/oversized 32-bit protocol varint.
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java
index 8e596a01..5ba787f3 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java
@@ -62,6 +62,8 @@ public final class SegmentRing implements QuietCloseable {
     public static final long BACKPRESSURE_NO_SPARE = -1L;
     /** Sentinel: append failed because the payload doesn't fit in a fresh segment. */
     public static final long PAYLOAD_TOO_LARGE = -2L;
+    static final String LEGACY_READER_GUARD_A = ".qwp-v2-guard-a.sfa";
+    static final String LEGACY_READER_GUARD_B = ".qwp-v2-guard-b.sfa";
     private static final Logger LOG = LoggerFactory.getLogger(SegmentRing.class);
     // Tally of baseSeq comparisons performed by sortByBaseSeq across every
     // openExisting() recovery on this JVM. Used by SegmentRingTest to
@@ -178,7 +180,9 @@ public static SegmentRing openExisting(String sfDir, long maxBytesPerSegment) {
                 int rc = 1;
                 while (rc > 0) {
                     String name = Files.utf8ToString(Files.findName(find));
-                    if (name != null && name.endsWith(".sfa")) {
+                    if (name != null
+                            && name.endsWith(".sfa")
+                            && !isLegacyReaderGuard(name)) {
                         String path = sfDir + "/" + name;
                         MmapSegment seg = null;
                         try {
@@ -325,6 +329,23 @@ public long ackedFsn() {
         return ackedFsn;
     }
 
+    /**
+     * Installs an on-disk rollback barrier before any v2 segment can be
+     * created or appended. The guards are valid v1 segments so a legacy
+     * reader cannot skip them as an unknown format. Their deliberately
+     * non-contiguous FSN ranges make its recovery fail closed before replay;
+     * current readers recognize the reserved names and omit them from the
+     * logical ring.
+     */
+    static void installLegacyReaderBarrier(String sfDir) {
+        MmapSegment.createLegacyReaderGuard(
+                sfDir + '/' + LEGACY_READER_GUARD_A,
+                Long.MAX_VALUE - 4L);
+        MmapSegment.createLegacyReaderGuard(
+                sfDir + '/' + LEGACY_READER_GUARD_B,
+                Long.MAX_VALUE - 1L);
+    }
+
     /**
      * I/O thread (or anyone tracking ACK) advances the ACK cursor. {@code seq}
      * is cumulative -- the server has confirmed every FSN up to and including
@@ -586,7 +607,7 @@ public synchronized long collectReplaySymbolsAbove(
      * active segment. The returned native suffix remains owned by the caller.
      */
     synchronized RecoveredFrameAnalysis analyzeRecovery(int symbolBaseline) {
-        RecoveredFrameAnalysis analysis = new RecoveredFrameAnalysis(symbolBaseline);
+        RecoveredFrameAnalysis analysis = new RecoveredFrameAnalysis(symbolBaseline, ackedFsn);
         try {
             for (int i = 0, n = sealedSegments.size(); i < n; i++) {
                 sealedSegments.get(i).scanRecovery(analysis);
@@ -859,6 +880,10 @@ private static void sortByBaseSeq(ObjList list, int lo, int hi) {
         }
     }
 
+    private static boolean isLegacyReaderGuard(String name) {
+        return LEGACY_READER_GUARD_A.equals(name) || LEGACY_READER_GUARD_B.equals(name);
+    }
+
     private static void swap(ObjList list, int i, int j) {
         if (i == j) return;
         MmapSegment tmp = list.get(i);
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java
index 14d6c846..10d4ef32 100644
--- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java
@@ -414,7 +414,8 @@ private static int countSegmentFiles(java.nio.file.Path dir) {
         int n = 0;
         if (files != null) {
             for (java.io.File f : files) {
-                if (f.getName().endsWith(".sfa")) {
+                if (f.getName().endsWith(".sfa")
+                        && !f.getName().startsWith(".qwp-v2-guard-")) {
                     n++;
                 }
             }
@@ -1495,7 +1496,9 @@ private static boolean hasSegmentFile(java.nio.file.Path dir) {
         java.io.File[] files = dir.toFile().listFiles();
         if (files != null) {
             for (java.io.File f : files) {
-                if (f.getName().endsWith(".sfa") && f.length() > 0) {
+                if (f.getName().endsWith(".sfa")
+                        && !f.getName().startsWith(".qwp-v2-guard-")
+                        && f.length() > 0) {
                     return true;
                 }
             }
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/RecoveryReplayTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/RecoveryReplayTest.java
index e868dafc..274d39bf 100644
--- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/RecoveryReplayTest.java
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/RecoveryReplayTest.java
@@ -154,7 +154,9 @@ private static int countPopulatedSegmentFiles(String dir) {
             int rc = 1;
             while (rc > 0) {
                 String name = Files.utf8ToString(Files.findName(find));
-                if (name != null && name.endsWith(".sfa")) {
+                if (name != null
+                        && name.endsWith(".sfa")
+                        && !name.startsWith(".qwp-v2-guard-")) {
                     try {
                         try (MmapSegment seg = MmapSegment.openExisting(dir + "/" + name)) {
                             if (seg.frameCount() > 0) n++;
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java
index b534d53a..0c88bc11 100644
--- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java
@@ -316,6 +316,21 @@ public void testOversizedTableSplitStrandsNothing() throws Exception {
                         Assert.assertTrue(e.getMessage(),
                                 e.getMessage().contains("too large for server batch cap"));
                     }
+                    // Sender.flush() is retryable: a rejected flush must leave the
+                    // source rows intact. Repeating it under the same cap therefore
+                    // has to reject the same batch again. Before the fix, the first
+                    // failure reset every table buffer and this second flush was a
+                    // silent no-op -- the caller's rows had been lost.
+                    try {
+                        sender.flush();
+                        Assert.fail("retrying the retained oversized batch must throw again");
+                    } catch (LineSenderException e) {
+                        Assert.assertTrue(e.getMessage(),
+                                e.getMessage().contains("too large for server batch cap"));
+                    }
+                    // Explicit reset is the caller-controlled discard boundary and
+                    // prevents close() from making a third retry in this test.
+                    sender.reset();
                     // close() drains the ring: pre-fix, the stranded "small" frame
                     // would be sent (and committed) here.
                 }
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineTest.java
index 0e874ba5..0bdddb3e 100644
--- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineTest.java
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineTest.java
@@ -298,6 +298,51 @@ public void testRecoveryAdvancesAckedFsnPastWatermark() throws Exception {
         });
     }
 
+    @Test
+    public void testV2DiskSlotForcesLegacyReaderToFailClosed() throws Exception {
+        TestUtils.assertMemoryLeak(() -> {
+            long buf = Unsafe.malloc(16, MemoryTag.NATIVE_DEFAULT);
+            try {
+                try (CursorSendEngine engine = new CursorSendEngine(tmpDir, 4096)) {
+                    assertEquals(0L, engine.appendBlocking(buf, 16));
+                }
+
+                // Real segments carry v2: their payload may depend on the
+                // persisted dictionary and must never be replayed by a v1-only
+                // client as though every frame were self-sufficient.
+                try (MmapSegment data = MmapSegment.openExisting(tmpDir + "/sf-initial.sfa")) {
+                    assertEquals(MmapSegment.VERSION, data.version());
+                    assertEquals(1L, data.frameCount());
+                }
+
+                // A legacy reader rejects the v2 data segment, but accepts both
+                // guards as ordinary non-empty v1 segments. Their ranges cannot
+                // be contiguous, so its mandatory global recovery check throws
+                // before any replay or fresh-slot fallback can happen.
+                try (MmapSegment guardA = MmapSegment.openExisting(
+                        tmpDir + "/.qwp-v2-guard-a.sfa");
+                     MmapSegment guardB = MmapSegment.openExisting(
+                             tmpDir + "/.qwp-v2-guard-b.sfa")) {
+                    assertEquals(MmapSegment.LEGACY_VERSION, guardA.version());
+                    assertEquals(MmapSegment.LEGACY_VERSION, guardB.version());
+                    assertEquals(1L, guardA.frameCount());
+                    assertEquals(1L, guardB.frameCount());
+                    assertTrue("legacy guard ranges must force an FSN gap",
+                            guardA.baseSeq() + guardA.frameCount() != guardB.baseSeq());
+                }
+
+                // Current recovery skips the reserved guards and adopts only
+                // the real v2 log.
+                try (CursorSendEngine recovered = new CursorSendEngine(tmpDir, 4096)) {
+                    assertTrue(recovered.wasRecoveredFromDisk());
+                    assertEquals(0L, recovered.publishedFsn());
+                }
+            } finally {
+                Unsafe.free(buf, 16, MemoryTag.NATIVE_DEFAULT);
+            }
+        });
+    }
+
     @Test
     public void testRecoveryIgnoresWatermarkAbovePublishedFsn() throws Exception {
         // Pin the corruption defence: a watermark higher than what the
@@ -413,7 +458,11 @@ public void testRestartIntoNonEmptySfDirContinuesFsnSequence() throws Exception
                     int rc = 1;
                     while (rc > 0) {
                         String name = Files.utf8ToString(Files.findName(find));
-                        if (name != null && name.endsWith(".sfa")) sfaCount++;
+                        if (name != null
+                                && name.endsWith(".sfa")
+                                && !name.startsWith(".qwp-v2-guard-")) {
+                            sfaCount++;
+                        }
                         rc = Files.findNext(find);
                     }
                 } finally {
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopOrphanTailTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopOrphanTailTest.java
index 52235637..c96bfefe 100644
--- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopOrphanTailTest.java
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopOrphanTailTest.java
@@ -28,6 +28,7 @@
 import io.questdb.client.cutlass.http.client.WebSocketClient;
 import io.questdb.client.cutlass.http.client.WebSocketFrameHandler;
 import io.questdb.client.cutlass.qwp.client.WebSocketResponse;
+import io.questdb.client.cutlass.qwp.client.sf.cursor.AckWatermark;
 import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine;
 import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorWebSocketSendLoop;
 import io.questdb.client.network.PlainSocketFactory;
@@ -398,6 +399,55 @@ public void testRecoveryScansFramesOnceAndReusesCachedSymbolSuffix() throws Exce
         });
     }
 
+    @Test
+    public void testSelfSufficientFrameRepairsAckedRecoveryGap() throws Exception {
+        // fsn 0 models the tail of an old delta epoch whose registering frames
+        // have already been trimmed: with an empty persisted dictionary its
+        // deltaStart=1 is a gap. It is durably ACKed, so it will not replay.
+        // fsn 1 starts a new, self-sufficient epoch from id 0 and is the first
+        // frame that WILL replay. Recovery must use that full frame as the new
+        // source of truth instead of permanently latching the earlier gap.
+        TestUtils.assertMemoryLeak(() -> {
+            try (CursorSendEngine engine = newEngine()) {
+                appendDeltaFrame(engine, false, 1, 0);
+                appendDeltaSymbolFrame(engine, 0, 'a');
+            }
+            try (AckWatermark watermark = AckWatermark.open(tmpDir)) {
+                assertNotNull(watermark);
+                watermark.write(0L);
+            }
+
+            try (CursorSendEngine engine = newEngine()) {
+                assertEquals(0L, engine.ackedFsn());
+                ObjList recovered = new ObjList<>();
+                assertEquals("the full frame must re-anchor replay coverage",
+                        1L, engine.collectReplaySymbolsAbove(0, recovered));
+                assertEquals(1, recovered.size());
+                assertEquals("a", recovered.getQuick(0));
+            }
+        });
+    }
+
+    @Test
+    public void testSelfSufficientFrameCannotHideUnackedRecoveryGap() throws Exception {
+        // Safety twin: when the gapped frame itself is unacked, it reaches the
+        // wire before the later full frame. Recovery must keep the gap latched
+        // and quarantine rather than pretending the later reset repairs the
+        // invalid replay order.
+        TestUtils.assertMemoryLeak(() -> {
+            try (CursorSendEngine engine = newEngine()) {
+                appendDeltaFrame(engine, false, 1, 0);
+                appendDeltaSymbolFrame(engine, 0, 'a');
+            }
+            try (CursorSendEngine engine = newEngine()) {
+                ObjList recovered = new ObjList<>();
+                assertEquals("an unacked gap remains unreplayable",
+                        -1L, engine.collectReplaySymbolsAbove(0, recovered));
+                assertEquals(0, recovered.size());
+            }
+        });
+    }
+
     // ---------------------------------------------------------------------
     // harness
     // ---------------------------------------------------------------------
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopTornDictGuardTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopTornDictGuardTest.java
index 7dabc741..5a666dc7 100644
--- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopTornDictGuardTest.java
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopTornDictGuardTest.java
@@ -119,7 +119,8 @@ private static int countSegmentFiles(Path dir) {
         int n = 0;
         if (files != null) {
             for (File f : files) {
-                if (f.getName().endsWith(".sfa")) {
+                if (f.getName().endsWith(".sfa")
+                        && !f.getName().startsWith(".qwp-v2-guard-")) {
                     n++;
                 }
             }
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/MmapSegmentTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/MmapSegmentTest.java
index 177ee5f6..4707c233 100644
--- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/MmapSegmentTest.java
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/MmapSegmentTest.java
@@ -35,6 +35,8 @@
 import org.junit.Before;
 import org.junit.Test;
 
+import java.io.RandomAccessFile;
+
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertNotEquals;
@@ -113,6 +115,31 @@ public void testCreateAppendCloseReopenScansAllFrames() throws Exception {
         });
     }
 
+    @Test
+    public void testCurrentReaderStillOpensLegacyV1Segment() throws Exception {
+        TestUtils.assertMemoryLeak(() -> {
+            String path = tmpDir + "/seg-v1.sfa";
+            long buf = Unsafe.malloc(8, MemoryTag.NATIVE_DEFAULT);
+            try {
+                try (MmapSegment seg = MmapSegment.create(path, 7L, 4096)) {
+                    assertEquals(MmapSegment.VERSION, seg.version());
+                    assertTrue(seg.tryAppend(buf, 8) >= 0L);
+                }
+                try (RandomAccessFile file = new RandomAccessFile(path, "rw")) {
+                    file.seek(4L);
+                    file.writeByte(MmapSegment.LEGACY_VERSION);
+                }
+                try (MmapSegment legacy = MmapSegment.openExisting(path)) {
+                    assertEquals(MmapSegment.LEGACY_VERSION, legacy.version());
+                    assertEquals(7L, legacy.baseSeq());
+                    assertEquals(1L, legacy.frameCount());
+                }
+            } finally {
+                Unsafe.free(buf, 8, MemoryTag.NATIVE_DEFAULT);
+            }
+        });
+    }
+
     @Test
     public void testCreateFailsCleanlyWhenAllocateReturnsFalse() throws Exception {
         TestUtils.assertMemoryLeak(() -> {
diff --git a/core/src/test/java/io/questdb/client/test/impl/SenderPoolSfTest.java b/core/src/test/java/io/questdb/client/test/impl/SenderPoolSfTest.java
index cfef2feb..2f213c3a 100644
--- a/core/src/test/java/io/questdb/client/test/impl/SenderPoolSfTest.java
+++ b/core/src/test/java/io/questdb/client/test/impl/SenderPoolSfTest.java
@@ -1847,7 +1847,9 @@ private static boolean hasSegmentFile(String slotPath) {
             while (rc > 0) {
                 String name = Files.utf8ToString(Files.findName(find));
                 rc = Files.findNext(find);
-                if (name != null && name.endsWith(".sfa")) {
+                if (name != null
+                        && name.endsWith(".sfa")
+                        && !name.startsWith(".qwp-v2-guard-")) {
                     return true;
                 }
             }

From 350e4a397c7134a817b013fb7b19265141a410fe Mon Sep 17 00:00:00 2001
From: glasstiger 
Date: Thu, 16 Jul 2026 14:17:23 +0100
Subject: [PATCH 76/96] Restore QWP API compatibility

Restore the previous QwpWebSocketSender connect overload and the
BackgroundDrainer constructor. Delegate both to the new catch-up dwell
defaults.

Use a saturating conversion for catch-up dwell milliseconds so large
valid values cannot wrap negative and quarantine slots prematurely.
---
 .../qwp/client/QwpWebSocketSender.java        | 39 ++++++++
 .../client/sf/cursor/BackgroundDrainer.java   | 25 +++++
 .../sf/cursor/CursorWebSocketSendLoop.java    |  7 +-
 .../client/QwpPublicApiCompatibilityTest.java | 96 +++++++++++++++++++
 ...WebSocketSendLoopCatchUpAlignmentTest.java | 27 ++++++
 5 files changed, 193 insertions(+), 1 deletion(-)
 create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpPublicApiCompatibilityTest.java

diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java
index e0dd4da8..f24b1ab1 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java
@@ -702,6 +702,45 @@ public static QwpWebSocketSender connect(
                 CursorWebSocketSendLoop.DEFAULT_CATCHUP_CAP_GAP_MIN_ESCALATION_WINDOW_MILLIS);
     }
 
+    /**
+     * Compatibility overload retained for callers compiled before the
+     * catch-up cap-gap dwell was added to the master signature.
+     */
+    public static QwpWebSocketSender connect(
+            List endpoints,
+            ClientTlsConfiguration tlsConfig,
+            int autoFlushRows,
+            int autoFlushBytes,
+            long autoFlushIntervalNanos,
+            String authorizationHeader,
+            boolean requestDurableAck,
+            CursorSendEngine cursorEngine,
+            long closeFlushTimeoutMillis,
+            long reconnectMaxDurationMillis,
+            long reconnectInitialBackoffMillis,
+            long reconnectMaxBackoffMillis,
+            Sender.InitialConnectMode initialConnectMode,
+            SenderErrorHandler errorHandler,
+            int errorInboxCapacity,
+            long durableAckKeepaliveIntervalMillis,
+            long authTimeoutMs,
+            int connectTimeoutMs,
+            SenderConnectionListener connectionListener,
+            int connectionListenerInboxCapacity,
+            int maxFrameRejections,
+            long poisonMinEscalationWindowMillis
+    ) {
+        return connect(endpoints, tlsConfig, autoFlushRows, autoFlushBytes,
+                autoFlushIntervalNanos, authorizationHeader, requestDurableAck,
+                cursorEngine, closeFlushTimeoutMillis, reconnectMaxDurationMillis,
+                reconnectInitialBackoffMillis, reconnectMaxBackoffMillis,
+                initialConnectMode, errorHandler, errorInboxCapacity,
+                durableAckKeepaliveIntervalMillis, authTimeoutMs, connectTimeoutMs,
+                connectionListener, connectionListenerInboxCapacity,
+                maxFrameRejections, poisonMinEscalationWindowMillis,
+                CursorWebSocketSendLoop.DEFAULT_CATCHUP_CAP_GAP_MIN_ESCALATION_WINDOW_MILLIS);
+    }
+
     /**
      * Master connect overload — also accepts the poison-frame detector
      * threshold ({@code max_frame_rejections}): consecutive server-active
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/BackgroundDrainer.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/BackgroundDrainer.java
index 8e287b4a..38b6b036 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/BackgroundDrainer.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/BackgroundDrainer.java
@@ -160,6 +160,31 @@ public BackgroundDrainer(
                 CursorWebSocketSendLoop.DEFAULT_CATCHUP_CAP_GAP_MIN_ESCALATION_WINDOW_MILLIS);
     }
 
+    /**
+     * Compatibility constructor retained for callers compiled before the
+     * catch-up cap-gap dwell was added to the master signature.
+     */
+    public BackgroundDrainer(
+            String slotPath,
+            long segmentSizeBytes,
+            long sfMaxTotalBytes,
+            CursorWebSocketSendLoop.ReconnectFactory clientFactory,
+            long reconnectMaxDurationMillis,
+            long reconnectInitialBackoffMillis,
+            long reconnectMaxBackoffMillis,
+            boolean requestDurableAck,
+            long durableAckKeepaliveIntervalMillis,
+            int maxHeadFrameRejections,
+            long poisonMinEscalationWindowMillis
+    ) {
+        this(slotPath, segmentSizeBytes, sfMaxTotalBytes, clientFactory,
+                reconnectMaxDurationMillis, reconnectInitialBackoffMillis,
+                reconnectMaxBackoffMillis, requestDurableAck,
+                durableAckKeepaliveIntervalMillis, maxHeadFrameRejections,
+                poisonMinEscalationWindowMillis,
+                CursorWebSocketSendLoop.DEFAULT_CATCHUP_CAP_GAP_MIN_ESCALATION_WINDOW_MILLIS);
+    }
+
     /**
      * Master constructor — also accepts the poison-frame detector threshold
      * ({@code max_frame_rejections}) forwarded to the drain loop's
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java
index 5c59823c..1f1dc034 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java
@@ -53,6 +53,7 @@
 import java.util.Arrays;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.concurrent.locks.LockSupport;
 
@@ -682,8 +683,12 @@ public CursorWebSocketSendLoop(WebSocketClient client, CursorSendEngine engine,
                     "catchUpCapGapMinEscalationWindowMillis must be >= 0: "
                             + catchUpCapGapMinEscalationWindowMillis);
         }
+        // TimeUnit conversion saturates at Long.MAX_VALUE. A raw multiply
+        // wraps large valid millisecond values negative, which makes the
+        // elapsed-time gate appear satisfied as soon as the strike threshold
+        // is reached and can quarantine a recoverable orphan slot prematurely.
         this.catchUpCapGapMinEscalationWindowNanos =
-                catchUpCapGapMinEscalationWindowMillis * 1_000_000L;
+                TimeUnit.MILLISECONDS.toNanos(catchUpCapGapMinEscalationWindowMillis);
         if (catchUpCapGapPolicy == null) {
             throw new IllegalArgumentException("catchUpCapGapPolicy must be non-null");
         }
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpPublicApiCompatibilityTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpPublicApiCompatibilityTest.java
new file mode 100644
index 00000000..d2770145
--- /dev/null
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpPublicApiCompatibilityTest.java
@@ -0,0 +1,96 @@
+/*******************************************************************************
+ *     ___                  _   ____  ____
+ *    / _ \ _   _  ___  ___| |_|  _ \| __ )
+ *   | | | | | | |/ _ \/ __| __| | | |  _ \
+ *   | |_| | |_| |  __/\__ \ |_| |_| | |_) |
+ *    \__\_\\__,_|\___||___/\__|____/|____/
+ *
+ *  Copyright (c) 2014-2019 Appsicle
+ *  Copyright (c) 2019-2026 QuestDB
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ ******************************************************************************/
+
+package io.questdb.client.test.cutlass.qwp.client;
+
+import io.questdb.client.ClientTlsConfiguration;
+import io.questdb.client.Sender;
+import io.questdb.client.SenderConnectionListener;
+import io.questdb.client.SenderErrorHandler;
+import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender;
+import io.questdb.client.cutlass.qwp.client.sf.cursor.BackgroundDrainer;
+import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine;
+import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorWebSocketSendLoop;
+import org.junit.Test;
+
+import java.lang.reflect.Constructor;
+import java.lang.reflect.Method;
+import java.lang.reflect.Modifier;
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+public class QwpPublicApiCompatibilityTest {
+
+    @Test
+    public void testBackgroundDrainerLegacyConstructorDescriptorIsPreserved() throws Exception {
+        Constructor constructor = BackgroundDrainer.class.getConstructor(
+                String.class,
+                long.class,
+                long.class,
+                CursorWebSocketSendLoop.ReconnectFactory.class,
+                long.class,
+                long.class,
+                long.class,
+                boolean.class,
+                long.class,
+                int.class,
+                long.class
+        );
+        assertTrue(Modifier.isPublic(constructor.getModifiers()));
+    }
+
+    @Test
+    public void testQwpWebSocketSenderLegacyConnectDescriptorIsPreserved() throws Exception {
+        Method method = QwpWebSocketSender.class.getMethod(
+                "connect",
+                List.class,
+                ClientTlsConfiguration.class,
+                int.class,
+                int.class,
+                long.class,
+                String.class,
+                boolean.class,
+                CursorSendEngine.class,
+                long.class,
+                long.class,
+                long.class,
+                long.class,
+                Sender.InitialConnectMode.class,
+                SenderErrorHandler.class,
+                int.class,
+                long.class,
+                long.class,
+                int.class,
+                SenderConnectionListener.class,
+                int.class,
+                int.class,
+                long.class
+        );
+        assertEquals(QwpWebSocketSender.class, method.getReturnType());
+        assertTrue(Modifier.isPublic(method.getModifiers()));
+        assertTrue(Modifier.isStatic(method.getModifiers()));
+    }
+}
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java
index 34ef3c4a..acc8102a 100644
--- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java
@@ -397,6 +397,33 @@ public void testCatchUpCapGapStrikesAloneDoNotLatchWithinTheEscalationWindow() t
         });
     }
 
+    @Test
+    public void testCatchUpCapGapDwellConversionSaturatesInsteadOfOverflowing() throws Exception {
+        TestUtils.assertMemoryLeak(() -> {
+            long maxExactMillis = Long.MAX_VALUE / 1_000_000L;
+            try (CursorSendEngine engine = newEngine()) {
+                CursorWebSocketSendLoop exactLoop = newLoop(
+                        engine, new CatchUpCapturingClient(0), maxExactMillis);
+                try {
+                    assertEquals(maxExactMillis * 1_000_000L,
+                            readLong(exactLoop, "catchUpCapGapMinEscalationWindowNanos"));
+                } finally {
+                    exactLoop.close();
+                }
+
+                CursorWebSocketSendLoop saturatedLoop = newLoop(
+                        engine, new CatchUpCapturingClient(0), maxExactMillis + 1L);
+                try {
+                    assertEquals("an oversized dwell must become effectively infinite, not negative",
+                            Long.MAX_VALUE,
+                            readLong(saturatedLoop, "catchUpCapGapMinEscalationWindowNanos"));
+                } finally {
+                    saturatedLoop.close();
+                }
+            }
+        });
+    }
+
     @Test
     public void testCapGapEpisodeWithANegativeAnchorStillEscalates() throws Exception {
         // A cap-gap episode anchored at a NEGATIVE nanoTime instant must escalate like any

From a341a9b70f850335d00f55f3a3b70c9cfcbed019 Mon Sep 17 00:00:00 2001
From: glasstiger 
Date: Thu, 16 Jul 2026 14:59:28 +0100
Subject: [PATCH 77/96] Optimize QWP symbol dictionary recovery

---
 .../qwp/client/QwpWebSocketSender.java        |  19 +-
 .../client/sf/cursor/CursorSendEngine.java    |  24 ++
 .../sf/cursor/CursorWebSocketSendLoop.java    |  91 +++--
 .../qwp/client/sf/cursor/MmapSegment.java     |  45 ++-
 .../client/sf/cursor/PersistedSymbolDict.java | 333 +++++++++++-------
 .../sf/cursor/RecoveredFrameAnalysis.java     |  28 +-
 ...WebSocketSendLoopCatchUpAlignmentTest.java |   1 +
 ...CursorWebSocketSendLoopMirrorLeakTest.java |  41 ++-
 ...CursorWebSocketSendLoopOrphanTailTest.java |  31 ++
 .../qwp/client/sf/cursor/MmapSegmentTest.java |   2 +
 .../sf/cursor/PersistedSymbolDictTest.java    |   7 +
 11 files changed, 410 insertions(+), 212 deletions(-)

diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java
index f24b1ab1..1fba1452 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java
@@ -3942,7 +3942,7 @@ private void resetSymbolDictStateForNewConnection() {
      * 
        
      *   
      1. the slot's persisted {@code .symbol-dict} -- its intact prefix; then
        2. 
      *   
      2. the surviving frames' OWN delta sections, for every id above that prefix
-     *       ({@link CursorSendEngine#collectReplaySymbolsAbove}).
        3. 
+     *       ({@link CursorSendEngine#addRecoveredSymbolsTo}).
      * 
      
      * Those are exactly the two sources, in exactly the order, that the send loop's mirror
      * is built from: its constructor seeds {@code sentDictCount} from the same dictionary,
@@ -3976,7 +3976,7 @@ private void resetSymbolDictStateForNewConnection() {
      * What still fails clean. A genuine GAP: the ids below a surviving frame's delta
      * start were introduced by frames that were acked and TRIMMED away, so they lived only in
      * the lost dictionary and nothing can rebuild them.
-     * {@code collectReplaySymbolsAbove} returns -1 for that and we throw. It is the same
+     * {@code addRecoveredSymbolsTo} returns -1 for that and we throw. It is the same
      * condition the send loop's replay guard ({@code deltaStart > sentDictCount}) trips on, so
      * producer and drainer now agree on exactly which slots are recoverable, instead of the
      * producer rejecting slots the drainer drains.
@@ -3990,17 +3990,13 @@ private void seedGlobalDictionaryFromPersisted(PersistedSymbolDict pd) {
         //    mirror also seeds sentDictCount from.
         int baseline = 0;
         if (pd != null && pd.size() > 0) {
-            ObjList persisted = pd.readLoadedSymbols();
-            for (int i = 0, n = persisted.size(); i < n; i++) {
-                globalSymbolDictionary.addRecoveredSymbol(persisted.getQuick(i));
-            }
+            pd.addLoadedSymbolsTo(globalSymbolDictionary);
             baseline = globalSymbolDictionary.size();
         }
         // 2. Everything the surviving frames define above that prefix, straight out of their
         //    own delta sections -- the same bytes, in the same order, accumulateSentDict will
         //    feed the mirror as those frames go back on the wire.
-        ObjList fromFrames = new ObjList<>();
-        long coverage = cursorEngine.collectReplaySymbolsAbove(baseline, fromFrames);
+        long coverage = cursorEngine.addRecoveredSymbolsTo(baseline, globalSymbolDictionary);
         if (coverage < 0) {
             // A gap: the surviving frames reference ids below their own delta start,
             // introduced by frames since acked and trimmed away, and the persisted
@@ -4013,8 +4009,8 @@ private void seedGlobalDictionaryFromPersisted(PersistedSymbolDict pd) {
             // "resend required" for delivered data AND -- because such a slot is fully
             // drained -- let build()'s connect-path close unlink the (already-delivered)
             // bytes the quarantine claims to preserve. So DON'T throw: seed the intact
-            // prefix only, leaving fromFrames unused exactly as the send loop's mirror
-            // does on a -1 (it adds nothing), so the producer baseline and the mirror's
+            // prefix only; addRecoveredSymbolsTo adds nothing on a -1 exactly as the
+            // send loop's mirror does, so the producer baseline and the mirror's
             // sentDictCount still agree by construction. The producer resumes above the
             // prefix and the fully-drained slot is cleaned up on close.
             long ackedFsn = cursorEngine.ackedFsn();
@@ -4040,9 +4036,6 @@ private void seedGlobalDictionaryFromPersisted(PersistedSymbolDict pd) {
                             + "holds them; the recovered dictionary holds only "
                             + (pd == null ? 0 : pd.size()) + " id(s) -- resend the affected data");
         }
-        for (int i = 0, n = fromFrames.size(); i < n; i++) {
-            globalSymbolDictionary.addRecoveredSymbol(fromFrames.getQuick(i));
-        }
         // Producer baseline == the coverage the replay will establish == the mirror's
         // sentDictCount once those frames have gone out. The first new frame therefore
         // starts its delta exactly at the tip, and the replay guard passes.
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java
index 38bca269..597c2bc7 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java
@@ -24,6 +24,7 @@
 
 package io.questdb.client.cutlass.qwp.client.sf.cursor;
 
+import io.questdb.client.cutlass.qwp.client.GlobalSymbolDictionary;
 import io.questdb.client.cutlass.qwp.protocol.QwpConstants;
 import io.questdb.client.std.Compat;
 import io.questdb.client.std.Files;
@@ -760,6 +761,24 @@ public long collectReplaySymbolsAbove(int baseline, ObjList out) {
         );
     }
 
+    /**
+     * Decodes the cached recovery suffix directly into the producer's global
+     * dictionary. Recovery always builds the analysis with the persisted
+     * prefix size as its baseline, so no intermediate cardinality-sized list is
+     * needed on the production path.
+     */
+    public long addRecoveredSymbolsTo(int baseline, GlobalSymbolDictionary target) {
+        if (recoveredFrameAnalysis == null) {
+            return baseline;
+        }
+        RecoveredFrameAnalysis analysis = checkedRecoveryAnalysis(baseline);
+        long coverage = analysis.coverage();
+        if (coverage >= 0L) {
+            analysis.addDecodedSymbolsTo(target);
+        }
+        return coverage;
+    }
+
     long recoveredSymbolCoverage(int baseline) {
         return checkedRecoveryAnalysis(baseline).coverage();
     }
@@ -785,6 +804,11 @@ public long recoveryFramesVisited() {
         return recoveredFrameAnalysis == null ? 0L : recoveredFrameAnalysis.framesVisited();
     }
 
+    @TestOnly
+    public long recoverySymbolEntriesVisited() {
+        return recoveredFrameAnalysis == null ? 0L : recoveredFrameAnalysis.symbolEntriesVisited();
+    }
+
     private RecoveredFrameAnalysis checkedRecoveryAnalysis(int baseline) {
         if (recoveredFrameAnalysis == null || recoveredFrameAnalysis.baseline() != baseline) {
             throw new IllegalStateException("recovery symbol baseline mismatch [expected="
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java
index 1f1dc034..3d33bfd0 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java
@@ -299,6 +299,9 @@ public final class CursorWebSocketSendLoop implements QuietCloseable {
     private long sentDictBytesAddr;
     private int sentDictBytesCapacity;
     private int sentDictBytesLen;
+    // False only while an orphan-drainer loop borrows the persisted prefix.
+    // Any growth first performs a copy-on-write; only owned buffers are freed.
+    private boolean sentDictBytesOwned;
     private int sentDictCount;
     // Reusable staging frame for dictionary catch-up chunks. A reconnect may split a
     // large dictionary into many chunks, and every later reconnect repeats that split;
@@ -721,24 +724,17 @@ public CursorWebSocketSendLoop(WebSocketClient client, CursorSendEngine engine,
         // tracks. When the dictionary could not be opened, pd is null, the mirror
         // simply starts empty and grows from the frames themselves.
         PersistedSymbolDict pd = engine.getPersistedSymbolDict();
+        int persistedPrefixLen = 0;
         if (pd != null && pd.size() > 0) {
             int len = pd.loadedEntriesLen();
             if (len > 0) {
-                // COPY the persisted dictionary's loaded-entries buffer into this
-                // loop's own mirror rather than taking ownership of it. The engine
-                // (and its PersistedSymbolDict) OUTLIVES this loop on the orphan
-                // drainer path: BackgroundDrainer builds a fresh send loop per wire
-                // session against the same engine on a durable-ack capability-gap
-                // recycle. A one-shot ownership transfer would leave every loop
-                // after the first with an EMPTY mirror -- it would then send no
-                // reconnect catch-up, and the first replayed delta frame
-                // (deltaStart > 0) would trip the torn-dict guard, falsely
-                // quarantining a healthy slot. Copying keeps the dictionary's
-                // loaded entries intact for the engine's lifetime so every
-                // recycled loop re-seeds; pd.close() (at engine close) frees the
-                // dictionary's copy, this loop frees its own copy on exit.
-                sentDictBytesAddr = Unsafe.malloc(len, MemoryTag.NATIVE_DEFAULT);
-                Unsafe.getUnsafe().copyMemory(pd.loadedEntriesAddr(), sentDictBytesAddr, len);
+                // Seed by reference. The foreground loop takes ownership after
+                // construction succeeds; orphan-drainer loops keep borrowing because
+                // one engine can create several sessions during capability-gap
+                // recycling. A borrowed mirror performs copy-on-write if a recovered
+                // frame suffix must extend it.
+                persistedPrefixLen = len;
+                sentDictBytesAddr = pd.loadedEntriesAddr();
                 sentDictBytesCapacity = len;
                 sentDictBytesLen = len;
                 // Set the count only alongside the bytes so sentDictCount can
@@ -786,16 +782,25 @@ public CursorWebSocketSendLoop(WebSocketClient client, CursorSendEngine engine,
                     }
                 }
             } catch (Throwable t) {
-                if (sentDictBytesAddr != 0) {
-                    Unsafe.free(sentDictBytesAddr, sentDictBytesCapacity, MemoryTag.NATIVE_DEFAULT);
-                    sentDictBytesAddr = 0;
-                    sentDictBytesCapacity = 0;
-                    sentDictBytesLen = 0;
-                    sentDictCount = 0;
-                }
+                releaseSentDictBytes();
                 throw t;
             }
         }
+        // QwpWebSocketSender seeds the producer dictionary before constructing
+        // its one foreground loop, so that loop can own the recovered prefix and
+        // eliminate the engine-side duplicate. BackgroundDrainer must retain the
+        // prefix for later recycled loops and therefore keeps borrowing it.
+        if (persistedPrefixLen > 0 && catchUpCapGapPolicy == CatchUpCapGapPolicy.RETRY_FOREVER) {
+            long persistedPrefixAddr = pd.takeLoadedEntries();
+            if (sentDictBytesOwned) {
+                // Copy-on-grow already produced the combined mirror. Retire the
+                // no-longer-needed persisted prefix after successful construction.
+                Unsafe.free(persistedPrefixAddr, persistedPrefixLen, MemoryTag.NATIVE_DEFAULT);
+            } else {
+                assert persistedPrefixAddr == sentDictBytesAddr;
+                sentDictBytesOwned = true;
+            }
+        }
         this.fsnAtZero = fsnAtZero;
         this.parkNanos = parkNanos;
         this.reconnectFactory = reconnectFactory;
@@ -1131,15 +1136,7 @@ public synchronized void close() {
         // thread may still be mid-send, so touching the mirror here would race.
         // A duplicate close observes sentDictBytesAddr == 0 and skips.
         if (loopNeverRan && sentDictBytesAddr != 0) {
-            Unsafe.free(sentDictBytesAddr, sentDictBytesCapacity, MemoryTag.NATIVE_DEFAULT);
-            sentDictBytesAddr = 0;
-            sentDictBytesCapacity = 0;
-            sentDictBytesLen = 0;
-            // Reset the count alongside the buffer so the mirror stays all-or-
-            // nothing: a hypothetical close()-then-start() (start() has no closed
-            // guard) must not observe a non-zero sentDictCount against a freed
-            // buffer and drive setWireBaselineWithCatchUp into a null-mirror catch-up.
-            sentDictCount = 0;
+            releaseSentDictBytes();
         }
         if (loopNeverRan) {
             freeCatchUpFrameBuffer();
@@ -2048,11 +2045,7 @@ private void ioLoop() {
             // The symbol-dict mirror is I/O-thread-owned; free it here, on the
             // owning thread's exit path, after the last send that could touch it.
             if (sentDictBytesAddr != 0) {
-                Unsafe.free(sentDictBytesAddr, sentDictBytesCapacity, MemoryTag.NATIVE_DEFAULT);
-                sentDictBytesAddr = 0;
-                sentDictBytesCapacity = 0;
-                sentDictBytesLen = 0;
-                sentDictCount = 0; // keep the mirror all-or-nothing (see close())
+                releaseSentDictBytes();
             }
             freeCatchUpFrameBuffer();
             shutdownLatch.countDown();
@@ -2438,10 +2431,34 @@ private void ensureSentDictCapacity(long required) {
         if (newCap > MAX_SENT_DICT_BYTES) {
             newCap = MAX_SENT_DICT_BYTES;
         }
-        sentDictBytesAddr = Unsafe.realloc(sentDictBytesAddr, sentDictBytesCapacity, (int) newCap, MemoryTag.NATIVE_DEFAULT);
+        if (sentDictBytesOwned) {
+            sentDictBytesAddr = Unsafe.realloc(
+                    sentDictBytesAddr,
+                    sentDictBytesCapacity,
+                    (int) newCap,
+                    MemoryTag.NATIVE_DEFAULT);
+        } else {
+            long newAddr = Unsafe.malloc((int) newCap, MemoryTag.NATIVE_DEFAULT);
+            if (sentDictBytesLen > 0) {
+                Unsafe.getUnsafe().copyMemory(sentDictBytesAddr, newAddr, sentDictBytesLen);
+            }
+            sentDictBytesAddr = newAddr;
+            sentDictBytesOwned = true;
+        }
         sentDictBytesCapacity = (int) newCap;
     }
 
+    private void releaseSentDictBytes() {
+        if (sentDictBytesAddr != 0 && sentDictBytesOwned) {
+            Unsafe.free(sentDictBytesAddr, sentDictBytesCapacity, MemoryTag.NATIVE_DEFAULT);
+        }
+        sentDictBytesAddr = 0;
+        sentDictBytesCapacity = 0;
+        sentDictBytesLen = 0;
+        sentDictBytesOwned = false;
+        sentDictCount = 0;
+    }
+
     private long readVarintAt(long p, long limit) {
         long value = 0;
         int shift = 0;
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java
index f74c8655..27523de4 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java
@@ -364,8 +364,9 @@ public static MmapSegment openExisting(FilesFacade ff, String path) {
                 throw new MmapSegmentException(
                         "bad baseSeq in " + path + ": " + baseSeq);
             }
-            long lastGood = scanFrames(addr, fileSize);
-            long count = countFrames(addr, lastGood);
+            FrameScan scan = scanFrames(addr, fileSize);
+            long lastGood = scan.lastGood;
+            long count = scan.frameCount;
             long tornTail = detectTornTail(addr, lastGood, fileSize);
             if (tornTail > 0) {
                 LOG.warn("SF segment {}: torn tail of {} bytes at offset {} "
@@ -907,14 +908,15 @@ private static boolean isMmapAccessFault(Throwable t) {
     }
 
     /**
-     * Forward scan that returns the offset just past the last frame whose
-     * CRC verifies. A torn-tail frame (declared length runs past EOF, or
-     * CRC mismatch) leaves both cursors at the start of that frame; the
-     * next {@link #tryAppend} will overwrite it. The scan only reads from
-     * the mapping — no syscalls.
+     * Forward scan that returns the offset just past the last frame whose CRC
+     * verifies and the number of verified frames. A torn-tail frame (declared
+     * length runs past EOF, or CRC mismatch) leaves both cursors at the start of
+     * that frame; the next {@link #tryAppend} will overwrite it. The scan only
+     * reads from the mapping — no syscalls.
      */
-    private static long scanFrames(long addr, long fileSize) {
+    private static FrameScan scanFrames(long addr, long fileSize) {
         long pos = HEADER_SIZE;
+        long frameCount = 0L;
         try {
             while (pos + FRAME_HEADER_SIZE <= fileSize) {
                 int crcRead = Unsafe.getUnsafe().getInt(addr + pos);
@@ -922,7 +924,7 @@ private static long scanFrames(long addr, long fileSize) {
                 // Defensive: a corrupt length field could be enormous or negative,
                 // both of which would otherwise overrun the mapping.
                 if (payloadLen < 0 || pos + FRAME_HEADER_SIZE + payloadLen > fileSize) {
-                    return pos;
+                    return new FrameScan(pos, frameCount);
                 }
                 // CRC over the contiguous (payloadLen, payload) pair, folded
                 // via Unsafe reads rather than the native Crc32c.update.
@@ -945,9 +947,10 @@ private static long scanFrames(long addr, long fileSize) {
                 // table CRC here is immaterial.
                 int crcCalc = crc32cRecovery(addr + pos + 4, 4L + payloadLen);
                 if (crcCalc != crcRead) {
-                    return pos;
+                    return new FrameScan(pos, frameCount);
                 }
                 pos += FRAME_HEADER_SIZE + payloadLen;
+                frameCount++;
             }
         } catch (InternalError e) {
             // The read at `pos` hit a mapped page that is not backed by real
@@ -973,7 +976,7 @@ private static long scanFrames(long addr, long fileSize) {
                             + "if this recurs.",
                     pos, fileSize);
         }
-        return pos;
+        return new FrameScan(pos, frameCount);
     }
 
     /**
@@ -1049,19 +1052,13 @@ private static long detectTornTail(long addr, long lastGood, long fileSize) {
         return 0L;
     }
 
-    /**
-     * Counts frames in {@code [HEADER_SIZE, lastGood)}. Walks the framing in
-     * lockstep with {@link #scanFrames} (which already validated CRCs); so
-     * this is just length-driven traversal, no CRC re-check.
-     */
-    private static long countFrames(long addr, long lastGood) {
-        long pos = HEADER_SIZE;
-        long count = 0;
-        while (pos < lastGood) {
-            int payloadLen = Unsafe.getUnsafe().getInt(addr + pos + 4);
-            pos += FRAME_HEADER_SIZE + payloadLen;
-            count++;
+    private static final class FrameScan {
+        private final long frameCount;
+        private final long lastGood;
+
+        private FrameScan(long lastGood, long frameCount) {
+            this.lastGood = lastGood;
+            this.frameCount = frameCount;
         }
-        return count;
     }
 }
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java
index 982f93c2..914e4ab5 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java
@@ -163,6 +163,7 @@ public final class PersistedSymbolDict implements QuietCloseable {
      */
     static final int MAX_SCRATCH_BYTES = Integer.MAX_VALUE - 8;
     static final byte VERSION = 3; // v3 moved the CRC-32C from per-entry to per-chunk
+    private static final int[] CRC32C_TABLE = buildCrc32cTable();
     private static final Logger LOG = LoggerFactory.getLogger(PersistedSymbolDict.class);
     private final int fd;
     // Filesystem seam. Production is FilesFacade.INSTANCE (straight to Files);
@@ -172,6 +173,10 @@ public final class PersistedSymbolDict implements QuietCloseable {
     // Production writes directly into segmented append mappings. Custom facades retain the
     // positioned-write path so fault tests can inject short writes through ff.write.
     private final boolean mappedAppend;
+    // True only when recovery parsed the file through a temporary read-only
+    // mmap instead of allocating a second native buffer as large as the file.
+    // Test-visible so the peak-memory regression has an observable contract.
+    private final boolean mappedRecoveryInput;
     private long appendMapAddr;
     private long appendMapCapacity;
     private long appendMapOffset;
@@ -182,21 +187,29 @@ public final class PersistedSymbolDict implements QuietCloseable {
     // In-memory copy of the WIRE entry region [len][utf8]... -- chunk headers and
     // CRCs stripped -- populated only when open() recovered existing chunks
     // (recovery / orphan-drain). Zero/empty for a freshly created file. READ (not
-    // consumed) to seed the producer's id map (readLoadedSymbols) and to seed the
-    // send loop's catch-up mirror, which COPIES it. This file retains ownership for
-    // the engine's lifetime -- the orphan drainer builds a fresh send loop per wire
-    // session against the same engine, and each must re-seed its mirror -- and frees
-    // this buffer in close().
+    // consumed) to seed the producer's id map and to seed the send loop's catch-up
+    // mirror. Foreground construction transfers ownership to its sole loop after
+    // producer seeding; orphan-drainer loops borrow it because one engine may create
+    // several wire sessions. If ownership was not transferred, close() frees it.
     private long loadedEntriesAddr;
     private int loadedEntriesLen;
     private long scratchAddr;
     private int scratchCap;
     private int size;
 
-    private PersistedSymbolDict(FilesFacade ff, int fd, long appendOffset, int size, long loadedEntriesAddr, int loadedEntriesLen) {
+    private PersistedSymbolDict(
+            FilesFacade ff,
+            int fd,
+            long appendOffset,
+            int size,
+            long loadedEntriesAddr,
+            int loadedEntriesLen,
+            boolean mappedRecoveryInput
+    ) {
         this.ff = ff;
         this.fd = fd;
         this.mappedAppend = ff == FilesFacade.INSTANCE;
+        this.mappedRecoveryInput = mappedRecoveryInput;
         this.appendOffset = appendOffset;
         this.size = size;
         this.loadedEntriesAddr = loadedEntriesAddr;
@@ -243,13 +256,9 @@ public static PersistedSymbolDict open(FilesFacade ff, String slotDir) {
             return null;
         }
         if (existing >= HEADER_SIZE) {
-            // A dictionary at or past Integer.MAX_VALUE cannot be read back:
-            // openExisting reads it into ONE int-sized native buffer, and the (int)
-            // cast is either negative (malloc rejects it) or a small positive prefix
-            // (whose parse would then trim the real multi-GB file). Degrade to full
-            // self-sufficient frames and leave the file alone. Reaching this needs
-            // ~100M+ distinct symbols on one slot, far past realistic cardinality;
-            // the guard keeps the read/write size boundary safe anyway.
+            // Chunk lengths and the retained contiguous entry region are int-sized,
+            // so a dictionary at or past Integer.MAX_VALUE cannot be represented
+            // safely even though production recovery maps rather than reads it.
             if (existing >= Integer.MAX_VALUE) {
                 LOG.warn("symbol dict {} too large ({} bytes) to reopen; "
                         + "falling back to full-dictionary frames (file left intact)", filePath, existing);
@@ -541,17 +550,45 @@ public int loadedEntriesLen() {
     }
 
     /**
-     * Materialises the loaded entries as symbol strings in ascending-id order
-     * (entry {@code i} is symbol id {@code i}). Used once on recovery to
-     * repopulate the producer's global dictionary. Empty when nothing was
-     * recovered.
-     * 
      
-     * Construction-phase only -- like {@link #loadedEntriesAddr()}, this
-     * walks the native entry region {@link #close()} frees, with no closed-guard,
-     * so it must run before the I/O thread and any producer append start.
+     * Transfers the recovered entry buffer to the foreground send loop. The
+     * symbol count remains unchanged because it is also the append baseline;
+     * only the native-buffer ownership moves. Construction-phase only.
+     */
+    synchronized long takeLoadedEntries() {
+        long addr = loadedEntriesAddr;
+        loadedEntriesAddr = 0L;
+        loadedEntriesLen = 0;
+        return addr;
+    }
+
+    @TestOnly
+    public boolean usedMappedRecoveryInput() {
+        return mappedRecoveryInput;
+    }
+
+    /**
+     * Decodes the recovered entries directly into {@code target} in ascending-id
+     * order. This avoids materialising a cardinality-sized temporary list that
+     * the producer would immediately copy into the global dictionary.
+     * Construction-phase only; see {@link #loadedEntriesAddr()}.
      */
+    public void addLoadedSymbolsTo(GlobalSymbolDictionary target) {
+        decodeLoadedSymbols(target, null);
+    }
+
+    /**
+     * Materialises the loaded entries as symbol strings in ascending-id order.
+     * Retained for recovery-format tests; production decodes directly through
+     * {@link #addLoadedSymbolsTo(GlobalSymbolDictionary)}.
+     */
+    @TestOnly
     public ObjList readLoadedSymbols() {
         ObjList out = new ObjList<>(Math.max(size, 1));
+        decodeLoadedSymbols(null, out);
+        return out;
+    }
+
+    private void decodeLoadedSymbols(GlobalSymbolDictionary target, ObjList out) {
         long p = loadedEntriesAddr;
         long limit = p + loadedEntriesLen;
         for (int i = 0; i < size && p < limit; i++) {
@@ -575,10 +612,14 @@ public ObjList readLoadedSymbols() {
             if (p + len > limit) {
                 break; // defensive: torn tail (should not happen past parse in open)
             }
-            out.add(Utf8s.stringFromUtf8Bytes(p, p + len));
+            String symbol = Utf8s.stringFromUtf8Bytes(p, p + len);
+            if (target != null) {
+                target.addRecoveredSymbol(symbol);
+            } else {
+                out.add(symbol);
+            }
             p += len;
         }
-        return out;
     }
 
     /**
@@ -595,91 +636,48 @@ private static PersistedSymbolDict openExisting(FilesFacade ff, String filePath,
                     + "falling back to full-dictionary frames (file left intact)", filePath, fd);
             return null;
         }
-        long buf = 0L;
+        int len = (int) fileLen; // open() bounds fileLen to [HEADER_SIZE, Integer.MAX_VALUE)
+        boolean mappedInput = ff == FilesFacade.INSTANCE;
+        long inputAddr = 0L;
         long entriesAddr = 0L;
-        int entriesCap = 0;
+        int entriesLen = 0;
         try {
-            int len = (int) fileLen; // open() bounds fileLen to [HEADER_SIZE, Integer.MAX_VALUE)
-            buf = Unsafe.malloc(len, MemoryTag.NATIVE_DEFAULT);
-            long read = ff.read(fd, buf, len, 0);
-            if (read != len
-                    || Unsafe.getUnsafe().getInt(buf) != FILE_MAGIC
-                    || Unsafe.getUnsafe().getByte(buf + 4) != VERSION) {
-                LOG.warn("symbol dict {} unreadable, bad magic or unknown version; "
-                        + "falling back to full-dictionary frames (file left intact)", filePath);
-                Unsafe.free(buf, len, MemoryTag.NATIVE_DEFAULT);
-                buf = 0L; // null after free so the catch below cannot double-free if ff.close throws
-                int fdToClose = fd;
-                fd = -1; // relinquish before close so the catch cannot double-close if close throws
-                ff.close(fdToClose);
-                return null;
-            }
-            // Parse the chunks after the header, copying each chunk's entry region
-            // into entriesAddr AS WE VALIDATE IT -- one pass over the file, not two.
-            // Every chunk sheds its two header varints and its CRC, so the entry
-            // region is strictly smaller than the file and len - HEADER_SIZE is a safe
-            // upper bound to allocate up front; we shrink to the exact size below.
-            //
-            // Stop at the first torn/incomplete OR crc-mismatched chunk. The CRC turns
-            // an interior tear (a lost page reading back as zeroes) or a stale chunk
-            // left past the end by a failed truncate into a clean stop point, so
-            // recovery trusts only the intact prefix instead of silently mis-parsing a
-            // corrupt chunk and shifting the dense id->symbol map.
-            entriesCap = len - HEADER_SIZE;
-            long dst = 0L;
-            if (entriesCap > 0) {
-                entriesAddr = Unsafe.malloc(entriesCap, MemoryTag.NATIVE_DEFAULT);
-                dst = entriesAddr;
-            }
-            Varint v = new Varint();
-            int diskPos = HEADER_SIZE;
-            int count = 0;
-            while (diskPos < len) {
-                if (!v.decode(buf, diskPos, len)) {
-                    break; // torn entryCount varint
-                }
-                long entryCount = v.value;
-                if (!v.decode(buf, v.end, len)) {
-                    break; // torn entryBytes varint
-                }
-                long entryBytes = v.value;
-                int entriesStart = v.end;
-                // entryCount/entryBytes stay long so a corrupt multi-gigabyte value
-                // cannot wrap an int back under the bound checks.
-                long chunkEnd = (long) entriesStart + entryBytes; // end of the entry region
-                if (chunkEnd + CRC_SIZE > len) {
-                    break; // torn/incomplete trailing chunk (its CRC doesn't fit)
+            if (mappedInput) {
+                inputAddr = Files.mmap(fd, fileLen, 0L, Files.MAP_RO, MemoryTag.MMAP_DEFAULT);
+                if (inputAddr == Files.FAILED_MMAP_ADDRESS) {
+                    inputAddr = 0L;
+                    throw new IllegalStateException("could not mmap symbol dictionary for recovery");
                 }
-                int chunkEndI = (int) chunkEnd;
-                int crcStored = Unsafe.getUnsafe().getInt(buf + chunkEndI);
-                int crcCalc = Crc32c.update(Crc32c.INIT, buf + diskPos, chunkEndI - diskPos);
-                if (crcCalc != crcStored) {
-                    break; // corrupt/stale chunk -- stop before it (fail-clean)
+            } else {
+                // Fault-injection facades retain the positioned-read path so tests
+                // can still force short reads without mapping around the seam.
+                inputAddr = Unsafe.malloc(len, MemoryTag.NATIVE_DEFAULT);
+                long read = ff.read(fd, inputAddr, len, 0);
+                if (read != len) {
+                    throw new IllegalStateException("short read while recovering symbol dictionary "
+                            + "[expected=" + len + ", actual=" + read + ']');
                 }
-                // A chunk carries at least one entry, and the ids are int-dense. A
-                // CRC-valid chunk cannot violate either at realistic cardinality, but
-                // keep the int narrowing honest rather than wrapping the id space.
-                if (entryCount <= 0 || (long) count + entryCount > Integer.MAX_VALUE) {
-                    break;
-                }
-                Unsafe.getUnsafe().copyMemory(buf + entriesStart, dst, entryBytes);
-                dst += entryBytes;
-                diskPos = chunkEndI + CRC_SIZE;
-                count += (int) entryCount;
             }
-            int entriesLen = entriesAddr != 0L ? (int) (dst - entriesAddr) : 0;
-            int diskConsumed = diskPos - HEADER_SIZE; // valid chunks incl. headers and CRCs
-            if (entriesAddr != 0L && entriesLen == 0) {
-                Unsafe.free(entriesAddr, entriesCap, MemoryTag.NATIVE_DEFAULT);
-                entriesAddr = 0L;
-                entriesCap = 0;
-            } else if (entriesAddr != 0L && entriesLen < entriesCap) {
-                // Shrink the upper-bound allocation to what the trusted prefix used.
-                entriesAddr = Unsafe.realloc(entriesAddr, entriesCap, entriesLen, MemoryTag.NATIVE_DEFAULT);
-                entriesCap = entriesLen;
+            if (Unsafe.getUnsafe().getInt(inputAddr) != FILE_MAGIC
+                    || Unsafe.getUnsafe().getByte(inputAddr + 4) != VERSION) {
+                throw new IllegalStateException("bad magic or unknown symbol dictionary version");
+            }
+            // First pass validates chunks and totals only the entry bytes that
+            // survive. Production reads through a file-backed mapping, so this
+            // pass does not allocate an anonymous whole-file copy. The second
+            // pass copies directly into one exact-sized retained allocation.
+            RecoveryScan scan = scanRecoveredChunks(inputAddr, len);
+            entriesLen = scan.entriesLen;
+            if (entriesLen > 0) {
+                entriesAddr = Unsafe.malloc(entriesLen, MemoryTag.NATIVE_DEFAULT);
+                copyRecoveredEntries(inputAddr, scan.validLen, entriesAddr, entriesLen);
             }
-            Unsafe.free(buf, len, MemoryTag.NATIVE_DEFAULT);
-            buf = 0L;
+            if (mappedInput) {
+                Files.munmap(inputAddr, fileLen, MemoryTag.MMAP_DEFAULT);
+            } else {
+                Unsafe.free(inputAddr, len, MemoryTag.NATIVE_DEFAULT);
+            }
+            inputAddr = 0L;
             // Drop any torn/stale trailing bytes so a LATER, shorter append cannot
             // leave residue past its own end. The truncate result IS checked: a file
             // we cannot trim could still expose stale post-end bytes whose
@@ -687,36 +685,23 @@ private static PersistedSymbolDict openExisting(FilesFacade ff, String filePath,
             // position, so a failed truncate makes the file untrusted -- return null
             // (the sender falls back to full self-sufficient frames) and, per the
             // never-destroy contract, leave every byte on disk.
-            long validLen = HEADER_SIZE + diskConsumed;
-            if (validLen < len && !ff.truncate(fd, validLen)) {
-                LOG.warn("symbol dict {} could not drop its torn/stale tail (truncate failed); "
-                        + "falling back to full-dictionary frames (file left intact)", filePath);
-                if (entriesAddr != 0L) {
-                    Unsafe.free(entriesAddr, entriesCap, MemoryTag.NATIVE_DEFAULT);
-                    // Null after freeing (like buf above) so the catch below cannot
-                    // double-free entriesAddr if the following ff.close throws.
-                    entriesAddr = 0L;
-                    entriesCap = 0;
-                }
-                int fdToClose = fd;
-                fd = -1; // relinquish before close so the catch cannot double-close if close throws
-                ff.close(fdToClose);
-                return null;
+            if (scan.validLen < len && !ff.truncate(fd, scan.validLen)) {
+                throw new IllegalStateException("could not drop torn/stale symbol dictionary tail");
             }
-            return new PersistedSymbolDict(ff, fd, validLen, count, entriesAddr, entriesLen);
+            return new PersistedSymbolDict(
+                    ff, fd, scan.validLen, scan.count, entriesAddr, entriesLen, mappedInput);
         } catch (Throwable t) {
-            if (buf != 0L) {
-                Unsafe.free(buf, (int) fileLen, MemoryTag.NATIVE_DEFAULT);
+            if (inputAddr != 0L) {
+                if (mappedInput) {
+                    Files.munmap(inputAddr, fileLen, MemoryTag.MMAP_DEFAULT);
+                } else {
+                    Unsafe.free(inputAddr, len, MemoryTag.NATIVE_DEFAULT);
+                }
             }
-            // Free entriesAddr if it was allocated and not yet handed off. The success
-            // path transfers it to the returned dict, and every path that frees it
-            // earlier (the truncate-failure branch above) also nulls it, so this cannot
-            // double-free. Keeps the error path leak-free on any throw between its
-            // malloc and the return.
             if (entriesAddr != 0L) {
-                Unsafe.free(entriesAddr, entriesCap, MemoryTag.NATIVE_DEFAULT);
+                Unsafe.free(entriesAddr, entriesLen, MemoryTag.NATIVE_DEFAULT);
             }
-            if (fd >= 0) { // a branch that already closed fd relinquished it to -1
+            if (fd >= 0) {
                 ff.close(fd);
             }
             // Pass the throwable as a trailing argument with no matching placeholder so
@@ -728,6 +713,79 @@ private static PersistedSymbolDict openExisting(FilesFacade ff, String filePath,
         }
     }
 
+    private static void copyRecoveredEntries(long inputAddr, int validLen, long entriesAddr, int entriesLen) {
+        Varint v = new Varint();
+        int diskPos = HEADER_SIZE;
+        int dstPos = 0;
+        while (diskPos < validLen) {
+            if (!v.decode(inputAddr, diskPos, validLen)
+                    || !v.decode(inputAddr, v.end, validLen)) {
+                throw new IllegalStateException("validated symbol dictionary chunk could not be decoded");
+            }
+            int entryBytes = (int) v.value;
+            int entriesStart = v.end;
+            Unsafe.getUnsafe().copyMemory(inputAddr + entriesStart, entriesAddr + dstPos, entryBytes);
+            dstPos += entryBytes;
+            diskPos = entriesStart + entryBytes + CRC_SIZE;
+        }
+        assert dstPos == entriesLen;
+    }
+
+    private static int[] buildCrc32cTable() {
+        int[] table = new int[256];
+        for (int n = 0; n < 256; n++) {
+            int c = n;
+            for (int k = 0; k < 8; k++) {
+                c = (c & 1) != 0 ? (0x82F63B78 ^ (c >>> 1)) : (c >>> 1);
+            }
+            table[n] = c;
+        }
+        return table;
+    }
+
+    private static int crc32cRecovery(long addr, long len) {
+        int crc = ~Crc32c.INIT;
+        for (long i = 0; i < len; i++) {
+            crc = (crc >>> 8) ^ CRC32C_TABLE[(crc ^ Unsafe.getUnsafe().getByte(addr + i)) & 0xFF];
+        }
+        return ~crc;
+    }
+
+    private static RecoveryScan scanRecoveredChunks(long inputAddr, int len) {
+        Varint v = new Varint();
+        int count = 0;
+        int diskPos = HEADER_SIZE;
+        long entriesLen = 0L;
+        while (diskPos < len) {
+            if (!v.decode(inputAddr, diskPos, len)) {
+                break;
+            }
+            long entryCount = v.value;
+            if (!v.decode(inputAddr, v.end, len)) {
+                break;
+            }
+            long entryBytes = v.value;
+            int entriesStart = v.end;
+            long chunkEnd = (long) entriesStart + entryBytes;
+            if (chunkEnd + CRC_SIZE > len) {
+                break;
+            }
+            int chunkEndI = (int) chunkEnd;
+            int crcStored = Unsafe.getUnsafe().getInt(inputAddr + chunkEndI);
+            int crcCalc = crc32cRecovery(inputAddr + diskPos, chunkEndI - diskPos);
+            if (crcCalc != crcStored
+                    || entryCount <= 0
+                    || (long) count + entryCount > Integer.MAX_VALUE
+                    || entriesLen + entryBytes > Integer.MAX_VALUE) {
+                break;
+            }
+            entriesLen += entryBytes;
+            diskPos = chunkEndI + CRC_SIZE;
+            count += (int) entryCount;
+        }
+        return new RecoveryScan(count, (int) entriesLen, diskPos);
+    }
+
     private static PersistedSymbolDict openFresh(FilesFacade ff, String filePath) {
         int fd = ff.openCleanRW(filePath);
         if (fd < 0) {
@@ -770,7 +828,7 @@ private static PersistedSymbolDict openFresh(FilesFacade ff, String filePath) {
                 Unsafe.free(hdr, HEADER_SIZE, MemoryTag.NATIVE_DEFAULT);
             }
         }
-        return new PersistedSymbolDict(ff, fd, HEADER_SIZE, 0, 0L, 0);
+        return new PersistedSymbolDict(ff, fd, HEADER_SIZE, 0, 0L, 0, false);
     }
 
     /**
@@ -962,11 +1020,22 @@ private void writeChunkFromScratch(int entriesLen, int count) {
         flushChunk(recStart, hdrLen, entriesLen, count);
     }
 
+    private static final class RecoveryScan {
+        private final int count;
+        private final int entriesLen;
+        private final int validLen;
+
+        private RecoveryScan(int count, int entriesLen, int validLen) {
+            this.count = count;
+            this.entriesLen = entriesLen;
+            this.validLen = validLen;
+        }
+    }
+
     /**
-     * Zero-allocation LEB128 decoder, one instance per {@link #openExisting} call --
-     * not one per chunk. The previous {@code long[]}-returning decoder allocated once
-     * per ENTRY, so a million-symbol recovery churned a million throwaway arrays in a
-     * class that is otherwise strictly allocation-free.
+     * Zero-allocation LEB128 decoder, one instance per recovery pass -- not one
+     * per chunk. The previous {@code long[]}-returning decoder allocated once per
+     * entry, so a million-symbol recovery churned a million throwaway arrays.
      */
     private static final class Varint {
         int end;
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/RecoveredFrameAnalysis.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/RecoveredFrameAnalysis.java
index 386d0f1e..e94e68f5 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/RecoveredFrameAnalysis.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/RecoveredFrameAnalysis.java
@@ -24,6 +24,7 @@
 
 package io.questdb.client.cutlass.qwp.client.sf.cursor;
 
+import io.questdb.client.cutlass.qwp.client.GlobalSymbolDictionary;
 import io.questdb.client.cutlass.qwp.protocol.QwpConstants;
 import io.questdb.client.std.MemoryTag;
 import io.questdb.client.std.ObjList;
@@ -55,6 +56,7 @@ final class RecoveredFrameAnalysis implements QuietCloseable {
     private boolean runningUnackedGap;
     private long runningMaxDeltaEnd;
     private long runningMaxDeltaStart;
+    private long symbolEntriesVisited;
     private long rawAddr;
     private int rawCapacity;
     private int runningRawCount;
@@ -95,6 +97,14 @@ void accept(long fsn, long payload, int payloadLen) {
     }
 
     void appendDecodedSymbols(ObjList out) {
+        decodeSymbols(null, out);
+    }
+
+    void addDecodedSymbolsTo(GlobalSymbolDictionary target) {
+        decodeSymbols(target, null);
+    }
+
+    private void decodeSymbols(GlobalSymbolDictionary target, ObjList out) {
         long p = rawAddr;
         long limit = rawAddr + committedRawLen;
         for (int i = 0; i < committedRawCount; i++) {
@@ -108,7 +118,12 @@ void appendDecodedSymbols(ObjList out) {
             if (symbolLen > limit - p) {
                 throw new IllegalStateException("truncated cached symbol dictionary suffix");
             }
-            out.add(Utf8s.stringFromUtf8Bytes(p, p + symbolLen));
+            String symbol = Utf8s.stringFromUtf8Bytes(p, p + symbolLen);
+            if (target != null) {
+                target.addRecoveredSymbol(symbol);
+            } else {
+                out.add(symbol);
+            }
             p += symbolLen;
         }
         if (p != limit) {
@@ -152,6 +167,10 @@ int rawLen() {
         return committedRawLen;
     }
 
+    long symbolEntriesVisited() {
+        return symbolEntriesVisited;
+    }
+
     @Override
     public void close() {
         if (rawAddr != 0L) {
@@ -227,9 +246,16 @@ private void foldDelta(long fsn, long p, long limit) {
             markGap(fsn);
             return;
         }
+        // The segment scan already CRC-validated this frame. When its entire
+        // range is covered, entry parsing cannot extend recovery state, so skip
+        // the cardinality-proportional payload walk.
+        if (deltaEnd <= runningCoverage) {
+            return;
+        }
 
         long id = deltaStart;
         for (long i = 0; i < deltaCount; i++, id++) {
+            symbolEntriesVisited++;
             long entryStart = p;
             long encodedLen = readVarint(p, limit);
             if (encodedLen < 0L) {
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java
index acc8102a..86ba58a0 100644
--- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java
@@ -960,6 +960,7 @@ private static void seedMirror(CursorWebSocketSendLoop loop, String... symbols)
             }
         }
         setField(loop, "sentDictBytesAddr", addr);
+        setBooleanField(loop, "sentDictBytesOwned", true);
         setIntField(loop, "sentDictBytesCapacity", total);
         setIntField(loop, "sentDictBytesLen", total);
         setIntField(loop, "sentDictCount", symbols.length);
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopMirrorLeakTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopMirrorLeakTest.java
index 8697eaee..116cbc55 100644
--- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopMirrorLeakTest.java
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopMirrorLeakTest.java
@@ -81,6 +81,7 @@ public void testSeededMirrorFreedWhenLoopClosedWithoutStart() throws Exception {
                     Assert.assertNotNull("disk-mode engine must open a persisted dict", pd);
                     Assert.assertTrue("recovery must load the persisted symbols (seeds the mirror)",
                             pd.size() > 0 && pd.loadedEntriesLen() > 0);
+                    long persistedAddr = pd.loadedEntriesAddr();
 
                     CursorWebSocketSendLoop loop = new CursorWebSocketSendLoop(
                             null, engine, 0, 1_000_000L,
@@ -92,6 +93,12 @@ public void testSeededMirrorFreedWhenLoopClosedWithoutStart() throws Exception {
                     // thread's to free, since the I/O loop never ran.
                     Assert.assertTrue("precondition: the ctor seeded a non-empty mirror",
                             readInt(loop, "sentDictCount") > 0);
+                    Assert.assertEquals("foreground loop must take the persisted buffer without copying",
+                            persistedAddr, readLong(loop, "sentDictBytesAddr"));
+                    Assert.assertEquals("ownership transfer must clear the persisted pointer",
+                            0L, pd.loadedEntriesAddr());
+                    Assert.assertTrue("foreground mirror must own the transferred buffer",
+                            readBoolean(loop, "sentDictBytesOwned"));
                     loop.close();
                     // close() must reset sentDictCount alongside freeing the buffer,
                     // so the mirror stays all-or-nothing: a hypothetical post-close
@@ -111,14 +118,14 @@ public void testRecycledLoopReSeedsMirrorFromPersistedDict() throws Exception {
         // C1 regression: the orphan drainer (BackgroundDrainer) builds a NEW
         // CursorWebSocketSendLoop per wire session against the SAME, persistent
         // engine when a durable-ack capability gap forces a mid-drain recycle. The
-        // recovery mirror seed must survive that recycle. If the first loop CONSUMED
+        // recovery mirror seed must survive that recycle. If the first loop consumes
         // the persisted dictionary's loaded entries (a one-shot ownership transfer),
         // the second loop seeds an EMPTY mirror (sentDictCount = 0), sends no
         // reconnect catch-up, and the first replayed delta frame (deltaStart > 0)
         // trips the torn-dict guard -- falsely quarantining a healthy slot with a
-        // bogus "resend required" terminal. Copying the entries (leaving the
-        // dictionary intact for the engine's lifetime) lets every recycled loop
-        // re-seed. Pre-fix, loop2's sentDictCount is 0 and this assertion fails.
+        // bogus "resend required" terminal. Borrowing the entries leaves the
+        // dictionary intact for the engine's lifetime without making another native
+        // copy, so every recycled loop can re-seed.
         Path sfDir = Files.createTempDirectory("qwp-mirror-reseed");
         try {
             populateRecoverableSlot(sfDir);
@@ -129,15 +136,22 @@ public void testRecycledLoopReSeedsMirrorFromPersistedDict() throws Exception {
                     Assert.assertNotNull(pd);
                     int dictSize = pd.size();
                     Assert.assertTrue("recovery must load a non-empty dictionary", dictSize > 0);
+                    long persistedAddr = pd.loadedEntriesAddr();
 
                     // Session 1 seeds its mirror from the persisted dictionary.
                     CursorWebSocketSendLoop loop1 = newRecoveryLoop(engine);
                     try {
                         Assert.assertEquals("session-1 mirror must seed from the persisted dict",
                                 dictSize, readInt(loop1, "sentDictCount"));
+                        Assert.assertEquals("orphan session must borrow the persisted bytes",
+                                persistedAddr, readLong(loop1, "sentDictBytesAddr"));
+                        Assert.assertFalse("borrowed orphan mirror must not own the persisted bytes",
+                                readBoolean(loop1, "sentDictBytesOwned"));
                     } finally {
                         loop1.close();
                     }
+                    Assert.assertEquals("closing a borrowed loop must leave the engine prefix alive",
+                            persistedAddr, pd.loadedEntriesAddr());
 
                     // Session 2 against the SAME engine (the drainer recycle): the
                     // seed must NOT have been consumed -- the mirror must re-seed to
@@ -147,6 +161,7 @@ public void testRecycledLoopReSeedsMirrorFromPersistedDict() throws Exception {
                         Assert.assertEquals("recycled session-2 mirror must re-seed from the "
                                         + "persisted dict (pre-fix it was 0)",
                                 dictSize, readInt(loop2, "sentDictCount"));
+                        Assert.assertEquals(persistedAddr, readLong(loop2, "sentDictBytesAddr"));
                     } finally {
                         loop2.close();
                     }
@@ -205,6 +220,8 @@ public void testCtorFreesSeededMirrorWhenFrameSeedThrows() throws Exception {
                     } finally {
                         setMirrorSeedFault(false);
                     }
+                    Assert.assertTrue("failed foreground construction must leave the persisted prefix owned",
+                            pd.loadedEntriesAddr() != 0L);
                     // The outer assertMemoryLeak proves the prefix-seeded mirror the ctor
                     // malloc'd was freed on the throw -- pre-fix it leaks here.
                 }
@@ -223,7 +240,9 @@ private static CursorWebSocketSendLoop newRecoveryLoop(CursorSendEngine engine)
                 () -> {
                     throw new IOException("no reconnect in this test");
                 },
-                0, 0, 1);
+                0, 0, 1,
+                false, 0L, 3, 0L, 0L,
+                CursorWebSocketSendLoop.CatchUpCapGapPolicy.TERMINAL_AFTER_SETTLE_BUDGET);
     }
 
     private static void populateRecoverableSlot(Path sfDir) throws Exception {
@@ -275,6 +294,18 @@ private static int readInt(CursorWebSocketSendLoop loop, String name) throws Exc
         return f.getInt(loop);
     }
 
+    private static boolean readBoolean(CursorWebSocketSendLoop loop, String name) throws Exception {
+        Field f = CursorWebSocketSendLoop.class.getDeclaredField(name);
+        f.setAccessible(true);
+        return f.getBoolean(loop);
+    }
+
+    private static long readLong(CursorWebSocketSendLoop loop, String name) throws Exception {
+        Field f = CursorWebSocketSendLoop.class.getDeclaredField(name);
+        f.setAccessible(true);
+        return f.getLong(loop);
+    }
+
     // Toggles the loop's @TestOnly mirror-seed fault flag. Reflection because the
     // flag is package-private in the production package (this test is in a sibling
     // test package), the same non-reflective-path-unavailable reason readInt uses.
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopOrphanTailTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopOrphanTailTest.java
index c96bfefe..d7010310 100644
--- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopOrphanTailTest.java
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopOrphanTailTest.java
@@ -27,10 +27,12 @@
 import io.questdb.client.DefaultHttpClientConfiguration;
 import io.questdb.client.cutlass.http.client.WebSocketClient;
 import io.questdb.client.cutlass.http.client.WebSocketFrameHandler;
+import io.questdb.client.cutlass.qwp.client.GlobalSymbolDictionary;
 import io.questdb.client.cutlass.qwp.client.WebSocketResponse;
 import io.questdb.client.cutlass.qwp.client.sf.cursor.AckWatermark;
 import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine;
 import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorWebSocketSendLoop;
+import io.questdb.client.cutlass.qwp.client.sf.cursor.PersistedSymbolDict;
 import io.questdb.client.network.PlainSocketFactory;
 import io.questdb.client.std.Files;
 import io.questdb.client.std.MemoryTag;
@@ -399,6 +401,35 @@ public void testRecoveryScansFramesOnceAndReusesCachedSymbolSuffix() throws Exce
         });
     }
 
+    @Test
+    public void testRecoverySkipsEntriesAlreadyCoveredByPersistedPrefix() throws Exception {
+        TestUtils.assertMemoryLeak(() -> {
+            try (CursorSendEngine engine = newEngine()) {
+                PersistedSymbolDict pd = engine.getPersistedSymbolDict();
+                assertNotNull(pd);
+                pd.appendSymbol("a");
+                pd.appendSymbol("b");
+                pd.appendSymbol("c");
+                appendDeltaSymbolFrame(engine, 0, 'a');
+                appendDeltaSymbolFrame(engine, 1, 'b');
+                appendDeltaSymbolFrame(engine, 2, 'c');
+            }
+
+            try (CursorSendEngine engine = newEngine()) {
+                assertEquals("covered delta payloads must not be parsed entry-by-entry",
+                        0L, engine.recoverySymbolEntriesVisited());
+                GlobalSymbolDictionary recovered = new GlobalSymbolDictionary();
+                PersistedSymbolDict pd = engine.getPersistedSymbolDict();
+                assertNotNull(pd);
+                pd.addLoadedSymbolsTo(recovered);
+                assertEquals(3L, engine.addRecoveredSymbolsTo(recovered.size(), recovered));
+                assertEquals("direct decode must not duplicate the covered frame entries",
+                        3, recovered.size());
+                assertEquals("c", recovered.getSymbol(2));
+            }
+        });
+    }
+
     @Test
     public void testSelfSufficientFrameRepairsAckedRecoveryGap() throws Exception {
         // fsn 0 models the tail of an old delta epoch whose registering frames
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/MmapSegmentTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/MmapSegmentTest.java
index 4707c233..da9c4494 100644
--- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/MmapSegmentTest.java
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/MmapSegmentTest.java
@@ -100,6 +100,8 @@ public void testCreateAppendCloseReopenScansAllFrames() throws Exception {
                     long expectedEnd = MmapSegment.HEADER_SIZE
                             + 100L * (MmapSegment.FRAME_HEADER_SIZE + 32);
                     assertEquals(expectedEnd, seg.publishedOffset());
+                    assertEquals("validation scan must return the frame count",
+                            100L, seg.frameCount());
                 }
 
                 // Re-open: scan must land at exactly the same offset.
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java
index aea6d1af..ee046d33 100644
--- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java
@@ -77,6 +77,13 @@ public void testAppendPersistsAcrossReopen() throws Exception {
                 Assert.assertEquals("GOOG", symbols.getQuick(1));
                 Assert.assertEquals("MSFT", symbols.getQuick(2));
                 Assert.assertTrue(reopened.loadedEntriesLen() > 0);
+                Assert.assertTrue("production recovery must parse through mmap instead of a heap file copy",
+                        reopened.usedMappedRecoveryInput());
+
+                GlobalSymbolDictionary recovered = new GlobalSymbolDictionary();
+                reopened.addLoadedSymbolsTo(recovered);
+                Assert.assertEquals("direct recovery decode must preserve dense ids", 3, recovered.size());
+                Assert.assertEquals("GOOG", recovered.getSymbol(1));
 
                 // Appending after recovery continues from the recovered tip.
                 reopened.appendSymbol("TSLA");

From 5e467f3763dc1fc9a139bd735419ff780757728f Mon Sep 17 00:00:00 2001
From: glasstiger 
Date: Thu, 16 Jul 2026 15:52:07 +0100
Subject: [PATCH 78/96] fix: harden reconnect and test cleanup handling

---
 .../qwp/client/QwpWebSocketSender.java        |   2 +-
 .../client/sf/cursor/BackgroundDrainer.java   |  11 +-
 .../sf/cursor/CursorWebSocketSendLoop.java    | 220 +++++++++------
 .../cutlass/qwp/client/ReconnectTest.java     |  30 +--
 .../BackgroundDrainerDurableAckRetryTest.java |   4 +-
 ...SendLoopForegroundReconnectPolicyTest.java | 254 ++++++++++++++++++
 .../questdb/client/test/tools/TestUtils.java  |  85 +++++-
 .../client/test/tools/TestUtilsTest.java      |  65 +++++
 8 files changed, 551 insertions(+), 120 deletions(-)
 create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopForegroundReconnectPolicyTest.java
 create mode 100644 core/src/test/java/io/questdb/client/test/tools/TestUtilsTest.java

diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java
index 1fba1452..557af871 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java
@@ -3430,7 +3430,7 @@ private void ensureConnected() {
                     maxFrameRejections,
                     poisonMinEscalationWindowMillis,
                     catchUpCapGapMinEscalationWindowMillis,
-                    CursorWebSocketSendLoop.CatchUpCapGapPolicy.RETRY_FOREVER);
+                    CursorWebSocketSendLoop.ReconnectPolicy.FOREGROUND);
             // Plug the async-delivery sink before start() so the I/O thread
             // never observes a null dispatcher between recordFatal and
             // notification — the test for null in dispatchError handles
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/BackgroundDrainer.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/BackgroundDrainer.java
index 38b6b036..03ebee3e 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/BackgroundDrainer.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/BackgroundDrainer.java
@@ -243,8 +243,10 @@ public BackgroundDrainer() {
      * durable ack -- i.e. the symptom of a misconfigured cluster or a
      * rolling-upgrade transient.
      * 
      
-     * For the foreground sender that condition is loud-fail: the producer
-     * is actively pushing data. The drainer is asymmetric: source data is
+     * For a foreground sender's initial connection that condition is loud-fail;
+     * after the sender has been live, its reconnect loop keeps buffering and
+     * retrying through a rolling capability change. The drainer is asymmetric:
+     * source data is
      * pinned (durable-ack-mode trims only on STATUS_DURABLE_ACK frames,
      * which the offending endpoints by definition do not send), so we
      * give the cluster a budget to settle before quarantining the slot.
@@ -318,8 +320,7 @@ public WebSocketClient connectWithDurableAckRetry() {
             } catch (QwpAuthFailedException | WebSocketUpgradeException e) {
                 // Genuinely non-retriable across the cluster (auth 401/403, or a
                 // non-421 upgrade reject): waiting will not fix it, so quarantine
-                // immediately -- exactly as the live sender's background loop
-                // (CursorWebSocketSendLoop.connectLoop) halts on these errors.
+                // immediately under the orphan reconnect policy.
                 String msg = e.getMessage();
                 LOG.error("drainer terminal upgrade/auth error for slot {}: {}", slotPath, msg);
                 lastErrorMessage = msg;
@@ -639,7 +640,7 @@ public void run() {
                         maxHeadFrameRejections,
                         poisonMinEscalationWindowMillis,
                         catchUpCapGapMinEscalationWindowMillis,
-                        CursorWebSocketSendLoop.CatchUpCapGapPolicy.TERMINAL_AFTER_SETTLE_BUDGET);
+                        CursorWebSocketSendLoop.ReconnectPolicy.ORPHAN);
                 loop.start();
 
                 while (!stopRequestedOrInterrupted()) {
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java
index 3d33bfd0..e9af2d5d 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java
@@ -70,8 +70,10 @@
  *   
    6. On wire failure, runs the configured reconnect policy: capped
  *       exponential backoff with jitter, retried indefinitely (Invariant B --
  *       a store-and-forward drainer never gives up on a wall-clock budget),
- *       with only auth-style failures (401/403/non-101 upgrade reject) treated
- *       as terminal. On reconnect success, repositions the cursor at
+ *       with endpoint-policy failures terminal only for an initial foreground
+ *       connect or an orphan drainer. A previously-live foreground sender keeps
+ *       retrying so credential and rolling-capability changes remain contained
+ *       by store-and-forward. On reconnect success, repositions the cursor at
  *       {@code ackedFsn+1} and replays.
      7. 
  *
    * No locks on the steady-state path. The producer thread (user) writes @@ -225,6 +227,7 @@ public final class CursorWebSocketSendLoop implements QuietCloseable { // user-facing 5-minute default is applied at the config layer. private final long catchUpCapGapMinEscalationWindowNanos; private final CatchUpCapGapPolicy catchUpCapGapPolicy; + private final ReconnectPolicy reconnectPolicy; private final CursorSendEngine engine; private final long parkNanos; // FIFO of OK-acked batches awaiting durable-upload confirmation. Used only @@ -370,9 +373,8 @@ public final class CursorWebSocketSendLoop implements QuietCloseable { // Sticky flag: false until the very first time a live client is installed // (either via the constructor in SYNC/OFF mode or via swapClient on a // successful connect attempt in any mode). Once true, stays true. Used to - // distinguish a "never reached the server" terminal failure (looks like a - // config typo or firewall block) from "lost connection after we were - // up" (looks transient). + // distinguish an initial endpoint-policy failure (fail fast) from a + // post-start failure that a foreground sender must contain and retry. private volatile boolean hasEverConnected; private volatile Thread ioThread; // Typed marker for a durable-ack CAPABILITY-GAP terminal: set (before the @@ -381,8 +383,8 @@ public final class CursorWebSocketSendLoop implements QuietCloseable { // QwpDurableAckMismatchException. The orphan drainer consults it to route // a mid-drain capability gap into its budgeted settle-retry // (BackgroundDrainer.connectWithDurableAckRetry) instead of quarantining - // the slot on the first sweep; the foreground sender ignores it and keeps - // its spec'd loud-fail (sf-client.md section 8.1). Write-once alongside + // the slot on the first sweep. Foreground reconnects never set this marker; + // they keep retrying after a successful initial connection. Write-once alongside // terminalError: the only writer runs on the I/O thread under the same // first-writer-wins latch. private volatile QwpDurableAckMismatchException capabilityGapTerminal; @@ -512,9 +514,9 @@ public final class CursorWebSocketSendLoop implements QuietCloseable { * {@code client} may be {@code null} only if {@code reconnectFactory} * is non-null — this is the async-initial-connect path: the I/O thread * runs the same retry loop on its first iteration to obtain a live - * client, and a terminal failure (auth/upgrade reject) is delivered - * through the dispatcher rather than thrown to the constructor's - * caller; plain connect failures are retried indefinitely + * client, and an initial terminal failure (auth/upgrade reject or durable-ack + * mismatch) is delivered through the dispatcher rather than thrown to the + * constructor's caller; plain connect failures are retried indefinitely * (Invariant B: no wall-clock budget give-up). */ public CursorWebSocketSendLoop(WebSocketClient client, CursorSendEngine engine, @@ -656,10 +658,9 @@ public CursorWebSocketSendLoop(WebSocketClient client, CursorSendEngine engine, } /** - * Policy-aware master constructor. Foreground senders must pass - * {@link CatchUpCapGapPolicy#RETRY_FOREVER}; only orphan drainers may pass - * {@link CatchUpCapGapPolicy#TERMINAL_AFTER_SETTLE_BUDGET} and quarantine a slot - * after the attempt and dwell thresholds are both exhausted. + * Compatibility policy-aware constructor. New production call sites should + * use the {@link ReconnectPolicy} overload below, which names the foreground + * versus orphan ownership distinction directly. */ public CursorWebSocketSendLoop(WebSocketClient client, CursorSendEngine engine, long fsnAtZero, long parkNanos, @@ -696,6 +697,9 @@ public CursorWebSocketSendLoop(WebSocketClient client, CursorSendEngine engine, throw new IllegalArgumentException("catchUpCapGapPolicy must be non-null"); } this.catchUpCapGapPolicy = catchUpCapGapPolicy; + this.reconnectPolicy = catchUpCapGapPolicy == CatchUpCapGapPolicy.RETRY_FOREVER + ? ReconnectPolicy.FOREGROUND + : ReconnectPolicy.ORPHAN; if (engine == null) { throw new IllegalArgumentException("engine must be non-null"); } @@ -831,6 +835,41 @@ public CursorWebSocketSendLoop(WebSocketClient client, CursorSendEngine engine, } } + /** + * Policy-aware master constructor. A foreground sender fails fast while + * establishing its first connection, then retries endpoint-policy failures + * indefinitely after it has been live. An orphan drainer returns such failures + * to its owner so the slot can follow its settle/quarantine policy. + */ + public CursorWebSocketSendLoop(WebSocketClient client, CursorSendEngine engine, + long fsnAtZero, long parkNanos, + ReconnectFactory reconnectFactory, + long reconnectMaxDurationMillis, + long reconnectInitialBackoffMillis, + long reconnectMaxBackoffMillis, + boolean durableAckMode, + long durableAckKeepaliveIntervalMillis, + int maxHeadFrameRejections, + long poisonMinEscalationWindowMillis, + long catchUpCapGapMinEscalationWindowMillis, + ReconnectPolicy reconnectPolicy) { + this(client, engine, fsnAtZero, parkNanos, reconnectFactory, + reconnectMaxDurationMillis, reconnectInitialBackoffMillis, + reconnectMaxBackoffMillis, durableAckMode, + durableAckKeepaliveIntervalMillis, maxHeadFrameRejections, + poisonMinEscalationWindowMillis, catchUpCapGapMinEscalationWindowMillis, + catchUpPolicyFor(reconnectPolicy)); + } + + private static CatchUpCapGapPolicy catchUpPolicyFor(ReconnectPolicy reconnectPolicy) { + if (reconnectPolicy == null) { + throw new IllegalArgumentException("reconnectPolicy must be non-null"); + } + return reconnectPolicy == ReconnectPolicy.FOREGROUND + ? CatchUpCapGapPolicy.RETRY_FOREVER + : CatchUpCapGapPolicy.TERMINAL_AFTER_SETTLE_BUDGET; + } + /** * Maps a server status byte to a {@link SenderError.Category}. Exposed for unit tests. */ @@ -1487,13 +1526,15 @@ private void connectLoop(Throwable initial, String phase, long paceFirstAttemptM LOG.warn("cursor I/O loop entering {} loop: {}", phase, initial.getMessage()); long outageStartNanos = System.nanoTime(); - // INVARIANT B: a store-and-forward drainer must NEVER terminate on a + // INVARIANT B: a store-and-forward loop must NEVER terminate on a // wall-clock reconnect budget. A replica-only / all-endpoints-replica // window is TRANSIENT -- a replica gets promoted, a primary reappears -- // so this background loop retries for as long as it is running, backing - // off between attempts. The ONLY terminal conditions are a genuinely - // non-retriable upgrade (auth / non-421 upgrade / durable-ack capability - // gap), which return directly below, or the sender being stopped. SF + // off between attempts. Endpoint-policy failures (auth / non-421 + // upgrade / durable-ack capability gap) are terminal only for orphan + // drainers and a foreground sender's first connection. Once a foreground + // sender has connected, those states are retried so a credential or cluster + // capability rotation cannot stop its producer. SF // exhaustion is surfaced to the PRODUCER as append backpressure, never // here. reconnect_max_duration_millis is intentionally NOT consulted: it // bounds only the blocking (non-lazy) initial connect in @@ -1557,65 +1598,78 @@ private void connectLoop(Throwable initial, String phase, long paceFirstAttemptM // observation. Do not let the time spent retrying it satisfy orphan dwell. resetCatchUpCapGapEpisode(); } catch (QwpAuthFailedException | WebSocketUpgradeException e) { - // Terminal across all configured endpoints per spec sf-client.md - // section 13.3: auth (401/403) bypasses reconnect and surfaces as - // SECURITY_ERROR. WebSocketUpgradeException reaching here is always - // non-421: QwpUpgradeFailures.classify upstream converts a - // 421-with-X-QuestDB-Role to QwpIngressRoleRejectedException, and a - // 421 without that header walks the transport-error path in - // buildAndConnect and lands as a LineSenderException, falling into - // the Throwable branch below. - LOG.error("terminal upgrade error during {} -- won't retry: {}", - phase, e.getMessage()); - long fromFsn = engine.ackedFsn() + 1L; - long toFsn = Math.max(fromFsn, engine.publishedFsn()); - SenderError err = new SenderError( - SenderError.Category.SECURITY_ERROR, - SenderError.Policy.TERMINAL, - SenderError.NO_STATUS_BYTE, - "ws-upgrade-failed: " + e.getMessage(), - SenderError.NO_MESSAGE_SEQUENCE, - fromFsn, - toFsn, - null, - System.nanoTime() - ); - totalServerErrors.incrementAndGet(); - recordFatal(new LineSenderServerException(err)); - dispatchError(err); - return; + if (endpointPolicyFailureIsTerminal()) { + // Orphans return control to their quarantine owner; a + // foreground sender that never connected still fails fast. + // WebSocketUpgradeException reaching here is always non-421: + // role rejects are classified into the transient branch below. + LOG.error("terminal upgrade error during {} -- won't retry: {}", + phase, e.getMessage()); + long fromFsn = engine.ackedFsn() + 1L; + long toFsn = Math.max(fromFsn, engine.publishedFsn()); + SenderError err = new SenderError( + SenderError.Category.SECURITY_ERROR, + SenderError.Policy.TERMINAL, + SenderError.NO_STATUS_BYTE, + "ws-upgrade-failed: " + e.getMessage(), + SenderError.NO_MESSAGE_SEQUENCE, + fromFsn, + toFsn, + null, + System.nanoTime() + ); + totalServerErrors.incrementAndGet(); + recordFatal(new LineSenderServerException(err)); + dispatchError(err); + return; + } + resetCatchUpCapGapEpisode(); + lastReconnectError = e; + long now = System.nanoTime(); + if (now - lastLogNanos >= RECONNECT_LOG_THROTTLE_NANOS) { + LOG.warn("{} attempt {}: foreground auth/upgrade policy rejected the connection; " + + "retrying because this sender was previously live -- {}", + phase, attempts, e.getMessage()); + lastLogNanos = now; + } } catch (QwpDurableAckMismatchException e) { - // Per spec sf-client.md section 8.1: the client opted into durable - // ack but the cluster cannot honour it. Loud fail at connect rather - // than silently waiting for ack frames that will never arrive. - // Classified as PROTOCOL_VIOLATION (config/capability mismatch), - // not SECURITY_ERROR -- this is not an auth failure. - LOG.error("durable-ack mismatch during {} -- won't retry: {}", - phase, e.getMessage()); - if (terminalError == null) { - // Mirror recordFatal's first-writer-wins latch: only the - // sweep that owns the terminal may mark the gap, and the - // marker must be visible before the terminalError volatile - // write that checkError() keys on. - capabilityGapTerminal = e; + if (endpointPolicyFailureIsTerminal()) { + // Orphans hand a capability gap back to BackgroundDrainer's + // settle budget. An initial foreground connect remains loud. + LOG.error("durable-ack mismatch during {} -- won't retry: {}", + phase, e.getMessage()); + if (terminalError == null) { + // Publish the marker before terminalError, which is the + // volatile first-writer-wins latch observed by the owner. + capabilityGapTerminal = e; + } + long fromFsn = engine.ackedFsn() + 1L; + long toFsn = Math.max(fromFsn, engine.publishedFsn()); + SenderError err = new SenderError( + SenderError.Category.PROTOCOL_VIOLATION, + SenderError.Policy.TERMINAL, + SenderError.NO_STATUS_BYTE, + "durable-ack-mismatch: " + e.getMessage(), + SenderError.NO_MESSAGE_SEQUENCE, + fromFsn, + toFsn, + null, + System.nanoTime() + ); + totalServerErrors.incrementAndGet(); + recordFatal(new LineSenderServerException(err)); + dispatchError(err); + return; + } + resetCatchUpCapGapEpisode(); + lastReconnectError = e; + long now = System.nanoTime(); + if (now - lastLogNanos >= RECONNECT_LOG_THROTTLE_NANOS) { + LOG.warn("{} attempt {}: foreground durable-ack capability is temporarily " + + "unavailable; retrying because this sender was previously live -- {}", + phase, attempts, e.getMessage()); + lastLogNanos = now; } - long fromFsn = engine.ackedFsn() + 1L; - long toFsn = Math.max(fromFsn, engine.publishedFsn()); - SenderError err = new SenderError( - SenderError.Category.PROTOCOL_VIOLATION, - SenderError.Policy.TERMINAL, - SenderError.NO_STATUS_BYTE, - "durable-ack-mismatch: " + e.getMessage(), - SenderError.NO_MESSAGE_SEQUENCE, - fromFsn, - toFsn, - null, - System.nanoTime() - ); - totalServerErrors.incrementAndGet(); - recordFatal(new LineSenderServerException(err)); - dispatchError(err); - return; } catch (QwpRoleMismatchException | QwpIngressRoleRejectedException e) { // Role mismatch: every reachable endpoint role-rejected the // upgrade -- right now they are all replicas / primary-catchup. @@ -1704,6 +1758,10 @@ private void connectLoop(Throwable initial, String phase, long paceFirstAttemptM phase, elapsedMs, attempts, lastMsg); } + private boolean endpointPolicyFailureIsTerminal() { + return reconnectPolicy == ReconnectPolicy.ORPHAN || !hasEverConnected; + } + /** * Send {@code err} to the async-delivery dispatcher if one is configured. * Producer-side typed throw (TERMINAL) goes through {@code recordFatal} + @@ -2949,13 +3007,21 @@ public enum CatchUpCapGapPolicy { TERMINAL_AFTER_SETTLE_BUDGET } + /** Identifies who owns data while the reconnect loop is unavailable. */ + public enum ReconnectPolicy { + /** A live producer keeps buffering and retries endpoint-policy failures. */ + FOREGROUND, + /** An orphan drainer returns terminal states to its quarantine owner. */ + ORPHAN + } + /** * Factory used by the I/O loop to build a fresh, connected, upgraded * {@link WebSocketClient} after a wire failure. Implementations close * the old client (if needed), build a new one with the same auth/TLS * config, connect, perform the WebSocket upgrade, and return it ready - * to send. Throw on a terminal failure (auth rejection, etc.) — the - * I/O loop will treat the throw as fatal. + * to send. The loop's {@link ReconnectPolicy} decides whether endpoint-policy + * failures are retried or returned as terminal. */ @FunctionalInterface public interface ReconnectFactory { diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/ReconnectTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/ReconnectTest.java index 7ba7a1fe..72c8c021 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/ReconnectTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/ReconnectTest.java @@ -25,7 +25,6 @@ package io.questdb.client.test.cutlass.qwp.client; import io.questdb.client.Sender; -import io.questdb.client.cutlass.line.LineSenderException; import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer; import org.junit.Assert; import org.junit.Test; @@ -203,14 +202,13 @@ public void testReconnectNeverGivesUpInvariantB() throws Exception { } @Test - public void testTerminalUpgradeErrorAbortsReconnect() throws Exception { + public void testPostStartAuthErrorRemainsBuffered() throws Exception { // Bespoke raw-socket fixture: first connection completes the // WebSocket upgrade and feeds back STATUS_OK ACKs; any subsequent // connection gets HTTP 401 Unauthorized — exercising the - // auth-terminal path. With reconnect_max_duration_millis=10s and - // a 401 happening on the very first reconnect, the cursor I/O - // loop should surface the terminal error within hundreds of ms, - // not after 10s. + // post-start auth-rotation path. Once a foreground sender has been + // live, store-and-forward owns its unacked data: repeated 401s must + // remain contained and retried rather than stopping the producer. try (Auth401AfterFirstConnectionFixture fixture = new Auth401AfterFirstConnectionFixture()) { fixture.start(); @@ -224,9 +222,8 @@ public void testTerminalUpgradeErrorAbortsReconnect() throws Exception { // Wait for first connection to ACK + close waitFor(() -> fixture.acceptedConnections.get() >= 2, 5_000); - long t0 = System.nanoTime(); Throwable observed = null; - long deadline = System.currentTimeMillis() + 5_000; + long deadline = System.currentTimeMillis() + 750; while (System.currentTimeMillis() < deadline) { try { sender.table("foo").longColumn("v", 2L).atNow(); @@ -237,23 +234,10 @@ public void testTerminalUpgradeErrorAbortsReconnect() throws Exception { } Thread.sleep(50); } - long elapsedMs = (System.nanoTime() - t0) / 1_000_000L; - Assert.assertNotNull("expected terminal error after auth rejection", + Assert.assertNull("post-start auth rejection must stay buffered and retriable", observed); - Assert.assertTrue( - "terminal upgrade error must surface well inside the cap; took " - + elapsedMs + "ms (cap was 10000ms)", - elapsedMs < 5_000); - String msg = observed.getMessage() == null ? "" : observed.getMessage(); - Assert.assertTrue( - "error must mention the terminal upgrade failure: " + msg, - msg.contains("WebSocket upgrade failed") - || msg.contains("I/O thread failed") - || msg.contains("401")); - } catch (LineSenderException ignored) { + waitFor(() -> fixture.acceptedConnections.get() >= 3, 5_000); } - // close() rethrows the latched terminal upgrade error - // (commit 052f6ee). Already observed and asserted above. } } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/BackgroundDrainerDurableAckRetryTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/BackgroundDrainerDurableAckRetryTest.java index 0a6b69f8..82885ba7 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/BackgroundDrainerDurableAckRetryTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/BackgroundDrainerDurableAckRetryTest.java @@ -241,8 +241,8 @@ public void testTerminalUpgradeMarksFailedImmediately() throws Exception { CountingListener listener = new CountingListener(); // A genuinely non-retriable upgrade error (non-421 5xx upgrade reject) is // terminal -- waiting will not fix it -- so the drainer quarantines on the - // first attempt, exactly like the live sender's background loop halts on - // auth/upgrade. A TRANSPORT error, by contrast, is transient and is + // first attempt under the orphan reconnect policy. A TRANSPORT error, + // by contrast, is transient and is // retried (see testTransportErrorNeverQuarantinesInvariantB). ScriptedFactory factory = ScriptedFactory.alwaysFailing( () -> new WebSocketUpgradeException(500, null, "server error during upgrade")); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopForegroundReconnectPolicyTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopForegroundReconnectPolicyTest.java new file mode 100644 index 00000000..064edf3f --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopForegroundReconnectPolicyTest.java @@ -0,0 +1,254 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client.sf.cursor; + +import io.questdb.client.cutlass.http.client.WebSocketClient; +import io.questdb.client.cutlass.http.client.WebSocketClientFactory; +import io.questdb.client.cutlass.qwp.client.QwpAuthFailedException; +import io.questdb.client.cutlass.qwp.client.QwpDurableAckMismatchException; +import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine; +import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorWebSocketSendLoop; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.Unsafe; +import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer; +import io.questdb.client.test.tools.TestUtils; +import org.junit.Assert; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.charset.StandardCharsets; +import java.util.IdentityHashMap; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +public class CursorWebSocketSendLoopForegroundReconnectPolicyTest { + private static final long SEGMENT_SIZE_BYTES = 16_384L; + + @Rule + public final TemporaryFolder sfDir = TemporaryFolder.builder().assureDeletion().build(); + + @Test + public void testPostStartAuthFailureRetriesUntilCredentialsRecover() throws Exception { + assertForegroundRecovers(false, + () -> new QwpAuthFailedException(401, "localhost", 1)); + } + + @Test + public void testPostStartDurableAckMismatchRetriesUntilCapabilityRecovers() throws Exception { + assertForegroundRecovers(true, + () -> new QwpDurableAckMismatchException("localhost", 1, "primary")); + } + + private void assertForegroundRecovers(boolean durableAck, FailureSupplier failureSupplier) throws Exception { + TestUtils.assertMemoryLeak(() -> { + DropFirstConnectionHandler handler = new DropFirstConnectionHandler(durableAck); + try (TestWebSocketServer server = new TestWebSocketServer(handler, true); + CursorSendEngine engine = new CursorSendEngine( + sfDir.newFolder().getAbsolutePath(), SEGMENT_SIZE_BYTES)) { + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + + WebSocketClient initialClient = connect(server.getPort(), durableAck); + ScriptedFactory factory = new ScriptedFactory( + server.getPort(), durableAck, failureSupplier); + CursorWebSocketSendLoop loop = new CursorWebSocketSendLoop( + initialClient, + engine, + 0L, + CursorWebSocketSendLoop.DEFAULT_PARK_NANOS, + factory, + 100L, + 1L, + 4L, + durableAck, + durableAck ? 10L : 0L, + CursorWebSocketSendLoop.DEFAULT_MAX_HEAD_FRAME_REJECTIONS, + 0L, + 0L, + CursorWebSocketSendLoop.ReconnectPolicy.FOREGROUND); + try { + appendFrame(engine, (byte) 1); + loop.start(); + + await(() -> loop.getTotalReconnects() >= 1L + || loop.getTerminalError() != null, + "foreground reconnect did not complete"); + Assert.assertNull("post-start endpoint-policy failures must not stop the producer", + loop.getTerminalError()); + Assert.assertTrue("the reconnect loop must retry both scripted failures", + factory.attempts() >= 3); + + long target = appendFrame(engine, (byte) 2); + await(() -> engine.ackedFsn() >= target || loop.getTerminalError() != null, + "foreground sender did not deliver after the policy failure cleared"); + Assert.assertNull(loop.getTerminalError()); + Assert.assertEquals(target, engine.ackedFsn()); + } finally { + loop.close(); + initialClient.close(); + } + } + }); + } + + private static long appendFrame(CursorSendEngine engine, byte marker) { + long buffer = Unsafe.malloc(16, MemoryTag.NATIVE_DEFAULT); + try { + for (int i = 0; i < 16; i++) { + Unsafe.getUnsafe().putByte(buffer + i, marker); + } + return engine.appendBlocking(buffer, 16); + } finally { + Unsafe.free(buffer, 16, MemoryTag.NATIVE_DEFAULT); + } + } + + private static void await(Condition condition, String failureMessage) throws Exception { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(10); + while (!condition.isTrue()) { + if (System.nanoTime() >= deadline) { + Assert.fail(failureMessage); + } + Thread.sleep(1L); + } + } + + private static WebSocketClient connect(int port, boolean durableAck) throws Exception { + WebSocketClient client = WebSocketClientFactory.newPlainTextInstance(); + try { + client.setQwpMaxVersion(1); + client.setQwpRequestDurableAck(durableAck); + client.setConnectTimeout(5_000); + client.connect("localhost", port); + client.upgrade("/write/v4", 5_000, null); + return client; + } catch (Throwable e) { + client.close(); + throw e; + } + } + + @FunctionalInterface + private interface Condition { + boolean isTrue() throws Exception; + } + + @FunctionalInterface + private interface FailureSupplier { + Exception get(); + } + + private static final class ScriptedFactory implements CursorWebSocketSendLoop.ReconnectFactory { + private final AtomicInteger attempts = new AtomicInteger(); + private final boolean durableAck; + private final FailureSupplier failureSupplier; + private final int port; + + private ScriptedFactory(int port, boolean durableAck, FailureSupplier failureSupplier) { + this.port = port; + this.durableAck = durableAck; + this.failureSupplier = failureSupplier; + } + + int attempts() { + return attempts.get(); + } + + @Override + public WebSocketClient reconnect() throws Exception { + if (attempts.incrementAndGet() <= 2) { + throw failureSupplier.get(); + } + return connect(port, durableAck); + } + } + + private static final class DropFirstConnectionHandler + implements TestWebSocketServer.WebSocketServerHandler { + private static final String TABLE = "trades"; + private final boolean durableAck; + private final Map wireSeqByConnection = + new IdentityHashMap<>(); + private TestWebSocketServer.ClientHandler firstConnection; + + private DropFirstConnectionHandler(boolean durableAck) { + this.durableAck = durableAck; + } + + @Override + public synchronized void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] data) { + long[] sequence = wireSeqByConnection.get(client); + if (sequence == null) { + sequence = new long[1]; + wireSeqByConnection.put(client, sequence); + if (firstConnection == null) { + firstConnection = client; + } + } + long wireSeq = sequence[0]++; + try { + client.sendBinary(okFrame(wireSeq)); + if (durableAck) { + client.sendBinary(durableAckFrame(wireSeq)); + } + if (client == firstConnection) { + client.close(); + } + } catch (IOException ignored) { + // The deliberate close may race the acknowledgement write. + } + } + + private static byte[] durableAckFrame(long seqTxn) { + byte[] name = TABLE.getBytes(StandardCharsets.UTF_8); + ByteBuffer bb = ByteBuffer.allocate(1 + 2 + 2 + name.length + 8) + .order(ByteOrder.LITTLE_ENDIAN); + bb.put((byte) 0x02); + bb.putShort((short) 1); + bb.putShort((short) name.length); + bb.put(name); + bb.putLong(seqTxn); + return bb.array(); + } + + private static byte[] okFrame(long wireSeq) { + byte[] name = TABLE.getBytes(StandardCharsets.UTF_8); + ByteBuffer bb = ByteBuffer.allocate(1 + 8 + 2 + 2 + name.length + 8) + .order(ByteOrder.LITTLE_ENDIAN); + bb.put((byte) 0x00); + bb.putLong(wireSeq); + bb.putShort((short) 1); + bb.putShort((short) name.length); + bb.put(name); + bb.putLong(wireSeq); + return bb.array(); + } + } +} diff --git a/core/src/test/java/io/questdb/client/test/tools/TestUtils.java b/core/src/test/java/io/questdb/client/test/tools/TestUtils.java index 39755426..82f65e6b 100644 --- a/core/src/test/java/io/questdb/client/test/tools/TestUtils.java +++ b/core/src/test/java/io/questdb/client/test/tools/TestUtils.java @@ -37,9 +37,15 @@ import org.junit.Assert; import org.slf4j.Logger; +import java.io.IOException; import java.net.URISyntaxException; import java.net.URL; +import java.nio.file.FileVisitResult; +import java.nio.file.NoSuchFileException; +import java.nio.file.Path; import java.nio.file.Paths; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; import java.util.Arrays; import static org.junit.Assert.assertNotNull; @@ -228,23 +234,78 @@ public static void removeTmpDirRec(String tmpDir) { if (tmpDir == null) { return; } - java.nio.file.Path root = java.nio.file.Paths.get(tmpDir); - if (!java.nio.file.Files.exists(root)) { - return; - } - // try-with-resources: the walk Stream holds an open directory handle that - // must be closed, or each call leaks a descriptor. - try (java.util.stream.Stream walk = java.nio.file.Files.walk(root)) { - walk.sorted(java.util.Comparator.reverseOrder()).forEach(p -> { - try { - java.nio.file.Files.deleteIfExists(p); - } catch (java.io.IOException ignored) { + removeTmpDirRec(Paths.get(tmpDir), java.nio.file.Files::deleteIfExists); + } + + /** + * Injectable counterpart used to verify that cleanup remains best-effort but + * still fails the test when any path cannot be removed. + */ + static void removeTmpDirRec(Path root, PathDeleter deleter) { + final IOException[] firstFailure = new IOException[1]; + try { + java.nio.file.Files.walkFileTree(root, new SimpleFileVisitor() { + @Override + public FileVisitResult postVisitDirectory(Path dir, IOException exc) { + remember(exc); + delete(dir); + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) { + delete(file); + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult visitFileFailed(Path file, IOException exc) { + remember(exc); + return FileVisitResult.CONTINUE; + } + + private void delete(Path path) { + try { + deleter.delete(path); + } catch (IOException e) { + remember(e); + } + } + + private void remember(IOException failure) { + if (failure == null || failure instanceof NoSuchFileException) { + // The cleanup goal for this path is already satisfied. This + // also covers a child disappearing between directory listing + // and attribute lookup. + return; + } + if (firstFailure[0] == null) { + firstFailure[0] = failure; + } else if (firstFailure[0] != failure) { + firstFailure[0].addSuppressed(failure); + } } }); - } catch (java.io.IOException ignored) { + } catch (NoSuchFileException ignored) { + // A missing root is an already-complete cleanup. + } catch (IOException e) { + if (firstFailure[0] == null) { + firstFailure[0] = e; + } else { + firstFailure[0].addSuppressed(e); + } + } + if (firstFailure[0] != null) { + throw new AssertionError("could not recursively remove temp directory: " + root, + firstFailure[0]); } } + @FunctionalInterface + interface PathDeleter { + void delete(Path path) throws IOException; + } + /** * Java 8 stand-in for {@code String.repeat(int)} (added in Java 11). */ diff --git a/core/src/test/java/io/questdb/client/test/tools/TestUtilsTest.java b/core/src/test/java/io/questdb/client/test/tools/TestUtilsTest.java new file mode 100644 index 00000000..61cee092 --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/tools/TestUtilsTest.java @@ -0,0 +1,65 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.tools; + +import org.junit.Assert; +import org.junit.Test; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.concurrent.atomic.AtomicInteger; + +public class TestUtilsTest { + + @Test + public void testRecursiveCleanupRetainsFirstFailureAndContinuesDeleting() throws Exception { + Path root = Files.createTempDirectory("qdb-test-utils-cleanup-"); + Path child = Files.createDirectory(root.resolve("child")); + Path file = Files.createFile(child.resolve("blocked")); + IOException injected = new IOException("simulated deletion failure"); + AtomicInteger attempts = new AtomicInteger(); + try { + try { + TestUtils.removeTmpDirRec(root, path -> { + attempts.incrementAndGet(); + if (path.equals(file)) { + throw injected; + } + Files.deleteIfExists(path); + }); + Assert.fail("cleanup must report the injected deletion failure"); + } catch (AssertionError e) { + Assert.assertSame("the first deletion failure must be retained", injected, e.getCause()); + Assert.assertTrue("cleanup must continue with parent directories after a file failure", + attempts.get() >= 3); + Assert.assertTrue("later parent deletion failures must remain diagnostic", + e.getCause().getSuppressed().length >= 2); + } + } finally { + TestUtils.removeTmpDirRec(root.toString()); + } + } +} From 994554e909ceccc22f13bbc535f4dacfdad1bd7f Mon Sep 17 00:00:00 2001 From: glasstiger Date: Fri, 17 Jul 2026 10:50:25 +0100 Subject: [PATCH 79/96] fix: preserve QWP store-and-forward retries --- .../main/java/io/questdb/client/Sender.java | 26 ++--- .../qwp/client/QwpWebSocketSender.java | 13 +-- .../client/sf/cursor/BackgroundDrainer.java | 49 +++----- .../sf/cursor/CursorWebSocketSendLoop.java | 59 +++++----- .../qwp/client/CloseSafetyNetTest.java | 109 ++++++++++++------ .../qwp/client/InitialConnectAsyncTest.java | 96 +++++---------- .../BackgroundDrainerDurableAckRetryTest.java | 38 +++--- .../sf/cursor/CloseOwnershipRaceTest.java | 20 ++-- ...SendLoopForegroundReconnectPolicyTest.java | 106 ++++++++++++++++- 9 files changed, 299 insertions(+), 217 deletions(-) diff --git a/core/src/main/java/io/questdb/client/Sender.java b/core/src/main/java/io/questdb/client/Sender.java index be4fe409..837b1d7b 100644 --- a/core/src/main/java/io/questdb/client/Sender.java +++ b/core/src/main/java/io/questdb/client/Sender.java @@ -794,12 +794,9 @@ default Sender uuidColumn(CharSequence name, long lo, long hi) { * unconnected sender; the I/O thread runs the same retry loop in * the background. The user thread can call {@code at()} / * {@code flush()} immediately; rows accumulate in the cursor SF - * engine until the wire is up. Connect failures are retried - * indefinitely in the background; a terminal upgrade failure - * (auth reject, capability mismatch) is delivered to the async - * error inbox as a {@link io.questdb.client.SenderError} (no - * synchronous throw on the user call site). Wire - * {@code error_handler=...} to observe these.
    • + * engine until the wire is up. Transport, auth, upgrade and + * capability failures are retried indefinitely in the background; + * none is surfaced to producer calls or the async error inbox. *
*

* Default resolution when the caller does not pick a value: @@ -1077,8 +1074,8 @@ final class LineSenderBuilder { // explicitly", which build() resolves to SYNC when any reconnect_* // knob was tuned by the user, otherwise OFF. SYNC retries on the // user thread up to the reconnect cap. ASYNC returns immediately - // and lets the I/O thread retry in the background, surfacing - // terminal failures via the error inbox. + // and lets the I/O thread retry every endpoint/transport failure in + // the background without surfacing it to the producer. private InitialConnectMode initialConnectMode = null; private String keyId; private int maxBackgroundDrainers = DEFAULT_MAX_BACKGROUND_DRAINERS; @@ -2246,11 +2243,10 @@ public LineSenderBuilder initialConnectMode(InitialConnectMode mode) { } /** - * Opt in to retrying the initial connect with the same backoff / - * cap / auth-terminal policy as in-flight reconnect. Set true if - * your deployment expects the server to come up shortly after the - * sender. Auth failures (HTTP 401/403/non-101) stay terminal in - * either mode. + * Opt in to retrying the initial connect with backoff on the calling + * thread, bounded by the configured reconnect cap. Set true if your + * deployment expects the server to come up shortly after the sender. + * Auth failures (HTTP 401/403/non-101) stay terminal in this SYNC mode. *

* When this method is not called, the resolution rule documented * on {@link InitialConnectMode} applies: SYNC implicitly when any @@ -2591,8 +2587,8 @@ public LineSenderBuilder reconnectMaxBackoffMillis(long millis) { * with exponential backoff until connect succeeds or this many * millis elapse, then throws. The background reconnect loop * (mid-stream outages and async initial connect) does NOT consult - * this value: it retries indefinitely and halts only on a terminal - * auth/upgrade error or {@code close()}. + * this value: endpoint and transport failures are retried indefinitely + * until {@code close()}. *

* Default {@code 300_000} (5 minutes). Lower for fail-fast startup; * higher for tolerating a slow server boot. Must be positive: a zero diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index 557af871..2b1971e4 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -290,9 +290,8 @@ public class QwpWebSocketSender implements Sender { // still terminal. // ASYNC → user thread does not connect at all. The I/O thread runs // the reconnect loop in the background, indefinitely - // (Invariant B); terminal failures (auth/upgrade reject) - // are delivered to the SenderError dispatcher rather than - // thrown from the constructor. + // (Invariant B); endpoint-policy and transport failures stay + // contained in that loop and never reach the producer. private Sender.InitialConnectMode initialConnectMode = Sender.InitialConnectMode.OFF; private boolean ownsCursorEngine; private long pendingBytes; @@ -3399,10 +3398,10 @@ private void ensureConnected() { // Encoder stays at its default (V1 -- the only supported wire // version today). Frames written before the first successful // connect commit to V1 because cursor segments are immutable; - // a future version bump must account for that. Auth/upgrade - // rejects are surfaced via the error inbox by the I/O - // thread, not thrown here; plain connect failures retry - // indefinitely (Invariant B). + // a future version bump must account for that. Auth, upgrade, + // capability and transport failures all retry indefinitely on + // the I/O thread (Invariant B); none is surfaced here or later + // to the producer. client = null; break; case OFF: diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/BackgroundDrainer.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/BackgroundDrainer.java index 03ebee3e..38e7c4ee 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/BackgroundDrainer.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/BackgroundDrainer.java @@ -77,12 +77,10 @@ public final class BackgroundDrainer implements Runnable { * wall-clock budget {@code reconnectMaxDurationMillis} also caps this * capability-gap loop; whichever is hit first triggers escalation. Both * halves of the budget measure a capability-gap episode: the - * wall clock accumulates only across uninterrupted gap-to-gap intervals - * (never before the first gap is observed, and never across an - * intervening transport window -- an unreachable cluster is not - * "failing to settle"), and an intervening role reject restarts the - * episode -- it proves the topology changed, so the next capability-gap - * error is a fresh episode against a newly promoted node. 16 + * wall clock accumulates only across uninterrupted gap-to-gap intervals. + * Any intervening transport or role state restarts the episode: capability + * gaps separated by an unrelated transient are not consecutive evidence + * of a persistent cluster capability mismatch. 16 * attempts gives the cluster room to settle through a rolling upgrade * (each attempt walks every endpoint internally) without letting a genuine * cluster-wide misconfig hang the drainer forever. @@ -243,10 +241,9 @@ public BackgroundDrainer() { * durable ack -- i.e. the symptom of a misconfigured cluster or a * rolling-upgrade transient. *

- * For a foreground sender's initial connection that condition is loud-fail; - * after the sender has been live, its reconnect loop keeps buffering and - * retrying through a rolling capability change. The drainer is asymmetric: - * source data is + * Blocking foreground startup keeps its fail-fast policy, while asynchronous + * foreground startup and reconnect keep buffering and retrying through a + * rolling capability change. The orphan drainer is asymmetric: source data is * pinned (durable-ack-mode trims only on STATUS_DURABLE_ACK frames, * which the offending endpoints by definition do not send), so we * give the cluster a budget to settle before quarantining the slot. @@ -259,11 +256,8 @@ public BackgroundDrainer() { * {@link QwpDurableAckMismatchException} sweeps only. Transient * conditions -- an all-replica failover window (role reject) or a * transport error -- are retried indefinitely (Invariant B) and never - * consume the budget: the wall-clock half accumulates only across - * uninterrupted gap-to-gap intervals, so a mid-episode transport window - * pauses the clock (without touching the attempt count), and a role - * reject additionally restarts the episode, because it proves the - * topology changed under the rolling upgrade. + * consume the budget. Either transient restarts the attempt count and wall + * clock so only uninterrupted capability-gap sweeps can escalate. * Genuine terminals (auth failure, non-421 upgrade reject) preserve * the original behavior: mark failed, exit. * @@ -281,23 +275,17 @@ public WebSocketClient connectWithDurableAckRetry() { // QwpDurableAckMismatchException sweeps; the wall-clock half // accumulates ONLY across uninterrupted gap-to-gap intervals, so // transient churn (role reject, transport) can never burn the budget - // -- neither before the first gap is observed nor mid-episode (a - // cluster unreachable for longer than the whole budget that comes - // back still gapped has consumed none of it). An intervening role - // reject resets the episode (topology churn: the offending node is - // gone); a transport error neither increments nor resets the attempt - // count -- a dropped socket does not prove promotion churn, and - // resetting on it would let a flaky-but-misconfigured cluster evade - // the cap forever -- it only pauses the wall clock: the gap-to-gap - // interval spanning the transport window is not charged. + // -- neither before the first gap is observed nor mid-episode. Any + // intervening role or transport state resets the episode: after the + // cluster leaves the capability-gap state, later gaps must establish a + // fresh consecutive run before quarantine is permitted. int capabilityGapAttempts = 0; // Wall-clock time accumulated across uninterrupted gap-to-gap // intervals of the current episode; escalates once it reaches // capabilityGapBudgetNanos (or the attempt cap fires first). long capabilityGapElapsedNanos = 0L; // Timestamp of the previous capability-gap sweep; 0 = the next gap - // charges nothing (episode start, post-role-reject restart, or the - // interval was interrupted by a transport window). + // charges nothing because a fresh episode is starting. long lastCapabilityGapNanos = 0L; final long capabilityGapBudgetNanos = reconnectMaxDurationMillis * 1_000_000L; // Observability-only counter for the transient all-replica window; @@ -433,10 +421,11 @@ public WebSocketClient connectWithDurableAckRetry() { // Invariant B -- but it is NOT a transport outage, so log it // truthfully below rather than mislabelling it "cluster unreachable". lastErrorMessage = t.getMessage(); - // Pause the episode wall clock: the gap-to-gap interval this - // window interrupts is never charged. Attempts and elapsed - // already accumulated are preserved (anti-evasion: see the - // budget comment above). + // This unrelated state breaks the consecutive capability-gap + // run. Restart both halves of the settle budget so a later gap + // must establish a fresh episode before quarantine. + capabilityGapAttempts = 0; + capabilityGapElapsedNanos = 0L; lastCapabilityGapNanos = 0L; long nowWarn = System.nanoTime(); if (nowWarn - lastTransportWarnNanos >= 5_000_000_000L) { diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java index e9af2d5d..ca89766a 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java @@ -70,10 +70,10 @@ *

  • On wire failure, runs the configured reconnect policy: capped * exponential backoff with jitter, retried indefinitely (Invariant B -- * a store-and-forward drainer never gives up on a wall-clock budget), - * with endpoint-policy failures terminal only for an initial foreground - * connect or an orphan drainer. A previously-live foreground sender keeps - * retrying so credential and rolling-capability changes remain contained - * by store-and-forward. On reconnect success, repositions the cursor at + * with endpoint-policy failures terminal only for an orphan drainer. A + * foreground sender keeps retrying from asynchronous startup onward so + * credential and rolling-capability changes remain contained by + * store-and-forward. On reconnect success, repositions the cursor at * {@code ackedFsn+1} and replays.
    • * * No locks on the steady-state path. The producer thread (user) writes @@ -514,10 +514,10 @@ public final class CursorWebSocketSendLoop implements QuietCloseable { * {@code client} may be {@code null} only if {@code reconnectFactory} * is non-null — this is the async-initial-connect path: the I/O thread * runs the same retry loop on its first iteration to obtain a live - * client, and an initial terminal failure (auth/upgrade reject or durable-ack - * mismatch) is delivered through the dispatcher rather than thrown to the - * constructor's caller; plain connect failures are retried indefinitely - * (Invariant B: no wall-clock budget give-up). + * client. Every endpoint-policy and transport failure stays inside that + * background retry loop; it is never delivered to the foreground producer + * (Invariant B: no wall-clock budget give-up). Blocking OFF/SYNC startup + * performs its fail-fast policy before constructing this loop. */ public CursorWebSocketSendLoop(WebSocketClient client, CursorSendEngine engine, long fsnAtZero, long parkNanos, @@ -1468,10 +1468,10 @@ private void applyDurableAck() { /** * Drives the very first connect attempt on the I/O thread, used in the * async-initial-connect mode (constructed with {@code client == null}). - * Reuses the same retry+backoff machinery as {@link #fail(Throwable)} — - * connect failures are retried indefinitely (Invariant B), and a - * terminal upgrade reject is delivered through the dispatcher, not - * thrown to the producer. + * Reuses the same retry+backoff machinery as {@link #fail(Throwable)}. + * Transport, authentication, upgrade and capability failures are all + * retried indefinitely here (Invariant B); none is surfaced to the + * foreground producer. */ private void attemptInitialConnect() { connectLoop(new LineSenderException( @@ -1532,9 +1532,8 @@ private void connectLoop(Throwable initial, String phase, long paceFirstAttemptM // so this background loop retries for as long as it is running, backing // off between attempts. Endpoint-policy failures (auth / non-421 // upgrade / durable-ack capability gap) are terminal only for orphan - // drainers and a foreground sender's first connection. Once a foreground - // sender has connected, those states are retried so a credential or cluster - // capability rotation cannot stop its producer. SF + // drainers. Foreground senders retry them from asynchronous startup onward + // so a credential or cluster capability rotation cannot stop the producer. SF // exhaustion is surfaced to the PRODUCER as append backpressure, never // here. reconnect_max_duration_millis is intentionally NOT consulted: it // bounds only the blocking (non-lazy) initial connect in @@ -1599,8 +1598,7 @@ private void connectLoop(Throwable initial, String phase, long paceFirstAttemptM resetCatchUpCapGapEpisode(); } catch (QwpAuthFailedException | WebSocketUpgradeException e) { if (endpointPolicyFailureIsTerminal()) { - // Orphans return control to their quarantine owner; a - // foreground sender that never connected still fails fast. + // Orphans return control to their quarantine owner. // WebSocketUpgradeException reaching here is always non-421: // role rejects are classified into the transient branch below. LOG.error("terminal upgrade error during {} -- won't retry: {}", @@ -1628,14 +1626,14 @@ private void connectLoop(Throwable initial, String phase, long paceFirstAttemptM long now = System.nanoTime(); if (now - lastLogNanos >= RECONNECT_LOG_THROTTLE_NANOS) { LOG.warn("{} attempt {}: foreground auth/upgrade policy rejected the connection; " - + "retrying because this sender was previously live -- {}", + + "retrying while store-and-forward owns the buffered data -- {}", phase, attempts, e.getMessage()); lastLogNanos = now; } } catch (QwpDurableAckMismatchException e) { if (endpointPolicyFailureIsTerminal()) { // Orphans hand a capability gap back to BackgroundDrainer's - // settle budget. An initial foreground connect remains loud. + // settle budget. LOG.error("durable-ack mismatch during {} -- won't retry: {}", phase, e.getMessage()); if (terminalError == null) { @@ -1666,7 +1664,7 @@ private void connectLoop(Throwable initial, String phase, long paceFirstAttemptM long now = System.nanoTime(); if (now - lastLogNanos >= RECONNECT_LOG_THROTTLE_NANOS) { LOG.warn("{} attempt {}: foreground durable-ack capability is temporarily " - + "unavailable; retrying because this sender was previously live -- {}", + + "unavailable; retrying while store-and-forward owns the buffered data -- {}", phase, attempts, e.getMessage()); lastLogNanos = now; } @@ -1759,7 +1757,7 @@ private void connectLoop(Throwable initial, String phase, long paceFirstAttemptM } private boolean endpointPolicyFailureIsTerminal() { - return reconnectPolicy == ReconnectPolicy.ORPHAN || !hasEverConnected; + return reconnectPolicy == ReconnectPolicy.ORPHAN; } /** @@ -1911,8 +1909,8 @@ private void enqueuePendingOk(long wireSeq) { * listener both non-null), enters the per-outage retry loop: capped * exponential backoff with jitter, retried for as long as the loop is * running -- there is NO wall-clock give-up (Invariant B: a store-and- - * forward drainer only terminates on SF exhaustion or a genuinely non- - * retriable auth/upgrade reject). On the first successful reconnect the + * forward drainer does not give up on endpoint or transport state). On the + * first successful reconnect the * I/O loop resumes with reset wire state and replays from * {@code engine.ackedFsn() + 1}. *

    @@ -2018,11 +2016,10 @@ private void ioLoop() { // Async-initial-connect path: ctor accepted a null client because // a reconnect factory is wired. Drive the very first connect on // this thread so the producer thread never blocks on it. - // attemptInitialConnect either sets `client` (success) or records - // a terminal failure and clears `running` (auth/upgrade reject; - // plain connect failures retry indefinitely under Invariant B). - // Either way, the main loop below sees the outcome via the - // `running` and `client` fields. + // attemptInitialConnect retries every endpoint/transport failure + // indefinitely under Invariant B and returns only after it installs + // a client or close() clears `running`. The main loop below sees the + // outcome via the `running` and `client` fields. if (client == null && running) { attemptInitialConnect(); } @@ -2272,9 +2269,9 @@ private void sendDurableAckKeepaliveIfDue() { private void swapClient(WebSocketClient newClient) { WebSocketClient old = this.client; this.client = newClient; - // Sticky: once the wire is up, we've reached the server at least - // once for this sender's lifetime. Used downstream to classify a - // subsequent terminal failure as transient vs config-likely. + // Sticky: once the wire is up, we've reached the server at least once + // for this sender's lifetime. Exposed to the owning sender for + // connection-state observability; reconnect policy does not depend on it. this.hasEverConnected = true; if (old != null) { try { diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/CloseSafetyNetTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/CloseSafetyNetTest.java index 2a266212..639b7c8d 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/CloseSafetyNetTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/CloseSafetyNetTest.java @@ -30,6 +30,7 @@ import io.questdb.client.SenderErrorHandler; import io.questdb.client.cutlass.line.LineSenderException; import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender; +import io.questdb.client.cutlass.qwp.client.WebSocketResponse; import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer; import org.jetbrains.annotations.NotNull; import org.junit.Assert; @@ -37,14 +38,18 @@ import org.junit.Test; import org.junit.rules.TemporaryFolder; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.charset.StandardCharsets; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; /** * Pins both branches of {@code QwpWebSocketSender.close()}'s safety-net - * rethrow, strictly — unlike the close assertions in - * {@link InitialConnectAsyncTest}, which tolerate either outcome: + * rethrow using a deterministic server-side parse rejection: *

      *
    • a config-string-only caller (no custom error handler) who never calls * flush() MUST see the latched terminal thrown from close() — close() is the @@ -64,34 +69,33 @@ public class CloseSafetyNetTest { @Test(timeout = 30_000) public void testCloseRethrowsUnsurfacedTerminalWithoutCustomHandler() throws Exception { - // A 401 server, no handler: the I/O thread latches a genuine auth - // terminal (ws-upgrade-failed / SECURITY_ERROR) that nothing has - // surfaced to the user. close() must throw it. (Under Invariant B a - // mere connection error would retry forever and never latch -- only a - // genuine terminal like auth does.) - try (TestWebSocketServer server = new TestWebSocketServer(NOOP_HANDLER)) { - server.setRejectWithStatus(401, "Unauthorized"); + GatedHaltHandler handler = new GatedHaltHandler(); + try (TestWebSocketServer server = new TestWebSocketServer(handler)) { server.start(); Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); Sender sender = Sender.fromConfig(cfg(server.getPort())); - boolean closed = false; + boolean closeAttempted = false; try { + sender.table("foo").longColumn("v", 1L).atNow(); + sender.flush(); + handler.releaseRejection(); awaitLatchedTerminal((QwpWebSocketSender) sender); try { - closed = true; + closeAttempted = true; sender.close(); Assert.fail("close() must rethrow a terminal error that no synchronous " + "caller and no custom handler has seen"); } catch (LineSenderException e) { String msg = e.getMessage() == null ? "" : e.getMessage(); Assert.assertTrue("close() must rethrow the latched terminal: " + msg, - msg.contains("ws-upgrade-failed") || msg.contains("401")); + msg.contains("rejected") || msg.contains("parse error")); Assert.assertTrue("the latched instance is the typed server exception", e instanceof LineSenderServerException); } } finally { - if (!closed) { - sender.close(); + handler.releaseRejection(); + if (!closeAttempted) { + closeQuietly(sender); } } } @@ -99,22 +103,31 @@ public void testCloseRethrowsUnsurfacedTerminalWithoutCustomHandler() throws Exc @Test(timeout = 30_000) public void testCloseStaysSilentWhenCustomHandlerAlreadyDelivered() throws Exception { - // Same auth terminal, but the user installed a custom error handler and - // the dispatcher delivered the error to it. close() must NOT - // double-signal. - try (TestWebSocketServer server = new TestWebSocketServer(NOOP_HANDLER)) { - server.setRejectWithStatus(401, "Unauthorized"); + GatedHaltHandler handler = new GatedHaltHandler(); + try (TestWebSocketServer server = new TestWebSocketServer(handler)) { server.start(); Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); ErrorInbox inbox = new ErrorInbox(); Sender sender = Sender.builder(cfg(server.getPort())) .errorHandler(inbox) .build(); - Assert.assertTrue("terminal must reach the custom handler within 10s", - inbox.await(10, TimeUnit.SECONDS)); - Assert.assertNotNull(inbox.get()); - // The handler owns the error now; a rethrow here would double-signal. - sender.close(); + boolean closed = false; + try { + sender.table("foo").longColumn("v", 1L).atNow(); + sender.flush(); + handler.releaseRejection(); + Assert.assertTrue("terminal must reach the custom handler within 10s", + inbox.await(10, TimeUnit.SECONDS)); + Assert.assertNotNull(inbox.get()); + // The handler owns the error now; a rethrow here would double-signal. + sender.close(); + closed = true; + } finally { + handler.releaseRejection(); + if (!closed) { + closeQuietly(sender); + } + } } } @@ -136,19 +149,47 @@ private static void awaitLatchedTerminal(QwpWebSocketSender sender) { private String cfg(int port) { return "ws::addr=localhost:" + port + ";sf_dir=" + sfDir.getRoot().getAbsolutePath() - + ";initial_connect_retry=async" - + ";reconnect_max_duration_millis=400" - + ";reconnect_initial_backoff_millis=10" - + ";reconnect_max_backoff_millis=50" + ";close_flush_timeout_millis=0;"; } - private static final TestWebSocketServer.WebSocketServerHandler NOOP_HANDLER = - new TestWebSocketServer.WebSocketServerHandler() { - @Override - public void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] data) { - } - }; + private static void closeQuietly(Sender sender) { + try { + sender.close(); + } catch (Throwable ignored) { + // Cleanup after an earlier assertion; the primary failure wins. + } + } + + private static final class GatedHaltHandler implements TestWebSocketServer.WebSocketServerHandler { + private final CountDownLatch gate = new CountDownLatch(1); + private final AtomicLong nextSeq = new AtomicLong(); + + @Override + public void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] data) { + try { + gate.await(); + client.sendBinary(buildErrorAck(nextSeq.getAndIncrement())); + } catch (IOException | InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException(e); + } + } + + void releaseRejection() { + gate.countDown(); + } + + private static byte[] buildErrorAck(long seq) { + byte[] message = "test: parse error".getBytes(StandardCharsets.UTF_8); + byte[] response = new byte[1 + 8 + 2 + message.length]; + ByteBuffer buffer = ByteBuffer.wrap(response).order(ByteOrder.LITTLE_ENDIAN); + buffer.put(WebSocketResponse.STATUS_PARSE_ERROR); + buffer.putLong(seq); + buffer.putShort((short) message.length); + buffer.put(message); + return response; + } + } private static class ErrorInbox implements SenderErrorHandler { private final CountDownLatch latch = new CountDownLatch(1); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/InitialConnectAsyncTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/InitialConnectAsyncTest.java index 173374ec..f7fd0c63 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/InitialConnectAsyncTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/InitialConnectAsyncTest.java @@ -50,58 +50,47 @@ * Behavior of {@code initial_connect_retry=async}: the producer-thread * {@code Sender.fromConfig} must return immediately even when no server * is reachable; the I/O thread retries connect in the background. Plain - * connect failures are retried indefinitely (Invariant B: no wall-clock - * budget give-up); only genuine terminals (auth/upgrade reject, - * durable-ack capability gap) are delivered through the async error - * inbox rather than thrown at the call site. + * failures are retried indefinitely (Invariant B: no wall-clock budget + * give-up), including authentication, upgrade and durable-ack capability + * failures. Those endpoint states stay inside the I/O thread and never stop + * the producer. */ public class InitialConnectAsyncTest { @Test - public void testAsyncAuthFailureDeliversToErrorInbox() throws Exception { - // Server returns HTTP 401 on every upgrade attempt. Auth failures - // are terminal at the I/O thread; in async mode they are - // delivered as a SenderError, not thrown from fromConfig. + public void testAsyncAuthFailureRetriesForeverNoTerminal() throws Exception { + // Server returns HTTP 401 on every upgrade attempt. Once ASYNC build() + // has returned, store-and-forward owns the buffered rows: the I/O thread + // must keep retrying rather than terminalizing the producer. try (Always401Fixture fixture = new Always401Fixture()) { fixture.start(); int port = fixture.getPort(); ErrorInbox inbox = new ErrorInbox(); String cfg = "ws::addr=localhost:" + port + sfDirOpt() + ";initial_connect_retry=async" - + ";reconnect_max_duration_millis=10000" + + ";reconnect_max_duration_millis=200" + + ";reconnect_initial_backoff_millis=10" + + ";reconnect_max_backoff_millis=50" + ";close_flush_timeout_millis=0;"; Sender sender = Sender.builder(cfg) .errorHandler(inbox) .build(); try { - // Auth-terminal must surface within hundreds of ms even - // though the cap is 10s. - long t0 = System.nanoTime(); - Assert.assertTrue( - "401 upgrade reject must surface a SenderError within 5s", - inbox.await(5, TimeUnit.SECONDS)); - long elapsedMs = (System.nanoTime() - t0) / 1_000_000L; - SenderError err = inbox.get(); - Assert.assertNotNull( - "401 upgrade reject must surface a SenderError", - err); - Assert.assertTrue( - "auth-terminal must surface well inside the cap; took " - + elapsedMs + "ms (cap was 10000ms)", - elapsedMs < 5_000L); - Assert.assertEquals( - "category must be SECURITY_ERROR for ws-upgrade-failed", - SenderError.Category.SECURITY_ERROR, err.getCategory()); - Assert.assertEquals( - "auth failure is TERMINAL", - SenderError.Policy.TERMINAL, err.getAppliedPolicy()); - String msg = err.getServerMessage() == null ? "" : err.getServerMessage(); - Assert.assertTrue( - "error message must mention ws-upgrade-failed: " + msg, - msg.contains("ws-upgrade-failed") - || msg.contains("401")); + QwpWebSocketSender wss = (QwpWebSocketSender) sender; + awaitAtLeastOneConnectAttempt(wss); + + sender.table("foo").longColumn("v", 1L).atNow(); + sender.flush(); + + Assert.assertFalse( + "async 401 rejection must stay inside the retry loop", + inbox.await(1_500, TimeUnit.MILLISECONDS)); + Assert.assertNull("no terminal SenderError may be delivered for an async 401", + inbox.get()); + Assert.assertFalse("no upgrade has succeeded yet", wss.wasEverConnected()); + awaitReconnectAttemptsAdvance(wss); } finally { - assertCloseRethrowsTerminal(sender, "ws-upgrade-failed"); + closeQuietly(sender); } } } @@ -113,8 +102,8 @@ public void testAsyncNoServerRetriesForeverNoTerminal() throws Exception { // (it may appear; the data is safe in SF), so the I/O thread retries // forever. reconnect_max_duration_millis is IGNORED as a give-up deadline: // no SenderError lands, the sender stays usable, and wasEverConnected() - // stays false. Only a GENUINE terminal (auth/upgrade) or SF exhaustion may - // surface -- see testAsyncAuthFailureDeliversToErrorInbox. + // stays false. Endpoint-policy failures are covered by + // testAsyncAuthFailureRetriesForeverNoTerminal. int port = TestPorts.findUnusedPort(); ErrorInbox inbox = new ErrorInbox(); String cfg = "ws::addr=localhost:" + port @@ -373,31 +362,6 @@ private static void closeQuietly(Sender sender) { } } - /** - * Closes the sender and tolerates either outcome: - * * close() throws -- the latched terminal must mention the expected - * substring (safety-net rethrow path); - * * close() returns cleanly -- the user installed an async error - * handler in this test, so the dispatcher already delivered the - * error to the handler (or will, on shutdown). Rethrowing on top - * of that would mask try-with-resources cleanup in real callers, - * so close() suppresses the rethrow when a custom handler is - * installed. - * Either way, the inbox observation earlier in the test pins the - * primary contract -- this helper just guards against close() throwing - * with a wrong message. - */ - private static void assertCloseRethrowsTerminal(Sender sender, String expectedSubstring) { - try { - sender.close(); - } catch (Throwable t) { - String msg = t.getMessage() == null ? "" : t.getMessage(); - Assert.assertTrue( - "close() rethrow must mention " + expectedSubstring + ": " + msg, - msg.contains(expectedSubstring)); - } - } - /** * Returns a unique temp sf_dir snippet for embedding in a config * string. The builder does NOT require sf_dir for any @@ -478,9 +442,9 @@ public void onError(@NotNull SenderError err) { /** * Raw-socket fixture: every accepted connection responds with HTTP - * 401 Unauthorized and closes. Used to drive the async-init - * auth-terminal path: the I/O thread's first connect attempt classifies - * the response as a terminal upgrade failure. + * 401 Unauthorized and closes. Used to prove that the async-init I/O + * thread keeps retrying an endpoint-policy failure without terminalizing + * the producer. */ private static class Always401Fixture implements AutoCloseable { private final java.util.List openSockets = new java.util.concurrent.CopyOnWriteArrayList<>(); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/BackgroundDrainerDurableAckRetryTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/BackgroundDrainerDurableAckRetryTest.java index 82885ba7..e7e63a2e 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/BackgroundDrainerDurableAckRetryTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/BackgroundDrainerDurableAckRetryTest.java @@ -709,13 +709,11 @@ public void testRoleRejectAndCapabilityGapLandOnSeparateStreams() throws Excepti } @Test - public void testTransportErrorDoesNotResetCapabilityGapEpisode() throws Exception { + public void testTransportErrorResetsCapabilityGapEpisode() throws Exception { assertMemoryLeak(() -> { - // A transport blip between gap attempts does not prove promotion - // churn: it must neither consume the budget (no increment) nor - // restart it (no reset) -- otherwise a flaky-but-misconfigured - // cluster would evade the cap forever. 15 gaps, one transport error, - // one gap: escalates on that 16th gap attempt. + // A transport state breaks a consecutive capability-gap episode. + // After 15 gaps and one transport error, the drainer must observe a + // fresh run of 16 gaps before quarantining the slot. int cap = BackgroundDrainer.DEFAULT_MAX_DURABLE_ACK_MISMATCH_ATTEMPTS; CountingListener listener = new CountingListener(); AtomicInteger sweeps = new AtomicInteger(); @@ -732,30 +730,24 @@ public void testTransportErrorDoesNotResetCapabilityGapEpisode() throws Exceptio assertNull(out); assertEquals(BackgroundDrainer.DrainOutcome.FAILED, drainer.outcome()); assertEquals(1, listener.persistentFailures.get()); - assertEquals("transport blip must not restart the episode", + assertEquals("the fresh episode must exhaust its own full attempt budget", cap, listener.lastPersistentTotalAttempts.get()); - // 15 gap + 1 transport + 1 gap = 17 sweeps total. - assertEquals(cap + 1, factory.attempts()); + // 15 gaps + 1 transport + a fresh 16-gap episode. + assertEquals(2 * cap, factory.attempts()); + assertEquals(2 * (cap - 1), listener.unavailableAttempts.size()); + assertEquals("attempt numbering must restart after transport", + 1, (int) listener.unavailableAttempts.get(cap - 1)); assertTrue(Files.exists(slotPath + "/" + OrphanScanner.FAILED_SENTINEL_NAME)); }); } @Test - public void testTransportWindowDoesNotBurnCapabilityGapWallClock() throws Exception { + public void testTransportWindowResetsCapabilityGapWallClock() throws Exception { assertMemoryLeak(() -> { - // Red-first: the wall-clock half of the settle budget is anchored at - // gap #1, and a transport window BETWEEN gap sweeps must PAUSE it -- - // only gap-to-gap time is the cluster "failing to settle". Under the - // bug the deadline keeps ticking while the cluster is unreachable: - // gap #1 anchors the deadline, the cluster then drops off the network - // for longer than the entire budget (transport errors are retried - // "forever" and charge nothing else), and when it comes back still - // briefly gapped, gap #2 observes an expired deadline and quarantines - // the slot after just 2 gap sweeps -- contradicting both the - // 16-attempt settle intent and Invariant B's "transients never - // consume the budget". Evasion is not a concern: the attempt counter - // survives the window untouched, which - // testTransportErrorDoesNotResetCapabilityGapEpisode pins. + // The wall-clock half of the settle budget is anchored at gap #1. + // A transport window breaks that episode completely, so the next gap + // receives a fresh clock rather than inheriting time accumulated before + // or during the unrelated outage. // Here the cluster actually settles after the outage (two more gap // sweeps, then durable-ack-capable), so the drain must proceed -- // no escalation, no sentinel. diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CloseOwnershipRaceTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CloseOwnershipRaceTest.java index e04bf15c..bbff9a74 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CloseOwnershipRaceTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CloseOwnershipRaceTest.java @@ -60,20 +60,24 @@ public void closeOwnershipSnapshotNeverClaimsAnUnsurfacedError() { sfDir.getRoot().getAbsolutePath(), 16_384)) { Throwable leaked = null; for (int i = 0; i < ROUNDS && leaked == null; i++) { - // A null client and a reconnect factory that throws a genuine - // terminal auth reject: start()'s real I/O thread walks the - // production async-initial-connect path and latches a genuine - // (SECURITY_ERROR) terminal within microseconds. One authentic - // null->error latch transition per round. (Under Invariant B a - // connection error / budget would retry forever and never latch; - // only a genuine terminal like auth does.) + // A null client and an ORPHAN reconnect policy whose factory + // throws a terminal auth reject: start()'s real I/O thread latches + // a genuine SECURITY_ERROR within microseconds. Foreground ASYNC + // policy intentionally retries this state forever, while orphan + // policy must hand it back to the quarantine owner. CursorWebSocketSendLoop loop = new CursorWebSocketSendLoop( null, engine, 0, 1_000_000L, () -> { throw new QwpAuthFailedException(401, "localhost", 1); }, 0, - 1, 1); + 1, 1, + false, + 0, + CursorWebSocketSendLoop.DEFAULT_MAX_HEAD_FRAME_REJECTIONS, + 0, + 0, + CursorWebSocketSendLoop.ReconnectPolicy.ORPHAN); loop.start(); // Race close()'s exact ownership snapshot against the latch // transition, stopping once the latch has landed. Nothing in diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopForegroundReconnectPolicyTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopForegroundReconnectPolicyTest.java index 064edf3f..73e3b634 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopForegroundReconnectPolicyTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopForegroundReconnectPolicyTest.java @@ -54,6 +54,18 @@ public class CursorWebSocketSendLoopForegroundReconnectPolicyTest { @Rule public final TemporaryFolder sfDir = TemporaryFolder.builder().assureDeletion().build(); + @Test + public void testAsyncInitialAuthFailureRetriesUntilCredentialsRecover() throws Exception { + assertAsyncInitialForegroundRecovers(false, + () -> new QwpAuthFailedException(401, "localhost", 1)); + } + + @Test + public void testAsyncInitialDurableAckMismatchRetriesUntilCapabilityRecovers() throws Exception { + assertAsyncInitialForegroundRecovers(true, + () -> new QwpDurableAckMismatchException("localhost", 1, "primary")); + } + @Test public void testPostStartAuthFailureRetriesUntilCredentialsRecover() throws Exception { assertForegroundRecovers(false, @@ -66,9 +78,64 @@ public void testPostStartDurableAckMismatchRetriesUntilCapabilityRecovers() thro () -> new QwpDurableAckMismatchException("localhost", 1, "primary")); } + private void assertAsyncInitialForegroundRecovers( + boolean durableAck, + FailureSupplier failureSupplier + ) throws Exception { + TestUtils.assertMemoryLeak(() -> { + DropFirstConnectionHandler handler = new DropFirstConnectionHandler(durableAck, false); + try (TestWebSocketServer server = new TestWebSocketServer(handler, true); + CursorSendEngine engine = new CursorSendEngine( + sfDir.newFolder().getAbsolutePath(), SEGMENT_SIZE_BYTES)) { + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + + GatedFactory factory = new GatedFactory( + server.getPort(), durableAck, failureSupplier); + CursorWebSocketSendLoop loop = new CursorWebSocketSendLoop( + null, + engine, + 0L, + CursorWebSocketSendLoop.DEFAULT_PARK_NANOS, + factory, + 100L, + 1L, + 4L, + durableAck, + durableAck ? 10L : 0L, + CursorWebSocketSendLoop.DEFAULT_MAX_HEAD_FRAME_REJECTIONS, + 0L, + 0L, + CursorWebSocketSendLoop.ReconnectPolicy.FOREGROUND); + try { + appendFrame(engine, (byte) 1); + loop.start(); + + await(() -> factory.attempts() >= 2 || loop.getTerminalError() != null, + "async foreground did not retry the endpoint-policy failure"); + Assert.assertNull( + "async initial endpoint-policy failures must not stop the producer", + loop.getTerminalError()); + Assert.assertFalse(loop.hasEverConnected()); + + long target = appendFrame(engine, (byte) 2); + factory.allowConnect(); + await(() -> engine.ackedFsn() >= target || loop.getTerminalError() != null, + "async foreground did not deliver after endpoint policy recovered"); + Assert.assertNull(loop.getTerminalError()); + Assert.assertTrue(loop.hasEverConnected()); + Assert.assertEquals(target, engine.ackedFsn()); + } finally { + factory.allowConnect(); + loop.close(); + } + } + }); + } + private void assertForegroundRecovers(boolean durableAck, FailureSupplier failureSupplier) throws Exception { TestUtils.assertMemoryLeak(() -> { - DropFirstConnectionHandler handler = new DropFirstConnectionHandler(durableAck); + DropFirstConnectionHandler handler = new DropFirstConnectionHandler(durableAck, true); try (TestWebSocketServer server = new TestWebSocketServer(handler, true); CursorSendEngine engine = new CursorSendEngine( sfDir.newFolder().getAbsolutePath(), SEGMENT_SIZE_BYTES)) { @@ -190,16 +257,49 @@ public WebSocketClient reconnect() throws Exception { } } + private static final class GatedFactory implements CursorWebSocketSendLoop.ReconnectFactory { + private final AtomicInteger attempts = new AtomicInteger(); + private final boolean durableAck; + private final FailureSupplier failureSupplier; + private final int port; + private volatile boolean connectAllowed; + + private GatedFactory(int port, boolean durableAck, FailureSupplier failureSupplier) { + this.port = port; + this.durableAck = durableAck; + this.failureSupplier = failureSupplier; + } + + void allowConnect() { + connectAllowed = true; + } + + int attempts() { + return attempts.get(); + } + + @Override + public WebSocketClient reconnect() throws Exception { + attempts.incrementAndGet(); + if (!connectAllowed) { + throw failureSupplier.get(); + } + return connect(port, durableAck); + } + } + private static final class DropFirstConnectionHandler implements TestWebSocketServer.WebSocketServerHandler { private static final String TABLE = "trades"; private final boolean durableAck; + private final boolean dropFirstConnection; private final Map wireSeqByConnection = new IdentityHashMap<>(); private TestWebSocketServer.ClientHandler firstConnection; - private DropFirstConnectionHandler(boolean durableAck) { + private DropFirstConnectionHandler(boolean durableAck, boolean dropFirstConnection) { this.durableAck = durableAck; + this.dropFirstConnection = dropFirstConnection; } @Override @@ -218,7 +318,7 @@ public synchronized void onBinaryMessage(TestWebSocketServer.ClientHandler clien if (durableAck) { client.sendBinary(durableAckFrame(wireSeq)); } - if (client == firstConnection) { + if (dropFirstConnection && client == firstConnection) { client.close(); } } catch (IOException ignored) { From 20df0eb168e28f9eb1af7052bc6f4aacd403a4cb Mon Sep 17 00:00:00 2001 From: glasstiger Date: Fri, 17 Jul 2026 11:07:10 +0100 Subject: [PATCH 80/96] fix: harden QWP recovery cleanup --- .../client/sf/cursor/BackgroundDrainer.java | 5 + .../client/sf/cursor/CursorSendEngine.java | 30 ++++++ .../sf/cursor/CursorWebSocketSendLoop.java | 34 +++--- .../sf/cursor/RecoveredFrameAnalysis.java | 28 +++++ .../qwp/client/sf/cursor/SegmentRing.java | 1 + .../qwp/client/DeltaDictRecoveryTest.java | 35 +++++- .../BackgroundDrainerOrphanTailTest.java | 102 ++++++++++++++++++ ...CursorWebSocketSendLoopOrphanTailTest.java | 45 ++++++++ 8 files changed, 259 insertions(+), 21 deletions(-) create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/BackgroundDrainerOrphanTailTest.java diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/BackgroundDrainer.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/BackgroundDrainer.java index 38e7c4ee..fc5116fa 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/BackgroundDrainer.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/BackgroundDrainer.java @@ -594,6 +594,11 @@ public void run() { logicalSlotLock.close(); logicalSlotLock = null; } + // A recovered deferred-only tail is an aborted transaction and can + // be retired locally once everything below it is already ACKed. + // Do this before opening a socket: auth/upgrade failures must not + // quarantine a slot that has no wire-visible work left. + engine.retireRecoveredOrphanTailIfReady(); long target = engine.publishedFsn(); if (engine.ackedFsn() >= target) { LOG.info("orphan slot already drained: {} (acked={} target={})", diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java index 597c2bc7..bdff0c9b 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java @@ -809,6 +809,11 @@ public long recoverySymbolEntriesVisited() { return recoveredFrameAnalysis == null ? 0L : recoveredFrameAnalysis.symbolEntriesVisited(); } + @TestOnly + public int recoverySymbolNativeCapacity() { + return recoveredFrameAnalysis == null ? 0 : recoveredFrameAnalysis.rawCapacity(); + } + private RecoveredFrameAnalysis checkedRecoveryAnalysis(int baseline) { if (recoveredFrameAnalysis == null || recoveredFrameAnalysis.baseline() != baseline) { throw new IllegalStateException("recovery symbol baseline mismatch [expected=" @@ -960,6 +965,31 @@ public long recoveredOrphanTipFsn() { return recoveredOrphanTipFsn; } + /** + * Retires a recovered deferred tail once every frame below it is ACKed. + * The operation is local and idempotent: no wire sequence ever referred + * to these aborted-transaction frames. + * + * @return true if no orphan tail remains, false if lower frames still need + * server ACKs + */ + public boolean retireRecoveredOrphanTailIfReady() { + long orphanTip = recoveredOrphanTipFsn; + if (orphanTip < 0L) { + return true; + } + long orphanStart = recoveredCommitBoundaryFsn + 1L; + if (ackedFsn() < orphanStart - 1L) { + return false; + } + LOG.warn("retiring orphaned deferred tail: {} frame(s) [fsn {}..{}] belong to a transaction " + + "whose commit was never published; aborting them (never transmitted, slots trimmed)", + orphanTip - orphanStart + 1L, orphanStart, orphanTip); + acknowledge(orphanTip); + recoveredOrphanTipFsn = -1L; + return true; + } + /** * Unlinks every {@code .sfa} file under {@code dir}. Called only on * clean shutdown when the ring confirms every published FSN has been diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java index ca89766a..76005346 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java @@ -306,6 +306,13 @@ public final class CursorWebSocketSendLoop implements QuietCloseable { // Any growth first performs a copy-on-write; only owned buffers are freed. private boolean sentDictBytesOwned; private int sentDictCount; + // True when replay frames can start above dictionary id zero and therefore + // depend on a catch-up on a fresh connection. Delta-enabled live engines + // always have this dependency. A recovered delta slot whose dictionary + // failed to open is identified by its non-zero recovered delta start. The + // remaining false case is full-dictionary fallback: every frame re-registers + // from id zero, so sending the mirror first would duplicate the dictionary. + private final boolean hasReplayDictionaryDependency; // Reusable staging frame for dictionary catch-up chunks. A reconnect may split a // large dictionary into many chunks, and every later reconnect repeats that split; // retaining one growable loop-owned buffer turns the old malloc/free-per-chunk path @@ -372,9 +379,8 @@ public final class CursorWebSocketSendLoop implements QuietCloseable { private long fsnAtZero; // Sticky flag: false until the very first time a live client is installed // (either via the constructor in SYNC/OFF mode or via swapClient on a - // successful connect attempt in any mode). Once true, stays true. Used to - // distinguish an initial endpoint-policy failure (fail fast) from a - // post-start failure that a foreground sender must contain and retry. + // successful connect attempt in any mode). Once true, stays true. This is + // connection-state observability only; reconnect policy is role-based. private volatile boolean hasEverConnected; private volatile Thread ioThread; // Typed marker for a durable-ack CAPABILITY-GAP terminal: set (before the @@ -709,6 +715,8 @@ public CursorWebSocketSendLoop(WebSocketClient client, CursorSendEngine engine, } this.client = client; this.engine = engine; + this.hasReplayDictionaryDependency = engine.isDeltaDictEnabled() + || engine.recoveredMaxSymbolDeltaStart() > 0L; // Recovery / orphan-drain: the loop starts with a fresh in-memory mirror, // so seed it from the slot's persisted dictionary. That way the very first // connection re-registers the whole dictionary (via a catch-up frame) @@ -2320,15 +2328,11 @@ private void setWireBaselineWithCatchUp(long replayStart) { // handleServerRejection can tell "only the catch-up went out" from "the // head data frame went out". dataFrameSentThisConnection = false; - // Not gated on isDeltaDictEnabled() -- see the constructor. The mirror is - // non-empty exactly when this loop has already put symbols on a wire, and a - // FRESH server holds none of them, so it must be re-registered before any - // frame that back-references an id. In full-dict mode the mirror is non-empty - // too and the catch-up is then redundant (the next frame re-registers from id - // 0 anyway) but harmless: the server overwrites identical ids with identical - // symbols. Redundant-but-uniform beats a mode flag that can go false while the - // frames on disk are still deltas. - if (client != null && sentDictCount > 0) { + // Do not gate solely on isDeltaDictEnabled(): a recovered delta slot can + // lose access to its side-file while its on-disk frames still start above + // zero. hasReplayDictionaryDependency preserves that distinction while + // keeping full-dictionary fallback catch-up-free across every reconnect. + if (client != null && sentDictCount > 0 && hasReplayDictionaryDependency) { this.nextWireSeq = 0L; // The catch-up may span several frames when the dictionary exceeds the // server's batch cap; each consumes a wire sequence (0 .. n-1) that maps @@ -2981,13 +2985,9 @@ private boolean tryRetireOrphanTail() { if (orphanSkipTipFsn < 0) { return true; } - if (engine.ackedFsn() < orphanSkipStartFsn - 1L) { + if (!engine.retireRecoveredOrphanTailIfReady()) { return false; } - LOG.warn("retiring orphaned deferred tail: {} frame(s) [fsn {}..{}] belong to a transaction " - + "whose commit was never published; aborting them (never transmitted, slots trimmed)", - orphanSkipTipFsn - orphanSkipStartFsn + 1, orphanSkipStartFsn, orphanSkipTipFsn); - engine.acknowledge(orphanSkipTipFsn); orphanSkipStartFsn = -1L; orphanSkipTipFsn = -1L; return true; diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/RecoveredFrameAnalysis.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/RecoveredFrameAnalysis.java index e94e68f5..cde5e06b 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/RecoveredFrameAnalysis.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/RecoveredFrameAnalysis.java @@ -147,6 +147,30 @@ long framesVisited() { return framesVisited; } + /** + * Discards native bytes accumulated only while scanning an uncommitted + * deferred tail. Call exactly once after the ordered recovery walk. + */ + void finish() { + runningRawLen = committedRawLen; + runningRawCount = committedRawCount; + if (rawCapacity == committedRawLen) { + return; + } + if (committedRawLen == 0) { + Unsafe.free(rawAddr, rawCapacity, MemoryTag.NATIVE_DEFAULT); + rawAddr = 0L; + rawCapacity = 0; + return; + } + rawAddr = Unsafe.realloc( + rawAddr, + rawCapacity, + committedRawLen, + MemoryTag.NATIVE_DEFAULT); + rawCapacity = committedRawLen; + } + long maxDeltaEnd() { return committedMaxDeltaEnd; } @@ -167,6 +191,10 @@ int rawLen() { return committedRawLen; } + int rawCapacity() { + return rawCapacity; + } + long symbolEntriesVisited() { return symbolEntriesVisited; } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java index 5ba787f3..735e1d0f 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java @@ -613,6 +613,7 @@ synchronized RecoveredFrameAnalysis analyzeRecovery(int symbolBaseline) { sealedSegments.get(i).scanRecovery(analysis); } active.scanRecovery(analysis); + analysis.finish(); return analysis; } catch (Throwable t) { analysis.close(); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java index 10d4ef32..61c4495e 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java @@ -1338,7 +1338,9 @@ public void testFullDictFramesRecoverInFullDictModeInsteadOfBricking() throws Ex // Phase 2: recover. build() must SUCCEED (not throw the torn-dict guard), // and the self-sufficient frames replay against a fresh server gap-free. - DictReconstructingHandler handler = new DictReconstructingHandler(); + // Drop the first replayed data frame before ACK so a second connection + // must replay it and prove full-dict mode stays catch-up-free. + DictReconstructingHandler handler = new DictReconstructingHandler(true); try (TestWebSocketServer good = new TestWebSocketServer(handler)) { int port = good.getPort(); good.start(); @@ -1347,13 +1349,17 @@ public void testFullDictFramesRecoverInFullDictModeInsteadOfBricking() throws Ex try (Sender ignored = Sender.fromConfig(cfg)) { // must NOT throw long deadline = System.currentTimeMillis() + 5_000; while (System.currentTimeMillis() < deadline - && handler.maxDictSize() < DISTINCT_SYMBOLS) { + && (handler.dataFrameCount() < 2 + || handler.maxDictSize() < DISTINCT_SYMBOLS)) { Thread.sleep(20); } } + Assert.assertTrue("test must force a reconnect and replay the full-dict frame", + handler.dataFrameCount() >= 2); // Full-dict recovery re-ships the whole dictionary inline on every - // frame, so there is NO 0-table catch-up frame (that is the delta-mode - // reconnect path). The reconstructed dictionary is still gap-free. + // frame, including after the forced reconnect, so there is NO 0-table + // catch-up frame (that is the delta-mode reconnect path). The + // reconstructed dictionary is still gap-free. Assert.assertFalse("full-dict recovery must NOT send a catch-up frame", handler.sawCatchUpFrame); List reconstructed = handler.dictSnapshot(); @@ -1382,10 +1388,21 @@ private static class DictReconstructingHandler implements TestWebSocketServer.We volatile boolean sawCatchUpFrame; private final List ackSequenceStarts = new ArrayList<>(); private final List dict = new ArrayList<>(); + private final boolean dropFirstDataFrame; private final AtomicLong nextSeq = new AtomicLong(0); private TestWebSocketServer.ClientHandler currentClient; + private int dataFrameCount; + private boolean firstDataFrameDropped; private volatile int lastDataDeltaStart = -1; + private DictReconstructingHandler() { + this(false); + } + + private DictReconstructingHandler(boolean dropFirstDataFrame) { + this.dropFirstDataFrame = dropFirstDataFrame; + } + synchronized List ackSequenceStarts() { return new ArrayList<>(ackSequenceStarts); } @@ -1394,6 +1411,10 @@ synchronized List dictSnapshot() { return new ArrayList<>(dict); } + synchronized int dataFrameCount() { + return dataFrameCount; + } + synchronized int maxDictSize() { return dict.size(); } @@ -1415,7 +1436,13 @@ public synchronized void onBinaryMessage(TestWebSocketServer.ClientHandler clien if (tableCount == 0 && QwpWireTestUtils.hasDelta(data)) { sawCatchUpFrame = true; } else if (tableCount > 0 && QwpWireTestUtils.hasDelta(data)) { + dataFrameCount++; lastDataDeltaStart = QwpWireTestUtils.readVarint(data, new int[]{12}); + if (dropFirstDataFrame && !firstDataFrameDropped) { + firstDataFrameDropped = true; + client.close(); + return; + } } try { long ackSequence = nextSeq.getAndIncrement(); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/BackgroundDrainerOrphanTailTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/BackgroundDrainerOrphanTailTest.java new file mode 100644 index 00000000..0fce703a --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/BackgroundDrainerOrphanTailTest.java @@ -0,0 +1,102 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client.sf.cursor; + +import io.questdb.client.cutlass.qwp.client.sf.cursor.BackgroundDrainer; +import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine; +import io.questdb.client.cutlass.qwp.client.sf.cursor.OrphanScanner; +import io.questdb.client.std.Files; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.Unsafe; +import io.questdb.client.test.tools.TestUtils; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.util.concurrent.atomic.AtomicInteger; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; + +public class BackgroundDrainerOrphanTailTest { + + private static final int FLAG_DEFER_COMMIT = 0x01; + private static final int HEADER_OFFSET_FLAGS = 5; + private static final int MAGIC_MESSAGE = 0x31505751; + private static final long SEGMENT_SIZE_BYTES = 16_384L; + + @Rule + public final TemporaryFolder temporaryFolder = TemporaryFolder.builder().assureDeletion().build(); + + @Test + public void testDeferredOnlyRecoveredTailRetiresWithoutConnecting() throws Exception { + String slotPath = temporaryFolder.newFolder("slot").getAbsolutePath(); + TestUtils.assertMemoryLeak(() -> { + try (CursorSendEngine engine = new CursorSendEngine(slotPath, SEGMENT_SIZE_BYTES)) { + appendDeferredFrame(engine); + appendDeferredFrame(engine); + appendDeferredFrame(engine); + } + + AtomicInteger connectAttempts = new AtomicInteger(); + BackgroundDrainer drainer = new BackgroundDrainer( + slotPath, + SEGMENT_SIZE_BYTES, + 1L << 20, + () -> { + connectAttempts.incrementAndGet(); + throw new AssertionError("deferred-only tail must retire before connect"); + }, + 5_000L, + 1L, + 5L, + true, + 200L); + + drainer.run(); + + assertEquals("local retirement must not invoke the reconnect factory", + 0, connectAttempts.get()); + assertEquals(BackgroundDrainer.DrainOutcome.SUCCESS, drainer.outcome()); + assertFalse("local retirement must not quarantine the slot", + Files.exists(slotPath + "/" + OrphanScanner.FAILED_SENTINEL_NAME)); + }); + } + + private static void appendDeferredFrame(CursorSendEngine engine) { + long buffer = Unsafe.malloc(16, MemoryTag.NATIVE_DEFAULT); + try { + for (int i = 0; i < 16; i++) { + Unsafe.getUnsafe().putByte(buffer + i, (byte) 0); + } + Unsafe.getUnsafe().putInt(buffer, MAGIC_MESSAGE); + Unsafe.getUnsafe().putByte(buffer + HEADER_OFFSET_FLAGS, (byte) FLAG_DEFER_COMMIT); + engine.appendBlocking(buffer, 16); + } finally { + Unsafe.free(buffer, 16, MemoryTag.NATIVE_DEFAULT); + } + } + +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopOrphanTailTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopOrphanTailTest.java index d7010310..ef5861ed 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopOrphanTailTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopOrphanTailTest.java @@ -28,6 +28,7 @@ import io.questdb.client.cutlass.http.client.WebSocketClient; import io.questdb.client.cutlass.http.client.WebSocketFrameHandler; import io.questdb.client.cutlass.qwp.client.GlobalSymbolDictionary; +import io.questdb.client.cutlass.qwp.client.NativeBufferWriter; import io.questdb.client.cutlass.qwp.client.WebSocketResponse; import io.questdb.client.cutlass.qwp.client.sf.cursor.AckWatermark; import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine; @@ -184,6 +185,22 @@ public void testFastPathRetiresWholeDeferredLogBeforeAnySend() throws Exception }); } + @Test + public void testRecoveryReleasesAbortedTailSymbolStorage() throws Exception { + TestUtils.assertMemoryLeak(() -> { + try (CursorSendEngine engine = newEngine()) { + appendDeltaSymbolFrame(engine, 0, 'a'); + appendLargeDeferredDeltaSymbolFrame(engine, 1, 8_000); + } + try (CursorSendEngine engine = newEngine()) { + assertEquals("native recovery storage must be trimmed to the committed prefix", + 2, engine.recoverySymbolNativeCapacity()); + assertEquals("large deferred frame is the aborted tail", + 1L, engine.recoveredOrphanTipFsn()); + } + }); + } + @Test public void testSlowPathReplaysBelowTailThenRetiresAndRecyclesOnce() throws Exception { // fsn 0 is commit-covered and unacked: it must replay. fsns 1..2 are @@ -589,6 +606,34 @@ private static void appendDeltaSymbolFrame(CursorSendEngine engine, int deltaSta } } + private static void appendLargeDeferredDeltaSymbolFrame( + CursorSendEngine engine, + int deltaStart, + int symbolLen + ) { + int size = HEADER_SIZE + + NativeBufferWriter.varintSize(deltaStart) + + NativeBufferWriter.varintSize(1) + + NativeBufferWriter.varintSize(symbolLen) + + symbolLen; + long buf = Unsafe.malloc(size, MemoryTag.NATIVE_DEFAULT); + try { + for (int i = 0; i < size; i++) { + Unsafe.getUnsafe().putByte(buf + i, (byte) 0); + } + Unsafe.getUnsafe().putInt(buf, MAGIC_MESSAGE); + Unsafe.getUnsafe().putByte(buf + HEADER_OFFSET_FLAGS, + (byte) (FLAG_DEFER_COMMIT | FLAG_DELTA_SYMBOL_DICT)); + long p = NativeBufferWriter.writeVarint(buf + HEADER_SIZE, deltaStart); + p = NativeBufferWriter.writeVarint(p, 1); + p = NativeBufferWriter.writeVarint(p, symbolLen); + Unsafe.getUnsafe().setMemory(p, symbolLen, (byte) 'z'); + engine.appendBlocking(buf, size); + } finally { + Unsafe.free(buf, size, MemoryTag.NATIVE_DEFAULT); + } + } + private static void awaitAckedFsn(CursorSendEngine engine, long target) throws InterruptedException { long deadline = System.nanoTime() + 10_000_000_000L; while (engine.ackedFsn() < target) { From 1596dab7b020a031336fa4c23e2aa3ae71566cbe Mon Sep 17 00:00:00 2001 From: glasstiger Date: Fri, 17 Jul 2026 11:28:33 +0100 Subject: [PATCH 81/96] Reduce QWP recovery memory and CRC cost Release the cached recovery symbol suffix after the foreground send loop copies it, while retaining orphan-drainer state across recycled sessions. Share a Java Unsafe slice-by-8 CRC implementation between dictionary and segment recovery so large backlogs avoid scalar checksum scans without moving mmap faults into JNI. --- .../client/sf/cursor/CursorSendEngine.java | 6 ++ .../sf/cursor/CursorWebSocketSendLoop.java | 27 +++--- .../qwp/client/sf/cursor/MmapSegment.java | 47 ++--------- .../client/sf/cursor/PersistedSymbolDict.java | 26 +----- .../sf/cursor/RecoveredFrameAnalysis.java | 25 ++++-- .../java/io/questdb/client/std/Crc32c.java | 64 +++++++++++++++ ...CursorWebSocketSendLoopMirrorLeakTest.java | 82 +++++++++++++++++-- .../questdb/client/test/std/Crc32cTest.java | 36 ++++++++ 8 files changed, 226 insertions(+), 87 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java index bdff0c9b..64242e41 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java @@ -799,6 +799,12 @@ void copyRecoveredSymbolSuffix(int baseline, long target) { } } + void releaseRecoveredSymbolStorage() { + if (recoveredFrameAnalysis != null) { + recoveredFrameAnalysis.releaseRawStorage(); + } + } + @TestOnly public long recoveryFramesVisited() { return recoveredFrameAnalysis == null ? 0L : recoveredFrameAnalysis.framesVisited(); diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java index 76005346..d0340f6c 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java @@ -799,19 +799,22 @@ public CursorWebSocketSendLoop(WebSocketClient client, CursorSendEngine engine, } } // QwpWebSocketSender seeds the producer dictionary before constructing - // its one foreground loop, so that loop can own the recovered prefix and - // eliminate the engine-side duplicate. BackgroundDrainer must retain the - // prefix for later recycled loops and therefore keeps borrowing it. - if (persistedPrefixLen > 0 && catchUpCapGapPolicy == CatchUpCapGapPolicy.RETRY_FOREVER) { - long persistedPrefixAddr = pd.takeLoadedEntries(); - if (sentDictBytesOwned) { - // Copy-on-grow already produced the combined mirror. Retire the - // no-longer-needed persisted prefix after successful construction. - Unsafe.free(persistedPrefixAddr, persistedPrefixLen, MemoryTag.NATIVE_DEFAULT); - } else { - assert persistedPrefixAddr == sentDictBytesAddr; - sentDictBytesOwned = true; + // its one foreground loop, so after the loop has built its mirror it can + // retire both engine-owned recovery sources. BackgroundDrainer must retain + // them for later recycled loops and therefore keeps borrowing them. + if (reconnectPolicy == ReconnectPolicy.FOREGROUND) { + if (persistedPrefixLen > 0) { + long persistedPrefixAddr = pd.takeLoadedEntries(); + if (sentDictBytesOwned) { + // Copy-on-grow already produced the combined mirror. Retire the + // no-longer-needed persisted prefix after successful construction. + Unsafe.free(persistedPrefixAddr, persistedPrefixLen, MemoryTag.NATIVE_DEFAULT); + } else { + assert persistedPrefixAddr == sentDictBytesAddr; + sentDictBytesOwned = true; + } } + engine.releaseRecoveredSymbolStorage(); } this.fsnAtZero = fsnAtZero; this.parkNanos = parkNanos; diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java index 27523de4..4d216f8f 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java @@ -67,7 +67,6 @@ public final class MmapSegment implements QuietCloseable { public static final int HEADER_SIZE = 24; public static final byte LEGACY_VERSION = 1; public static final byte VERSION = 2; - private static final int[] CRC32C_TABLE = buildCrc32cTable(); private static final Logger LOG = LoggerFactory.getLogger(MmapSegment.class); private final String path; @@ -943,9 +942,13 @@ private static FrameScan scanFrames(long addr, long fileSize) { // the same scan). Folding over Unsafe keeps every fault // catchable -- handled below as the boundary of recoverable // data; a page that instead reads back as zeroes just fails the - // CRC check and ends the scan. Recovery is cold, so the slower - // table CRC here is immaterial. - int crcCalc = crc32cRecovery(addr + pos + 4, 4L + payloadLen); + // CRC check and ends the scan. The shared Java/Unsafe + // slice-by-8 path keeps those faults catchable without imposing + // a scalar byte-at-a-time scan on large recovery backlogs. + int crcCalc = Crc32c.updateUnsafe( + Crc32c.INIT, + addr + pos + 4, + 4L + payloadLen); if (crcCalc != crcRead) { return new FrameScan(pos, frameCount); } @@ -979,42 +982,6 @@ private static FrameScan scanFrames(long addr, long fileSize) { return new FrameScan(pos, frameCount); } - /** - * CRC-32C (Castagnoli) of {@code [addr, addr + len)} read through - * {@link Unsafe}, seeded like {@code Crc32c.update(Crc32c.INIT, addr, len)} - * and bit-identical to it (verified) -- but every byte load is an Unsafe - * intrinsic, so a fault on an unbacked mapped page is a catchable - * {@link InternalError} instead of the uncatchable JNI SIGBUS the native - * {@link Crc32c} would raise. Byte-at-a-time via a precomputed table - * ({@code ~0.5 GiB/s}); used only on the cold recovery scan, never on the - * append hot path (which stays on the native, hardware-friendly path). - */ - private static int crc32cRecovery(long addr, long len) { - int crc = ~Crc32c.INIT; - for (long i = 0; i < len; i++) { - crc = (crc >>> 8) ^ CRC32C_TABLE[(crc ^ Unsafe.getUnsafe().getByte(addr + i)) & 0xFF]; - } - return ~crc; - } - - /** - * Standard reflected CRC-32C byte table (polynomial {@code 0x82F63B78}), - * matching {@code crc32c_table[0]} in the native {@code crc32c.c}. Computed - * at class init to avoid 256 hand-transcribed literals; drives - * {@link #crc32cRecovery}. - */ - private static int[] buildCrc32cTable() { - int[] table = new int[256]; - for (int n = 0; n < 256; n++) { - int c = n; - for (int k = 0; k < 8; k++) { - c = (c & 1) != 0 ? (0x82F63B78 ^ (c >>> 1)) : (c >>> 1); - } - table[n] = c; - } - return table; - } - /** * Distinguishes "torn tail" (writer attempted a write past the last valid * frame and failed — partial write, mid-stream corruption, bit rot) from diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java index 914e4ab5..f38a5b11 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java @@ -163,7 +163,6 @@ public final class PersistedSymbolDict implements QuietCloseable { */ static final int MAX_SCRATCH_BYTES = Integer.MAX_VALUE - 8; static final byte VERSION = 3; // v3 moved the CRC-32C from per-entry to per-chunk - private static final int[] CRC32C_TABLE = buildCrc32cTable(); private static final Logger LOG = LoggerFactory.getLogger(PersistedSymbolDict.class); private final int fd; // Filesystem seam. Production is FilesFacade.INSTANCE (straight to Files); @@ -731,26 +730,6 @@ private static void copyRecoveredEntries(long inputAddr, int validLen, long entr assert dstPos == entriesLen; } - private static int[] buildCrc32cTable() { - int[] table = new int[256]; - for (int n = 0; n < 256; n++) { - int c = n; - for (int k = 0; k < 8; k++) { - c = (c & 1) != 0 ? (0x82F63B78 ^ (c >>> 1)) : (c >>> 1); - } - table[n] = c; - } - return table; - } - - private static int crc32cRecovery(long addr, long len) { - int crc = ~Crc32c.INIT; - for (long i = 0; i < len; i++) { - crc = (crc >>> 8) ^ CRC32C_TABLE[(crc ^ Unsafe.getUnsafe().getByte(addr + i)) & 0xFF]; - } - return ~crc; - } - private static RecoveryScan scanRecoveredChunks(long inputAddr, int len) { Varint v = new Varint(); int count = 0; @@ -772,7 +751,10 @@ private static RecoveryScan scanRecoveredChunks(long inputAddr, int len) { } int chunkEndI = (int) chunkEnd; int crcStored = Unsafe.getUnsafe().getInt(inputAddr + chunkEndI); - int crcCalc = crc32cRecovery(inputAddr + diskPos, chunkEndI - diskPos); + int crcCalc = Crc32c.updateUnsafe( + Crc32c.INIT, + inputAddr + diskPos, + chunkEndI - diskPos); if (crcCalc != crcStored || entryCount <= 0 || (long) count + entryCount > Integer.MAX_VALUE diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/RecoveredFrameAnalysis.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/RecoveredFrameAnalysis.java index cde5e06b..ed709d16 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/RecoveredFrameAnalysis.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/RecoveredFrameAnalysis.java @@ -195,19 +195,30 @@ int rawCapacity() { return rawCapacity; } + /** + * Releases the cached suffix after a foreground producer and its one send + * loop have both consumed it. Recovery metadata remains available, but the + * raw entries must not be requested again. + */ + void releaseRawStorage() { + if (rawAddr != 0L) { + Unsafe.free(rawAddr, rawCapacity, MemoryTag.NATIVE_DEFAULT); + rawAddr = 0L; + rawCapacity = 0; + } + runningRawLen = 0; + runningRawCount = 0; + committedRawLen = 0; + committedRawCount = 0; + } + long symbolEntriesVisited() { return symbolEntriesVisited; } @Override public void close() { - if (rawAddr != 0L) { - Unsafe.free(rawAddr, rawCapacity, MemoryTag.NATIVE_DEFAULT); - rawAddr = 0L; - rawCapacity = 0; - runningRawLen = 0; - committedRawLen = 0; - } + releaseRawStorage(); } private void appendRaw(long entryStart, int entryLen) { diff --git a/core/src/main/java/io/questdb/client/std/Crc32c.java b/core/src/main/java/io/questdb/client/std/Crc32c.java index d0a2e6a8..3f05ee84 100644 --- a/core/src/main/java/io/questdb/client/std/Crc32c.java +++ b/core/src/main/java/io/questdb/client/std/Crc32c.java @@ -46,6 +46,7 @@ public final class Crc32c { /** Seed value to start a fresh CRC-32C accumulation. */ public static final int INIT = 0; + private static final int[] CRC32C_TABLE = buildCrc32cTable(); private Crc32c() { } @@ -65,6 +66,69 @@ private Crc32c() { */ public static native int update(int seed, long addr, long len); + /** + * Java/Unsafe slice-by-8 CRC-32C for memory that can fault while it is read, + * such as a recovery mmap over a sparse or concurrently truncated file. + * Keeping every load at an {@link Unsafe} intrinsic site lets HotSpot turn an + * mmap access fault into a catchable {@link InternalError}; the native + * {@link #update} path cannot provide that guarantee because a SIGBUS raised + * inside JNI aborts the JVM. + * + * @param seed previous CRC value, or {@link #INIT} to start fresh + * @param addr off-heap address of at least {@code len} readable bytes + * @param len number of bytes to consume + * @return the new CRC value + */ + public static int updateUnsafe(int seed, long addr, long len) { + assert len >= 0L : "CRC length must be non-negative"; + int crc = ~seed; + int[] table = CRC32C_TABLE; + while (len >= Long.BYTES) { + int b0 = Unsafe.getUnsafe().getByte(addr) & 0xFF; + int b1 = Unsafe.getUnsafe().getByte(addr + 1) & 0xFF; + int b2 = Unsafe.getUnsafe().getByte(addr + 2) & 0xFF; + int b3 = Unsafe.getUnsafe().getByte(addr + 3) & 0xFF; + int b4 = Unsafe.getUnsafe().getByte(addr + 4) & 0xFF; + int b5 = Unsafe.getUnsafe().getByte(addr + 5) & 0xFF; + int b6 = Unsafe.getUnsafe().getByte(addr + 6) & 0xFF; + int b7 = Unsafe.getUnsafe().getByte(addr + 7) & 0xFF; + crc = table[7 * 256 + ((crc ^ b0) & 0xFF)] + ^ table[6 * 256 + (((crc >>> 8) ^ b1) & 0xFF)] + ^ table[5 * 256 + (((crc >>> 16) ^ b2) & 0xFF)] + ^ table[4 * 256 + (((crc >>> 24) ^ b3) & 0xFF)] + ^ table[3 * 256 + b4] + ^ table[2 * 256 + b5] + ^ table[256 + b6] + ^ table[b7]; + addr += Long.BYTES; + len -= Long.BYTES; + } + while (len-- > 0L) { + crc = (crc >>> 8) ^ table[(crc ^ Unsafe.getUnsafe().getByte(addr++)) & 0xFF]; + } + return ~crc; + } + + private static int[] buildCrc32cTable() { + int[] table = new int[8 * 256]; + for (int n = 0; n < 256; n++) { + int c = n; + for (int k = 0; k < 8; k++) { + c = (c & 1) != 0 ? (0x82F63B78 ^ (c >>> 1)) : (c >>> 1); + } + table[n] = c; + } + for (int slice = 1; slice < 8; slice++) { + int previousOffset = (slice - 1) * 256; + int offset = slice * 256; + for (int n = 0; n < 256; n++) { + int previous = table[previousOffset + n]; + table[offset + n] = (previous >>> 8) ^ table[previous & 0xFF]; + } + } + return table; + } + static { Os.init(); } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopMirrorLeakTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopMirrorLeakTest.java index 116cbc55..c2902e11 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopMirrorLeakTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopMirrorLeakTest.java @@ -190,12 +190,7 @@ public void testCtorFreesSeededMirrorWhenFrameSeedThrows() throws Exception { // rebuilds c@2 from the surviving frame -- the append the fault interrupts. populateThreeFrameSlot(sfDir); Path slot = sfDir.resolve("default"); - try (PersistedSymbolDict torn = PersistedSymbolDict.openClean(slot.toString())) { - Assert.assertNotNull(torn); - torn.appendSymbol("a"); - torn.appendSymbol("b"); - Assert.assertEquals(2, torn.size()); - } + replacePersistedDictionaryWithTwoSymbolPrefix(slot); assertMemoryLeak(() -> { try (CursorSendEngine engine = new CursorSendEngine(slot.toString(), 4096)) { @@ -222,6 +217,8 @@ public void testCtorFreesSeededMirrorWhenFrameSeedThrows() throws Exception { } Assert.assertTrue("failed foreground construction must leave the persisted prefix owned", pd.loadedEntriesAddr() != 0L); + Assert.assertTrue("failed construction must retain the recovered suffix for retry", + engine.recoverySymbolNativeCapacity() > 0); // The outer assertMemoryLeak proves the prefix-seeded mirror the ctor // malloc'd was freed on the throw -- pre-fix it leaks here. } @@ -231,6 +228,70 @@ public void testCtorFreesSeededMirrorWhenFrameSeedThrows() throws Exception { } } + @Test + public void testForegroundLoopReleasesRecoveredSuffixAfterSeeding() throws Exception { + Path sfDir = Files.createTempDirectory("qwp-mirror-suffix-release"); + try { + populateThreeFrameSlot(sfDir); + Path slot = sfDir.resolve("default"); + replacePersistedDictionaryWithTwoSymbolPrefix(slot); + + assertMemoryLeak(() -> { + try (CursorSendEngine engine = new CursorSendEngine(slot.toString(), 4096)) { + Assert.assertTrue("recovery must retain c@2 above the persisted [a,b] prefix", + engine.recoverySymbolNativeCapacity() > 0); + CursorWebSocketSendLoop loop = new CursorWebSocketSendLoop( + null, engine, 0, 1_000_000L, + () -> { + throw new IOException("no reconnect in this test"); + }, + 0, 0, 1); + try { + Assert.assertEquals("foreground mirror must include prefix and recovered suffix", + 3, readInt(loop, "sentDictCount")); + Assert.assertEquals("engine must release its duplicate recovery suffix", + 0, engine.recoverySymbolNativeCapacity()); + } finally { + loop.close(); + } + } + }); + } finally { + rmDir(sfDir); + } + } + + @Test + public void testOrphanLoopsRetainRecoveredSuffixForRecycle() throws Exception { + Path sfDir = Files.createTempDirectory("qwp-mirror-suffix-recycle"); + try { + populateThreeFrameSlot(sfDir); + Path slot = sfDir.resolve("default"); + replacePersistedDictionaryWithTwoSymbolPrefix(slot); + + assertMemoryLeak(() -> { + try (CursorSendEngine engine = new CursorSendEngine(slot.toString(), 4096)) { + int suffixCapacity = engine.recoverySymbolNativeCapacity(); + Assert.assertTrue("recovery must retain c@2 above the persisted [a,b] prefix", + suffixCapacity > 0); + for (int session = 0; session < 2; session++) { + CursorWebSocketSendLoop loop = newRecoveryLoop(engine); + try { + Assert.assertEquals("orphan mirror must include prefix and recovered suffix", + 3, readInt(loop, "sentDictCount")); + Assert.assertEquals("orphan engine must retain suffix for the next session", + suffixCapacity, engine.recoverySymbolNativeCapacity()); + } finally { + loop.close(); + } + } + } + }); + } finally { + rmDir(sfDir); + } + } + // Constructs a recovery send loop but does NOT start it: the ctor seeds the // catch-up mirror synchronously, which is all these tests observe. The // reconnect factory is never invoked. @@ -288,6 +349,15 @@ private static void populateThreeFrameSlot(Path sfDir) throws Exception { } } + private static void replacePersistedDictionaryWithTwoSymbolPrefix(Path slot) { + try (PersistedSymbolDict torn = PersistedSymbolDict.openClean(slot.toString())) { + Assert.assertNotNull(torn); + torn.appendSymbol("a"); + torn.appendSymbol("b"); + Assert.assertEquals(2, torn.size()); + } + } + private static int readInt(CursorWebSocketSendLoop loop, String name) throws Exception { Field f = CursorWebSocketSendLoop.class.getDeclaredField(name); f.setAccessible(true); diff --git a/core/src/test/java/io/questdb/client/test/std/Crc32cTest.java b/core/src/test/java/io/questdb/client/test/std/Crc32cTest.java index 90c96ebd..9d61c8f2 100644 --- a/core/src/test/java/io/questdb/client/test/std/Crc32cTest.java +++ b/core/src/test/java/io/questdb/client/test/std/Crc32cTest.java @@ -153,12 +153,48 @@ public void testKnownVector() throws Exception { } int crc = Crc32c.update(Crc32c.INIT, buf, msg.length); assertEquals(0xE3069283, crc); + assertEquals(crc, Crc32c.updateUnsafe(Crc32c.INIT, buf, msg.length)); } finally { Unsafe.free(buf, msg.length, MemoryTag.NATIVE_DEFAULT); } }); } + @Test + public void testUnsafeMatchesNativeAcrossAlignmentsLengthsAndSeeds() throws Exception { + TestUtils.assertMemoryLeak(() -> { + int[] lengths = { + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 15, 16, 17, + 31, 32, 33, 63, 64, 65, 127, 128, 129, 255, 256, + 257, 1_023, 4_096 + }; + int[] seeds = {Crc32c.INIT, 0x1234_5678, -1}; + int capacity = 4_096 + Long.BYTES - 1; + long buf = Unsafe.malloc(capacity, MemoryTag.NATIVE_DEFAULT); + try { + int value = 0x1357_9BDF; + for (int i = 0; i < capacity; i++) { + value ^= value << 13; + value ^= value >>> 17; + value ^= value << 5; + Unsafe.getUnsafe().putByte(buf + i, (byte) value); + } + for (int offset = 0; offset < Long.BYTES; offset++) { + for (int len : lengths) { + for (int seed : seeds) { + Assert.assertEquals( + "offset=" + offset + " len=" + len + " seed=" + seed, + Crc32c.update(seed, buf + offset, len), + Crc32c.updateUnsafe(seed, buf + offset, len)); + } + } + } + } finally { + Unsafe.free(buf, capacity, MemoryTag.NATIVE_DEFAULT); + } + }); + } + @Test public void testZerosHaveStableCrc() throws Exception { TestUtils.assertMemoryLeak(() -> { From 4386ca1695f6d8332b2980d931d2177f34eff9a6 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Fri, 17 Jul 2026 11:57:35 +0100 Subject: [PATCH 82/96] Optimize QWP replay and dictionary catch-up --- .../cutlass/http/client/WebSocketClient.java | 36 ++++ .../sf/cursor/CursorWebSocketSendLoop.java | 160 ++++++++++++++---- .../client/sf/cursor/PersistedSymbolDict.java | 19 ++- .../qwp/client/sf/cursor/SegmentRing.java | 108 +++++++++--- .../http/client/WebSocketClientTest.java | 64 +++++++ .../qwp/client/DeltaDictRecoveryTest.java | 10 ++ ...WebSocketSendLoopCatchUpAlignmentTest.java | 40 ++++- .../sf/cursor/PersistedSymbolDictTest.java | 12 +- .../qwp/client/sf/cursor/SegmentRingTest.java | 28 +++ 9 files changed, 408 insertions(+), 69 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java index fd16dca7..ef5a6bf4 100644 --- a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java +++ b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java @@ -460,6 +460,35 @@ public void sendBinary(long dataPtr, int length, int timeout) { sendBuffer.reset(); } + /** + * Sends two native-memory slices as one WebSocket binary frame. The slices + * are copied directly into the WebSocket send buffer and masked there, so a + * caller assembling a small protocol prefix around a large immutable body + * does not need a second contiguous staging buffer. + * + * @param firstPtr pointer to the first payload slice + * @param firstLength first payload-slice length + * @param secondPtr pointer to the second payload slice + * @param secondLength second payload-slice length + * @param timeout timeout in milliseconds + */ + public void sendBinary( + long firstPtr, + int firstLength, + long secondPtr, + int secondLength, + int timeout + ) { + checkConnected(); + sendBuffer.reset(); + sendBuffer.beginFrame(); + sendBuffer.putBlockOfBytes(firstPtr, firstLength); + sendBuffer.putBlockOfBytes(secondPtr, secondLength); + WebSocketSendBuffer.FrameInfo frame = sendBuffer.endBinaryFrame(); + doSend(sendBuffer.getBufferPtr() + frame.offset, frame.length, timeout); + sendBuffer.reset(); + } + /** * Sends binary data with default timeout. */ @@ -467,6 +496,13 @@ public void sendBinary(long dataPtr, int length) { sendBinary(dataPtr, length, defaultTimeout); } + /** + * Sends two native-memory slices as one binary frame with the default timeout. + */ + public void sendBinary(long firstPtr, int firstLength, long secondPtr, int secondLength) { + sendBinary(firstPtr, firstLength, secondPtr, secondLength, defaultTimeout); + } + /** * Sends a close frame. */ diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java index d0340f6c..cfde65e6 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java @@ -193,6 +193,7 @@ public final class CursorWebSocketSendLoop implements QuietCloseable { // workload. The guard exists so that pathological growth fails loudly instead // of overflowing the int capacity math into a heap-corrupting copyMemory. private static final int MAX_SENT_DICT_BYTES = Integer.MAX_VALUE - 8; + private static final int MAX_SENT_DICT_ENTRIES = MAX_SENT_DICT_BYTES / Integer.BYTES; /** * Throttle "reconnect attempt N failed" WARN logs to one per 5 s. */ @@ -306,6 +307,12 @@ public final class CursorWebSocketSendLoop implements QuietCloseable { // Any growth first performs a copy-on-write; only owned buffers are freed. private boolean sentDictBytesOwned; private int sentDictCount; + // Relative byte end of every [len varint][utf8] entry in sentDictBytesAddr. + // The mirror is parsed once as it is built/recovered; reconnect catch-up then + // reads fixed-width offsets instead of decoding every length varint again. + private long sentDictEntryEndsAddr; + private int sentDictEntryEndsCapacity; + private int sentDictIndexedCount; // True when replay frames can start above dictionary id zero and therefore // depend on a catch-up on a fresh connection. Delta-enabled live engines // always have this dependency. A recovered delta slot whose dictionary @@ -313,12 +320,12 @@ public final class CursorWebSocketSendLoop implements QuietCloseable { // remaining false case is full-dictionary fallback: every frame re-registers // from id zero, so sending the mirror first would duplicate the dictionary. private final boolean hasReplayDictionaryDependency; - // Reusable staging frame for dictionary catch-up chunks. A reconnect may split a - // large dictionary into many chunks, and every later reconnect repeats that split; - // retaining one growable loop-owned buffer turns the old malloc/free-per-chunk path - // into amortized growth only. I/O-thread-owned, freed beside sentDictBytesAddr. + // Reusable staging buffer for the small QWP catch-up prefix (fixed header plus + // delta range varints). Symbol bytes stay in sentDictBytesAddr and are passed as a + // second WebSocket payload slice, avoiding a full-dictionary staging copy. private long catchUpFrameAddr; private int catchUpFrameCapacity; + private int catchUpEntryIndexBuildCount; private int catchUpFrameGrowthCount; // Orphan-policy cap-gap attempts -- catch-ups that reached a node and found an entry // too large for its batch cap -- with no intervening successful catch-up or unrelated @@ -767,16 +774,12 @@ public CursorWebSocketSendLoop(WebSocketClient client, CursorSendEngine engine, // re-registers its dictionary from id 0 as it replays, so seeding the mirror would buy // nothing and cost a catch-up frame on every connection -- full-dict slots must stay // catch-up-free. - if (engine.recoveredMaxSymbolDeltaStart() > 0L) { - // The prefix seed above may already have malloc'd the mirror, and - // ensureSentDictCapacity can still throw here (a native realloc OOM, or - // the MAX_SENT_DICT_BYTES ceiling). Such a throw - // propagates out of the constructor, so the half-built loop is never - // assigned to a reference -- neither ensureConnected's catch nor - // BackgroundDrainer's finally can close() it -- and the mirror would leak. - // Free it on any throw so the constructor leaves nothing behind, mirroring - // the loopNeverRan free in close() / ioLoop's exit. - try { + // The prefix seed above may be borrowed, while the entry-end index is always + // loop-owned native memory. Build/extend both inside one cleanup boundary: + // any allocation failure propagates out of the constructor before a caller + // can own the half-built loop, so nothing else could release them. + try { + if (engine.recoveredMaxSymbolDeltaStart() > 0L) { int baseline = sentDictCount; if (engine.recoveredSymbolCoverage(baseline) >= 0L) { int suffixLen = engine.recoveredSymbolSuffixLen(baseline); @@ -793,10 +796,13 @@ public CursorWebSocketSendLoop(WebSocketClient client, CursorSendEngine engine, sentDictCount += suffixCount; } } - } catch (Throwable t) { - releaseSentDictBytes(); - throw t; } + if (sentDictCount > 0) { + rebuildSentDictEntryIndex(); + } + } catch (Throwable t) { + releaseSentDictBytes(); + throw t; } // QwpWebSocketSender seeds the producer dictionary before constructing // its one foreground loop, so after the loop has built its mirror it can @@ -2415,6 +2421,7 @@ private void accumulateSentDict(long payloadAddr, int payloadLen, int deltaStart if (deltaCount <= 0 || deltaStart > sentDictCount || deltaEnd <= sentDictCount) { return; } + ensureSentDictEntryIndex(); // Walk past the already-held prefix [deltaStart, sentDictCount), then copy // the new tail [sentDictCount, deltaEnd). int skip = sentDictCount - deltaStart; @@ -2427,6 +2434,8 @@ private void accumulateSentDict(long payloadAddr, int payloadLen, int deltaStart } long regionStart = p; long newCount = deltaEnd - sentDictCount; + long requiredEntryCount = (long) sentDictCount + newCount; + ensureSentDictEntryEndsCapacity(requiredEntryCount); for (long i = 0; i < newCount; i++) { long len = readVarintAt(p, limit); p = varintEnd + len; @@ -2435,6 +2444,9 @@ private void accumulateSentDict(long payloadAddr, int payloadLen, int deltaStart // than corrupt the mirror. return; } + Unsafe.getUnsafe().putInt( + sentDictEntryEndsAddr + ((long) sentDictCount + i) * Integer.BYTES, + sentDictBytesLen + (int) (p - regionStart)); } int regionBytes = (int) (p - regionStart); // long sum: sentDictBytesLen + regionBytes can exceed Integer.MAX_VALUE on @@ -2445,6 +2457,7 @@ private void accumulateSentDict(long payloadAddr, int payloadLen, int deltaStart Unsafe.getUnsafe().copyMemory(regionStart, sentDictBytesAddr + sentDictBytesLen, regionBytes); sentDictBytesLen += regionBytes; sentDictCount += (int) newCount; + sentDictIndexedCount = sentDictCount; } /** @@ -2460,12 +2473,17 @@ private void appendSymbolToMirror(CharSequence symbol) { int utf8Len = Utf8s.utf8Bytes(symbol); int wireLen = NativeBufferWriter.varintSize(utf8Len) + utf8Len; ensureSentDictCapacity((long) sentDictBytesLen + wireLen); + ensureSentDictEntryEndsCapacity((long) sentDictCount + 1L); long p = NativeBufferWriter.writeVarint(sentDictBytesAddr + sentDictBytesLen, utf8Len); if (utf8Len > 0) { Utf8s.strCpyUtf8(symbol, p, utf8Len); } sentDictBytesLen += wireLen; + Unsafe.getUnsafe().putInt( + sentDictEntryEndsAddr + (long) sentDictCount * Integer.BYTES, + sentDictBytesLen); sentDictCount++; + sentDictIndexedCount = sentDictCount; } private void ensureSentDictCapacity(long required) { @@ -2514,11 +2532,76 @@ private void releaseSentDictBytes() { if (sentDictBytesAddr != 0 && sentDictBytesOwned) { Unsafe.free(sentDictBytesAddr, sentDictBytesCapacity, MemoryTag.NATIVE_DEFAULT); } + if (sentDictEntryEndsAddr != 0L) { + Unsafe.free( + sentDictEntryEndsAddr, + sentDictEntryEndsCapacity * Integer.BYTES, + MemoryTag.NATIVE_DEFAULT); + } sentDictBytesAddr = 0; sentDictBytesCapacity = 0; sentDictBytesLen = 0; sentDictBytesOwned = false; sentDictCount = 0; + sentDictEntryEndsAddr = 0L; + sentDictEntryEndsCapacity = 0; + sentDictIndexedCount = 0; + } + + private void ensureSentDictEntryEndsCapacity(long required) { + if (required <= sentDictEntryEndsCapacity) { + return; + } + if (required > MAX_SENT_DICT_ENTRIES) { + LineSenderException err = new LineSenderException( + "symbol dictionary entry index exceeds the maximum size [required=" + + required + ", max=" + MAX_SENT_DICT_ENTRIES + ']'); + recordFatal(err); + throw err; + } + long newCapacity = Math.max( + (long) sentDictEntryEndsCapacity * 2L, + Math.max(1024L, required)); + if (newCapacity > MAX_SENT_DICT_ENTRIES) { + newCapacity = MAX_SENT_DICT_ENTRIES; + } + sentDictEntryEndsAddr = Unsafe.realloc( + sentDictEntryEndsAddr, + sentDictEntryEndsCapacity * Integer.BYTES, + (int) newCapacity * Integer.BYTES, + MemoryTag.NATIVE_DEFAULT); + sentDictEntryEndsCapacity = (int) newCapacity; + } + + private void ensureSentDictEntryIndex() { + if (sentDictIndexedCount != sentDictCount) { + rebuildSentDictEntryIndex(); + } + } + + private void rebuildSentDictEntryIndex() { + ensureSentDictEntryEndsCapacity(sentDictCount); + long p = sentDictBytesAddr; + long limit = sentDictBytesAddr + sentDictBytesLen; + for (int i = 0; i < sentDictCount; i++) { + long len = readVarintAt(p, limit); + p = varintEnd + len; + if (p > limit) { + throw new LineSenderException( + "invalid symbol dictionary mirror while building the entry index [entry=" + + i + ", count=" + sentDictCount + ']'); + } + Unsafe.getUnsafe().putInt( + sentDictEntryEndsAddr + (long) i * Integer.BYTES, + (int) (p - sentDictBytesAddr)); + } + if (p != limit) { + throw new LineSenderException( + "invalid symbol dictionary mirror length while building the entry index [indexed=" + + (p - sentDictBytesAddr) + ", length=" + sentDictBytesLen + ']'); + } + sentDictIndexedCount = sentDictCount; + catchUpEntryIndexBuildCount++; } private long readVarintAt(long p, long limit) { @@ -2559,6 +2642,16 @@ private long readVarintAt(long p, long limit) { */ private int sendDictCatchUp() { int cap = client.getServerMaxBatchSize(); + long fullFrameLen = QwpConstants.HEADER_SIZE + + NativeBufferWriter.varintSize(0) + + NativeBufferWriter.varintSize(sentDictCount) + + (long) sentDictBytesLen; + if (cap <= 0 && fullFrameLen <= MAX_SENT_DICT_BYTES) { + sendCatchUpChunk(0, sentDictCount, sentDictBytesAddr, sentDictBytesLen); + resetCatchUpCapGapEpisode(); + return 1; + } + ensureSentDictEntryIndex(); // The frame ceiling a catch-up chunk must not exceed: the server's // advertised cap, or -- when the server advertises none (cap <= 0) -- // MAX_SENT_DICT_BYTES so sendCatchUpChunk's int frameLen (HEADER_SIZE + @@ -2582,13 +2675,15 @@ private int sendDictCatchUp() { long chunkStartAddr = sentDictBytesAddr; int chunkSymbols = 0; long chunkBytes = 0; - long p = sentDictBytesAddr; - long limit = sentDictBytesAddr + sentDictBytesLen; - while (p < limit) { - long entryStart = p; - long len = readVarintAt(p, limit); // entry length prefix; varintEnd -> just past prefix - long entryEnd = varintEnd + len; // just past [len varint][utf8 bytes] - long entryBytes = entryEnd - entryStart; + for (int entryId = 0; entryId < sentDictCount; entryId++) { + int entryStartOffset = entryId == 0 + ? 0 + : Unsafe.getUnsafe().getInt( + sentDictEntryEndsAddr + (long) (entryId - 1) * Integer.BYTES); + int entryEndOffset = Unsafe.getUnsafe().getInt( + sentDictEntryEndsAddr + (long) entryId * Integer.BYTES); + long entryStart = sentDictBytesAddr + entryStartOffset; + long entryBytes = entryEndOffset - entryStartOffset; // The exact table-less frame sendCatchUpChunk would build for THIS entry // alone: header + deltaStart varint (the entry's own global id) + // deltaCount varint (1) + the entry bytes. A cap gap exists only when even @@ -2676,7 +2771,6 @@ private int sendDictCatchUp() { } chunkSymbols++; chunkBytes += entryBytes; - p = entryEnd; } if (chunkSymbols > 0) { sendCatchUpChunk(chunkStartId, chunkSymbols, chunkStartAddr, (int) chunkBytes); @@ -2716,8 +2810,10 @@ private void sendCatchUpChunk(int deltaStart, int deltaCount, long symbolsAddr, throw new CatchUpSendException(err); } int payloadLen = (int) payloadLenL; - int frameLen = (int) frameLenL; - ensureCatchUpFrameCapacity(frameLen); + int prefixLen = QwpConstants.HEADER_SIZE + + NativeBufferWriter.varintSize(deltaStart) + + NativeBufferWriter.varintSize(deltaCount); + ensureCatchUpFrameCapacity(prefixLen); long frame = catchUpFrameAddr; try { Unsafe.getUnsafe().putByte(frame, (byte) 'Q'); @@ -2740,9 +2836,8 @@ private void sendCatchUpChunk(int deltaStart, int deltaCount, long symbolsAddr, Unsafe.getUnsafe().putShort(frame + 6, (short) 0); // tableCount Unsafe.getUnsafe().putInt(frame + 8, payloadLen); long q = NativeBufferWriter.writeVarint(frame + QwpConstants.HEADER_SIZE, deltaStart); - q = NativeBufferWriter.writeVarint(q, deltaCount); - Unsafe.getUnsafe().copyMemory(symbolsAddr, q, symbolsLen); - client.sendBinary(frame, frameLen); + NativeBufferWriter.writeVarint(q, deltaCount); + client.sendBinary(frame, prefixLen, symbolsAddr, symbolsLen); } catch (Throwable t) { // Do NOT fail() here -- see CatchUpSendException. Signal the failure // up so exactly one non-re-entrant reconnect follows. A JVM Error is @@ -2758,6 +2853,11 @@ private void sendCatchUpChunk(int deltaStart, int deltaCount, long symbolsAddr, totalFramesSent.incrementAndGet(); } + @TestOnly + public int catchUpEntryIndexBuildCount() { + return catchUpEntryIndexBuildCount; + } + @TestOnly public int catchUpFrameGrowthCount() { return catchUpFrameGrowthCount; diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java index f38a5b11..96e26df9 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java @@ -146,7 +146,10 @@ public final class PersistedSymbolDict implements QuietCloseable { static final int CRC_SIZE = 4; // u32 CRC-32C trailing every chunk static final int FILE_MAGIC = 0x31445953; // 'SYD1' little-endian static final int HEADER_SIZE = 8; - static final int INITIAL_APPEND_MAP_CAPACITY = 64 * 1024; + // One bounded, segment-sized append window avoids the allocate/unmap/mmap + // cycle every 64 KiB without geometrically reserving up to 2x a large + // dictionary. close() truncates the unused tail back to appendOffset. + static final int APPEND_MAP_CAPACITY = 4 * 1024 * 1024; /** * Upper bound on a chunk's two header varints ({@code entryCount} and * {@code entryBytes}): each is at most 5 bytes for a 32-bit value. The @@ -883,9 +886,9 @@ private void commitMappedChunk(long recStart, int hdrLen, int entriesLen, int co /** * Ensures the production append mmap covers the absolute file offset - * {@code required}. The log is mapped in 64 KiB-aligned windows: small flushes - * share a window, while a large existing dictionary does not force the process - * to reserve and remap its whole prefix (or geometrically over-allocate it). + * {@code required}. The log is mapped in 4 MiB-aligned windows: small flushes + * share a segment-sized window, while a large existing dictionary does not force + * the process to reserve and remap its whole prefix or geometrically over-allocate it. */ private void ensureAppendMap(long required) { if (appendMapAddr != 0L @@ -893,12 +896,12 @@ private void ensureAppendMap(long required) { && required <= appendMapOffset + appendMapCapacity) { return; } - long newOffset = appendOffset - appendOffset % INITIAL_APPEND_MAP_CAPACITY; + long newOffset = appendOffset - appendOffset % APPEND_MAP_CAPACITY; long needed = required - newOffset; - long newCapacity = Math.max(INITIAL_APPEND_MAP_CAPACITY, needed); - long remainder = newCapacity % INITIAL_APPEND_MAP_CAPACITY; + long newCapacity = Math.max(APPEND_MAP_CAPACITY, needed); + long remainder = newCapacity % APPEND_MAP_CAPACITY; if (remainder != 0L) { - long padding = INITIAL_APPEND_MAP_CAPACITY - remainder; + long padding = APPEND_MAP_CAPACITY - remainder; if (newCapacity > Long.MAX_VALUE - padding) { throw new IllegalStateException("symbol dict mmap capacity overflow to " + FILE_NAME + " [required=" + required + ']'); diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java index 735e1d0f..45aaebb3 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java @@ -71,6 +71,7 @@ public final class SegmentRing implements QuietCloseable { // (CI runner variance makes elapsed-millisecond bounds flaky). Cheap // in production: one volatile-free add per partition pass, dwarfed by // the mmap I/O the recovery does on every segment. + private static final int SEALED_PREFIX_COMPACTION_THRESHOLD = 64; private static long sortComparisons; private final long maxBytesPerSegment; // Sealed segments in baseSeq order, oldest first. Active is held separately. @@ -80,6 +81,16 @@ public final class SegmentRing implements QuietCloseable { // looks at sealedSegments after observing a higher ackedFsn, by which // point the producer thread's add to sealedSegments has retired. private final ObjList sealedSegments = new ObjList<>(); + // Logical deque head. Trimming nulls and advances this cursor instead of + // shifting the remaining ObjList on every ACK. When the dead prefix is at + // least half the backing list (and non-trivially large), one bulk compaction + // reclaims it. Each live reference is therefore moved at most once per + // halving, making sequential trim O(N) overall instead of O(N^2). + private int sealedHead; + // Deterministic performance probes: unlike elapsed-time assertions these + // let tests pin binary successor search and amortized prefix compaction. + private int lastNextSealedSearchComparisons; + private long sealedCompactionMoves; // High-water byte offset within the active segment at which we proactively // ask the segment manager to provision a spare (if one isn't already // installed). Computed once as 3/4 of segment capacity -- leaves the manager @@ -460,13 +471,14 @@ public synchronized void close() { hotSpare.close(); hotSpare = null; } - for (int i = 0, n = sealedSegments.size(); i < n; i++) { + for (int i = sealedHead, n = sealedSegments.size(); i < n; i++) { MmapSegment s = sealedSegments.get(i); if (s != null) { s.close(); } } sealedSegments.clear(); + sealedHead = 0; } /** @@ -483,9 +495,10 @@ public synchronized ObjList drainTrimmable() { // Sealed segments are in baseSeq order, oldest first; once we hit one // that isn't fully acked, none of the later ones can be either. // Synchronized so the I/O thread's snapshotSealedSegments() can't - // race against the remove(0) shuffling slots underneath it. - while (sealedSegments.size() > 0) { - MmapSegment s = sealedSegments.get(0); + // race against logical-head advancement or a bulk prefix compaction. + int n = sealedSegments.size(); + while (sealedHead < n) { + MmapSegment s = sealedSegments.get(sealedHead); long lastSeq = s.baseSeq() + s.frameCount() - 1; if (lastSeq > acked) { break; @@ -494,8 +507,9 @@ public synchronized ObjList drainTrimmable() { out = new ObjList<>(); } out.add(s); - sealedSegments.remove(0); + sealedSegments.setQuick(sealedHead++, null); } + compactSealedPrefixIfNeeded(); return out; } @@ -511,7 +525,7 @@ public synchronized ObjList drainTrimmable() { * scan cost doesn't matter. */ public synchronized MmapSegment findSegmentContaining(long fsn) { - for (int i = 0, n = sealedSegments.size(); i < n; i++) { + for (int i = sealedHead, n = sealedSegments.size(); i < n; i++) { MmapSegment s = sealedSegments.get(i); long base = s.baseSeq(); if (fsn >= base && fsn < base + s.frameCount()) { @@ -534,7 +548,7 @@ public synchronized MmapSegment findSegmentContaining(long fsn) { * fallback -- see {@link #nextSealedAfter(MmapSegment)}. */ public synchronized MmapSegment firstSealed() { - return sealedSegments.size() > 0 ? sealedSegments.get(0) : null; + return sealedHead < sealedSegments.size() ? sealedSegments.get(sealedHead) : null; } /** Active segment -- exposed for the I/O thread's "send next batch" path. */ @@ -557,7 +571,7 @@ public synchronized MmapSegment firstSealed() { */ public synchronized long findLastFsnWithoutPayloadFlag(int flagsOffset, int flagMask, int headerMagic, int minPayloadLen) { long best = -1L; - for (int i = 0, n = sealedSegments.size(); i < n; i++) { + for (int i = sealedHead, n = sealedSegments.size(); i < n; i++) { long fsn = sealedSegments.get(i).findLastFrameFsnWithoutPayloadFlag(flagsOffset, flagMask, headerMagic, minPayloadLen); if (fsn > best) { best = fsn; @@ -589,7 +603,7 @@ public synchronized long collectReplaySymbolsAbove( ObjList out ) { long coverage = baseline; - for (int i = 0, n = sealedSegments.size(); i < n; i++) { + for (int i = sealedHead, n = sealedSegments.size(); i < n; i++) { coverage = sealedSegments.get(i).collectReplaySymbolsAbove( headerMagic, flagsOffset, flagDeltaMask, qwpHeaderSize, minFsnInclusive, maxFsnInclusive, coverage, out); @@ -609,7 +623,7 @@ public synchronized long collectReplaySymbolsAbove( synchronized RecoveredFrameAnalysis analyzeRecovery(int symbolBaseline) { RecoveredFrameAnalysis analysis = new RecoveredFrameAnalysis(symbolBaseline, ackedFsn); try { - for (int i = 0, n = sealedSegments.size(); i < n; i++) { + for (int i = sealedHead, n = sealedSegments.size(); i < n; i++) { sealedSegments.get(i).scanRecovery(analysis); } active.scanRecovery(analysis); @@ -631,7 +645,7 @@ synchronized RecoveredFrameAnalysis analyzeRecovery(int symbolBaseline) { */ public synchronized long maxSymbolDeltaEnd(int headerMagic, int flagsOffset, int flagDeltaMask, int qwpHeaderSize, long maxFsnInclusive) { long maxEnd = 0L; - for (int i = 0, n = sealedSegments.size(); i < n; i++) { + for (int i = sealedHead, n = sealedSegments.size(); i < n; i++) { long end = sealedSegments.get(i).maxSymbolDeltaEnd(headerMagic, flagsOffset, flagDeltaMask, qwpHeaderSize, maxFsnInclusive); if (end > maxEnd) { maxEnd = end; @@ -650,7 +664,7 @@ public synchronized long maxSymbolDeltaEnd(int headerMagic, int flagsOffset, int */ public synchronized long maxSymbolDeltaStart(int headerMagic, int flagsOffset, int flagDeltaMask, int qwpHeaderSize, long maxFsnInclusive) { long maxStart = 0L; - for (int i = 0, n = sealedSegments.size(); i < n; i++) { + for (int i = sealedHead, n = sealedSegments.size(); i < n; i++) { long start = sealedSegments.get(i).maxSymbolDeltaStart(headerMagic, flagsOffset, flagDeltaMask, qwpHeaderSize, maxFsnInclusive); if (start > maxStart) { maxStart = start; @@ -670,7 +684,8 @@ public MmapSegment getActive() { * concurrent rotation. Cross-thread readers (typically the I/O loop) * should use {@link #snapshotSealedSegments(MmapSegment[])} instead. */ - public ObjList getSealedSegments() { + public synchronized ObjList getSealedSegments() { + compactSealedPrefix(); return sealedSegments; } @@ -722,13 +737,20 @@ public boolean needsHotSpare() { */ public synchronized MmapSegment nextSealedAfter(MmapSegment current) { long currentBase = current.baseSeq(); - for (int i = 0, n = sealedSegments.size(); i < n; i++) { - MmapSegment s = sealedSegments.get(i); - if (s.baseSeq() > currentBase) { - return s; + int lo = sealedHead; + int hi = sealedSegments.size(); + int comparisons = 0; + while (lo < hi) { + int mid = (lo + hi) >>> 1; + comparisons++; + if (sealedSegments.get(mid).baseSeq() <= currentBase) { + lo = mid + 1; + } else { + hi = mid; } } - return null; + lastNextSealedSearchComparisons = comparisons; + return lo < sealedSegments.size() ? sealedSegments.get(lo) : null; } /** @@ -777,15 +799,15 @@ public void setManagerWakeup(Runnable wakeup) { * the I/O loop is about to do. */ public synchronized int snapshotSealedSegments(MmapSegment[] target) { - int n = sealedSegments.size(); + int n = sealedSegments.size() - sealedHead; if (n > target.length) { for (int i = 0; i < target.length; i++) { - target[i] = sealedSegments.get(i); + target[i] = sealedSegments.get(sealedHead + i); } return -1; } for (int i = 0; i < n; i++) { - target[i] = sealedSegments.get(i); + target[i] = sealedSegments.get(sealedHead + i); } return n; } @@ -803,13 +825,23 @@ public synchronized long totalSegmentBytes() { if (a != null) total += a.sizeBytes(); MmapSegment hs = hotSpare; if (hs != null) total += hs.sizeBytes(); - for (int i = 0, n = sealedSegments.size(); i < n; i++) { + for (int i = sealedHead, n = sealedSegments.size(); i < n; i++) { MmapSegment s = sealedSegments.get(i); if (s != null) total += s.sizeBytes(); } return total; } + @TestOnly + public synchronized int getLastNextSealedSearchComparisons() { + return lastNextSealedSearchComparisons; + } + + @TestOnly + public synchronized long getSealedCompactionMoves() { + return sealedCompactionMoves; + } + /** * Returns the cumulative count of baseSeq comparisons performed by * {@link #sortByBaseSeq} since the last {@link #resetSortComparisons()} @@ -831,6 +863,38 @@ public static void resetSortComparisons() { sortComparisons = 0; } + private void compactSealedPrefix() { + if (sealedHead == 0) { + return; + } + int size = sealedSegments.size(); + int remaining = size - sealedHead; + if (remaining == 0) { + // Every retired slot was nulled while draining, so resetting pos + // neither retains references nor has to clear the backing array. + sealedSegments.setPos(0); + } else { + sealedCompactionMoves += remaining; + for (int i = 0; i < remaining; i++) { + sealedSegments.setQuick(i, sealedSegments.get(sealedHead + i)); + } + for (int i = remaining; i < size; i++) { + sealedSegments.setQuick(i, null); + } + sealedSegments.setPos(remaining); + } + sealedHead = 0; + } + + private void compactSealedPrefixIfNeeded() { + int size = sealedSegments.size(); + if (sealedHead == size + || (sealedHead >= SEALED_PREFIX_COMPACTION_THRESHOLD + && sealedHead >= size - sealedHead)) { + compactSealedPrefix(); + } + } + /** * In-place quicksort over {@code list[lo, hi)} keyed by ascending * {@code baseSeq}. Median-of-three pivot avoids the pathological O(N²) diff --git a/core/src/test/java/io/questdb/client/test/cutlass/http/client/WebSocketClientTest.java b/core/src/test/java/io/questdb/client/test/cutlass/http/client/WebSocketClientTest.java index a44c57f2..93a380f8 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/http/client/WebSocketClientTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/http/client/WebSocketClientTest.java @@ -32,6 +32,7 @@ import io.questdb.client.network.PlainSocketFactory; import io.questdb.client.network.Socket; import io.questdb.client.network.SocketReadinessWaiter; +import io.questdb.client.std.MemoryTag; import io.questdb.client.std.Unsafe; import org.junit.Assert; import org.junit.Test; @@ -312,6 +313,41 @@ public void onClose(int code, String reason) { }); } + @Test + public void testSendBinaryAssemblesTwoPayloadSlicesIntoOneMaskedFrame() throws Exception { + assertMemoryLeak(() -> { + CapturingSocket socket = new CapturingSocket(); + long first = Unsafe.malloc(3, MemoryTag.NATIVE_DEFAULT); + long second = Unsafe.malloc(4, MemoryTag.NATIVE_DEFAULT); + try (CapturingWebSocketClient client = new CapturingWebSocketClient(socket)) { + setUpgradedTrue(client); + for (int i = 0; i < 3; i++) { + Unsafe.getUnsafe().putByte(first + i, (byte) (i + 1)); + } + for (int i = 0; i < 4; i++) { + Unsafe.getUnsafe().putByte(second + i, (byte) (i + 4)); + } + + client.sendBinary(first, 3, second, 4); + + byte[] frame = socket.sent; + Assert.assertNotNull(frame); + Assert.assertEquals((byte) 0x82, frame[0]); + Assert.assertEquals((byte) (0x80 | 7), frame[1]); + byte[] payload = new byte[7]; + for (int i = 0; i < payload.length; i++) { + payload[i] = (byte) (frame[6 + i] ^ frame[2 + (i & 3)]); + } + Assert.assertArrayEquals( + new byte[]{1, 2, 3, 4, 5, 6, 7}, + payload); + } finally { + Unsafe.free(first, 3, MemoryTag.NATIVE_DEFAULT); + Unsafe.free(second, 4, MemoryTag.NATIVE_DEFAULT); + } + }); + } + @Test public void testSendCloseFrameDoesNotClobberSendBuffer() throws Exception { assertMemoryLeak(() -> { @@ -463,6 +499,34 @@ public boolean wantsTlsWrite() { } } + private static class CapturingSocket extends FakeSocket { + private byte[] sent; + + @Override + public int send(long bufferPtr, int bufferLen) { + sent = new byte[bufferLen]; + for (int i = 0; i < bufferLen; i++) { + sent[i] = Unsafe.getUnsafe().getByte(bufferPtr + i); + } + return bufferLen; + } + } + + private static class CapturingWebSocketClient extends WebSocketClient { + + CapturingWebSocketClient(CapturingSocket socket) { + super(DefaultHttpClientConfiguration.INSTANCE, (nf, log) -> socket); + } + + @Override + protected void ioWait(int timeout, int op) { + } + + @Override + protected void setupIoWait() { + } + } + /** * Socket that serves a fixed byte sequence from recv() and reports * every send() as fully written (so close-frame sends succeed). diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java index 61c4495e..c0b517d1 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java @@ -1392,6 +1392,7 @@ private static class DictReconstructingHandler implements TestWebSocketServer.We private final AtomicLong nextSeq = new AtomicLong(0); private TestWebSocketServer.ClientHandler currentClient; private int dataFrameCount; + private TestWebSocketServer.ClientHandler droppedClient; private boolean firstDataFrameDropped; private volatile int lastDataDeltaStart = -1; @@ -1425,6 +1426,14 @@ int lastDataDeltaStart() { @Override public synchronized void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] data) { + // Closing the socket below does not discard frames that its reader has + // already decoded. A faster replay loop can therefore deliver another + // callback for the deliberately dropped client while (or even after) + // the replacement connection starts. Those stale frames must not reset + // the fresh connection's synthetic dictionary / ACK sequence. + if (client == droppedClient) { + return; + } boolean newConnection = currentClient != client; if (newConnection) { currentClient = client; @@ -1440,6 +1449,7 @@ public synchronized void onBinaryMessage(TestWebSocketServer.ClientHandler clien lastDataDeltaStart = QwpWireTestUtils.readVarint(data, new int[]{12}); if (dropFirstDataFrame && !firstDataFrameDropped) { firstDataFrameDropped = true; + droppedClient = client; client.close(); return; } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java index 86ba58a0..00a0f230 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java @@ -163,29 +163,37 @@ public void testSplitCatchUpFramesAcksDoNotAdvanceTrimWatermark() throws Excepti } @Test - public void testSplitCatchUpReusesOneFrameBufferAcrossReconnects() throws Exception { + public void testSplitCatchUpReusesEntryIndexAndStagesOnlyPrefixAcrossReconnects() throws Exception { TestUtils.assertMemoryLeak(() -> { CatchUpCapturingClient client = new CatchUpCapturingClient(3_100); try (CursorSendEngine engine = newEngine()) { CursorWebSocketSendLoop loop = newLoop(engine, client); try { seedMirror(loop, TestUtils.repeat("x", 3_000), TestUtils.repeat("y", 3_000)); + assertEquals("the mirror is indexed once when it is built", + 1, loop.catchUpEntryIndexBuildCount()); invokeSetWireBaselineWithCatchUp(loop, 0L); assertEquals("the small cap must split the dictionary", 2, client.framesSent); - assertEquals("the split chunks fit the initial native buffer", + assertEquals("catch-up must send the symbol bytes as a second payload slice", + 2, client.multipartFramesSent); + assertEquals("the split chunks need one small prefix buffer", 1, loop.catchUpFrameGrowthCount()); client.cap = 7_000; invokeSetWireBaselineWithCatchUp(loop, 0L); assertEquals("the larger cap must combine the dictionary", 3, client.framesSent); - assertEquals("the combined frame must grow the retained native buffer once", - 2, loop.catchUpFrameGrowthCount()); + assertEquals("combining symbols must not grow the prefix-only buffer", + 1, loop.catchUpFrameGrowthCount()); + assertEquals("reconnect must reuse cached entry ends instead of reparsing", + 1, loop.catchUpEntryIndexBuildCount()); invokeSetWireBaselineWithCatchUp(loop, 0L); assertEquals("the next reconnect sends one combined frame", 4, client.framesSent); - assertEquals("the grown native frame buffer must be reused across reconnects", - 2, loop.catchUpFrameGrowthCount()); + assertEquals("the prefix buffer must be reused across reconnects", + 1, loop.catchUpFrameGrowthCount()); + assertEquals("later reconnects must still reuse the entry index", + 1, loop.catchUpEntryIndexBuildCount()); } finally { // assertMemoryLeak verifies that close releases the retained buffer. loop.close(); @@ -964,6 +972,10 @@ private static void seedMirror(CursorWebSocketSendLoop loop, String... symbols) setIntField(loop, "sentDictBytesCapacity", total); setIntField(loop, "sentDictBytesLen", total); setIntField(loop, "sentDictCount", symbols.length); + Method index = CursorWebSocketSendLoop.class.getDeclaredMethod( + "rebuildSentDictEntryIndex"); + index.setAccessible(true); + index.invoke(loop); } private static void setField(Object target, String name, long value) throws Exception { @@ -1021,6 +1033,7 @@ private static final class CatchUpCapturingClient extends WebSocketClient { private final Runnable onCapRead; private final boolean throwOnSend; private int framesSent; + private int multipartFramesSent; CatchUpCapturingClient(int cap) { this(cap, false); @@ -1052,6 +1065,21 @@ public int getServerQwpVersion() { @Override public void sendBinary(long dataPtr, int length) { + recordSend(); + } + + @Override + public void sendBinary( + long firstPtr, + int firstLength, + long secondPtr, + int secondLength + ) { + multipartFramesSent++; + recordSend(); + } + + private void recordSend() { if (throwOnSend) { throw new RuntimeException("transient wire failure during catch-up"); } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java index ee046d33..0a5c941d 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java @@ -46,6 +46,7 @@ public class PersistedSymbolDictTest { + private static final int APPEND_MAP_CAPACITY = 4 * 1024 * 1024; // On-disk geometry, mirrored from PersistedSymbolDict so the layout-derived // corruption tests below read as arithmetic rather than magic numbers. private static final int CRC_SIZE = 4; @@ -375,9 +376,10 @@ public void testBatchAppendWritesOneChunkWithOneChecksum() throws Exception { } @Test - public void testMappedAppendAmortizesFlushesWithoutPositionedWrites() throws Exception { + public void testMappedAppendUsesMultiMegabyteWindowsWithoutPositionedWrites() throws Exception { assertMemoryLeak(() -> { Path dir = newFolder("qwp-symdict"); + Path file = dir.resolve(PersistedSymbolDict.FILE_NAME); try (PersistedSymbolDict d = PersistedSymbolDict.open(dir.toString())) { Assert.assertNotNull(d); for (int i = 0; i < 10_000; i++) { @@ -386,9 +388,13 @@ public void testMappedAppendAmortizesFlushesWithoutPositionedWrites() throws Exc Assert.assertEquals(10_000, d.size()); Assert.assertEquals("production appends must write directly into the mmap", 0L, d.appendWriteCount()); - Assert.assertEquals("ten thousand flushes must require only three mapped windows", - 3, d.appendMapGrowthCount()); + Assert.assertEquals("ten thousand flushes must share one 4 MiB mapped window", + 1, d.appendMapGrowthCount()); + Assert.assertEquals("the active append window must reserve one bounded segment", + APPEND_MAP_CAPACITY, Files.size(file)); } + Assert.assertTrue("close must truncate the unused mmap reserve", + Files.size(file) < APPEND_MAP_CAPACITY); try (PersistedSymbolDict re = PersistedSymbolDict.open(dir.toString())) { Assert.assertNotNull(re); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentRingTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentRingTest.java index f1a0fcde..33f05ec1 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentRingTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SegmentRingTest.java @@ -467,8 +467,11 @@ public void testNextSealedAfterWalksThousandsOfSegmentsWithoutOverflow() throws assertEquals(0, cursor.baseSeq()); int visited = 1; long prevBase = cursor.baseSeq(); + int maxSearchComparisons = 0; while (true) { MmapSegment next = ring.nextSealedAfter(cursor); + maxSearchComparisons = Math.max(maxSearchComparisons, + ring.getLastNextSealedSearchComparisons()); if (next == null) break; assertTrue("baseSeq must strictly increase: prev=" + prevBase + " next=" + next.baseSeq(), @@ -480,6 +483,31 @@ public void testNextSealedAfterWalksThousandsOfSegmentsWithoutOverflow() throws assertEquals("must visit every sealed segment", sealedCount, visited); // Walking past the last sealed → null (caller falls through to active). assertNull(ring.nextSealedAfter(cursor)); + maxSearchComparisons = Math.max(maxSearchComparisons, + ring.getLastNextSealedSearchComparisons()); + int binarySearchBound = 32 - Integer.numberOfLeadingZeros(sealedCount) + 1; + assertTrue("successor search used " + maxSearchComparisons + + " comparisons for " + sealedCount + + " segments; expected binary search <= " + binarySearchBound, + maxSearchComparisons <= binarySearchBound); + + // Trim one segment per ACK. remove(0) shifts the whole suffix on + // every iteration (O(N^2)); the logical deque head only performs + // occasional bulk compactions whose total moved references stay + // linear in the original sealed count. + for (int i = 0; i < sealedCount; i++) { + ring.acknowledge(i); + ObjList drained = ring.drainTrimmable(); + assertNotNull(drained); + assertEquals(1, drained.size()); + assertEquals(i, drained.get(0).baseSeq()); + drained.get(0).close(); + } + assertNull(ring.firstSealed()); + assertTrue("prefix compaction moved " + ring.getSealedCompactionMoves() + + " references while trimming " + sealedCount + + " segments; expected amortized O(1) trim", + ring.getSealedCompactionMoves() < sealedCount); } } finally { Unsafe.free(buf, 16, MemoryTag.NATIVE_DEFAULT); From 111f341cdd6fea7d68f4c645d4e7b1865f871b2b Mon Sep 17 00:00:00 2001 From: glasstiger Date: Fri, 17 Jul 2026 12:10:14 +0100 Subject: [PATCH 83/96] Add QWP edge-case coverage --- .../main/java/io/questdb/client/Sender.java | 18 +++++-- .../qwp/client/DeltaDictRecoveryTest.java | 47 ++++++++++++++++ .../client/GlobalSymbolDictionaryTest.java | 17 ++++++ .../LineSenderBuilderWebSocketTest.java | 43 +++++++++++++++ .../qwp/client/NativeBufferWriterTest.java | 54 +++++++++++++++++++ 5 files changed, 176 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/io/questdb/client/Sender.java b/core/src/main/java/io/questdb/client/Sender.java index 837b1d7b..bb04411b 100644 --- a/core/src/main/java/io/questdb/client/Sender.java +++ b/core/src/main/java/io/questdb/client/Sender.java @@ -52,6 +52,7 @@ import io.questdb.client.std.Decimal256; import io.questdb.client.std.Decimal64; import io.questdb.client.std.Files; +import io.questdb.client.std.FilesFacade; import io.questdb.client.std.IntList; import io.questdb.client.std.Numbers; import io.questdb.client.std.NumericException; @@ -1012,6 +1013,8 @@ final class LineSenderBuilder { private static final int PROTOCOL_WEBSOCKET = 2; @TestOnly private static volatile Runnable quarantineAfterCloseHook; + @TestOnly + private static volatile FilesFacade quarantineFilesFacade = FilesFacade.INSTANCE; // Suffix for a slot set aside by quarantineTornSlot. Deliberately NOT the // sender's own slot name, so a restarted sender does not re-adopt it as its own; // quarantineTornSlot then marks it .failed, so the orphan drainer skips it too and @@ -3061,15 +3064,16 @@ private static CursorSendEngine quarantineTornSlot( hook.run(); } + FilesFacade ff = quarantineFilesFacade; String quarantinePath = null; for (int i = 0; i < MAX_QUARANTINE_SLOT_ATTEMPTS; i++) { String candidate = sfDir + "/" + senderId + QUARANTINE_SLOT_SUFFIX + i; - if (!Files.exists(candidate)) { + if (!ff.exists(candidate)) { quarantinePath = candidate; break; } } - if (quarantinePath == null || Files.rename(slotPath, quarantinePath) != 0) { + if (quarantinePath == null || ff.rename(slotPath, quarantinePath) != 0) { throw new LineSenderException( detail + "; the affected data must be resent. The slot could not be set aside " + "automatically (" + (quarantinePath == null @@ -3092,6 +3096,11 @@ public static void setQuarantineAfterCloseHookForTest(Runnable hook) { quarantineAfterCloseHook = hook; } + @TestOnly + public static void setQuarantineFilesFacadeForTest(FilesFacade ff) { + quarantineFilesFacade = ff == null ? FilesFacade.INSTANCE : ff; + } + private static int resolveIPv4(String host) { try { byte[] addr = InetAddress.getByName(host).getAddress(); @@ -4047,7 +4056,10 @@ public java.util.Map wsConfigSnapshotForTest() { m.put("max_background_drainers", maxBackgroundDrainers); m.put("max_frame_rejections", maxFrameRejections); m.put("poison_min_escalation_window_millis", poisonMinEscalationWindowMillis); - m.put("catchup_cap_gap_min_escalation_window_millis", catchUpCapGapMinEscalationWindowMillis); + m.put("catchup_cap_gap_min_escalation_window_millis", + catchUpCapGapMinEscalationWindowMillis == PARAMETER_NOT_SET_EXPLICITLY + ? CursorWebSocketSendLoop.DEFAULT_CATCHUP_CAP_GAP_MIN_ESCALATION_WINDOW_MILLIS + : catchUpCapGapMinEscalationWindowMillis); m.put("error_inbox_capacity", errorInboxCapacity); m.put("connection_listener_inbox_capacity", connectionListenerInboxCapacity); m.put("token", httpToken); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java index c0b517d1..8979036c 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java @@ -467,6 +467,53 @@ public void testQuarantineFailsLoudlyWhenAllSlotNamesSaturated() throws Exceptio }); } + @Test + public void testQuarantineRenameFailurePreservesOriginalSlot() throws Exception { + assertMemoryLeak(() -> { + writeAndTearUnreplayableSlot(); + java.nio.file.Path slot = Paths.get(sfDir, "default"); + int segmentCount = countSegmentFiles(slot); + Sender.LineSenderBuilder.setQuarantineFilesFacadeForTest(new DelegatingFilesFacade() { + @Override + public int rename(String oldPath, String newPath) { + return -1; + } + }); + try { + try (TestWebSocketServer good = new TestWebSocketServer(new DictReconstructingHandler())) { + int port = good.getPort(); + good.start(); + Assert.assertTrue(good.awaitStart(5, TimeUnit.SECONDS)); + String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";"; + try { + Sender.fromConfig(cfg).close(); + Assert.fail("build() must throw when the unreplayable slot rename fails"); + } catch (LineSenderException expected) { + Assert.assertEquals( + "recovered store-and-forward symbol dictionary is incomplete and cannot be rebuilt " + + "from the surviving frames (likely a host crash tore its unsynced tail): " + + "the frames reference symbol ids below their own delta start, which were " + + "introduced by frames since acked and trimmed away, so nothing still holds " + + "them; the recovered dictionary holds only 0 id(s) -- resend the affected " + + "data; the affected data must be resent. The slot could not be set aside " + + "automatically (rename to " + sfDir + "/default.unreplayable-0 failed), so " + + "this sender cannot start until " + sfDir + "/default is moved or removed " + + "by hand", + expected.getMessage()); + } + } + } finally { + Sender.LineSenderBuilder.setQuarantineFilesFacadeForTest(null); + } + Assert.assertTrue("rename failure must preserve the original slot directory", + java.nio.file.Files.isDirectory(slot)); + Assert.assertEquals("rename failure must preserve every segment", + segmentCount, countSegmentFiles(slot)); + Assert.assertFalse("failed rename must not leave a quarantine directory", + java.nio.file.Files.exists(Paths.get(sfDir, "default.unreplayable-0"))); + }); + } + // Writes 12 delta frames (each introducing a new symbol) into the default slot // across several small segments, then makes the slot GENUINELY unreplayable: trims // the segment holding the earliest ids (munmap + unlink, exactly what SegmentManager diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/GlobalSymbolDictionaryTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/GlobalSymbolDictionaryTest.java index 2f38ab02..1295dc9b 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/GlobalSymbolDictionaryTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/GlobalSymbolDictionaryTest.java @@ -61,6 +61,23 @@ public void testAddRecoveredSymbol_appendsWithoutDeduplicating() { assertEquals(3, recovered.getOrAddSymbol("brand-new")); } + @Test + public void testAddRecoveredSymbol_rejectsNullWithoutMutation() { + GlobalSymbolDictionary dict = new GlobalSymbolDictionary(); + assertEquals(0, dict.addRecoveredSymbol("AAPL")); + + try { + dict.addRecoveredSymbol(null); + fail("expected IllegalArgumentException"); + } catch (IllegalArgumentException expected) { + assertEquals("symbol cannot be null", expected.getMessage()); + } + + assertEquals(1, dict.size()); + assertEquals("AAPL", dict.getSymbol(0)); + assertEquals(0, dict.getId("AAPL")); + } + @Test public void testAddSymbol_assignsSequentialIds() { GlobalSymbolDictionary dict = new GlobalSymbolDictionary(); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderWebSocketTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderWebSocketTest.java index caec264b..b04ff686 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderWebSocketTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/LineSenderBuilderWebSocketTest.java @@ -222,6 +222,49 @@ public void testBuilderWithWebSocketTransportCreatesCorrectSenderType() throws E }); } + @Test + public void testCatchUpCapGapMinEscalationWindowBuilderNegativeRejected() { + assertThrows("catchup_cap_gap_min_escalation_window_millis must be >= 0: -1", + () -> Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .catchUpCapGapMinEscalationWindowMillis(-1)); + } + + @Test + public void testCatchUpCapGapMinEscalationWindowBuilderNotSupportedForTcp() { + assertThrows("catchup_cap_gap_min_escalation_window_millis is only supported for WebSocket transport", + () -> Sender.builder(Sender.Transport.TCP) + .address(LOCALHOST) + .catchUpCapGapMinEscalationWindowMillis(300_000)); + } + + @Test + public void testCatchUpCapGapMinEscalationWindowConfigStringNegativeRejected() { + assertThrows("catchup_cap_gap_min_escalation_window_millis must be >= 0: -1", + () -> Sender.builder("ws::addr=localhost:9000;" + + "catchup_cap_gap_min_escalation_window_millis=-1;")); + } + + @Test + public void testCatchUpCapGapMinEscalationWindowConfigStringNotSupportedForTcp() { + assertThrows("catchup_cap_gap_min_escalation_window_millis is only supported for WebSocket transport", + () -> Sender.builder("tcp::addr=localhost:9009;" + + "catchup_cap_gap_min_escalation_window_millis=300000;")); + } + + @Test + public void testCatchUpCapGapMinEscalationWindowDefaultsToFiveMinutes() { + Assert.assertEquals(300_000L, + Sender.builder(Sender.Transport.WEBSOCKET) + .address(LOCALHOST) + .wsConfigSnapshotForTest() + .get("catchup_cap_gap_min_escalation_window_millis")); + Assert.assertEquals(300_000L, + Sender.builder("ws::addr=localhost:9000;") + .wsConfigSnapshotForTest() + .get("catchup_cap_gap_min_escalation_window_millis")); + } + @Test public void testConnectionRefused() throws Exception { assertMemoryLeak(() -> { diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/NativeBufferWriterTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/NativeBufferWriterTest.java index 0fb85406..1eadc103 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/NativeBufferWriterTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/NativeBufferWriterTest.java @@ -273,6 +273,43 @@ public void testQwpBufferWriterUtf8LengthInvalidSurrogatePair() throws Exception }); } + @Test + public void testRawVarintBoundariesAndReturnedAddress() throws Exception { + assertMemoryLeak(() -> { + try (NativeBufferWriter writer = new NativeBufferWriter(16)) { + assertRawVarint(writer, 0, 0x00); + assertRawVarint(writer, 1, 0x01); + assertRawVarint(writer, 127, 0x7F); + assertRawVarint(writer, 128, 0x80, 0x01); + assertRawVarint(writer, 16_383, 0xFF, 0x7F); + assertRawVarint(writer, 16_384, 0x80, 0x80, 0x01); + assertRawVarint(writer, Long.MAX_VALUE, + 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x7F); + } + }); + } + + @Test + public void testRawVarintRejectsNegativeWithoutWriting() throws Exception { + assertMemoryLeak(() -> { + try (NativeBufferWriter writer = new NativeBufferWriter(16)) { + long addr = writer.getBufferPtr(); + Unsafe.getUnsafe().putByte(addr, (byte) 0x5A); + boolean rejected = false; + try { + NativeBufferWriter.writeVarint(addr, -1); + } catch (AssertionError expected) { + rejected = true; + assertTrue(expected.getMessage().contains( + "unsigned LEB128 varint requires a non-negative value: -1")); + } + assertTrue("negative raw varint must be rejected", rejected); + assertEquals("rejection must happen before the first write", + (byte) 0x5A, Unsafe.getUnsafe().getByte(addr)); + } + }); + } + @Test public void testReset() throws Exception { assertMemoryLeak(() -> { @@ -618,4 +655,21 @@ public void testWriteVarintSmall() throws Exception { } }); } + + private static void assertRawVarint(NativeBufferWriter writer, long value, int... expectedBytes) { + long addr = writer.getBufferPtr(); + for (int i = 0; i <= expectedBytes.length; i++) { + Unsafe.getUnsafe().putByte(addr + i, (byte) 0x5A); + } + long end = NativeBufferWriter.writeVarint(addr, value); + assertEquals("returned address for value=" + value, addr + expectedBytes.length, end); + assertEquals("varintSize for value=" + value, + expectedBytes.length, NativeBufferWriter.varintSize(value)); + for (int i = 0; i < expectedBytes.length; i++) { + assertEquals("byte " + i + " for value=" + value, + (byte) expectedBytes[i], Unsafe.getUnsafe().getByte(addr + i)); + } + assertEquals("writer overran value=" + value, + (byte) 0x5A, Unsafe.getUnsafe().getByte(end)); + } } From 8a1aaf0f8d4c0e83c76e673f51df343d6772f42f Mon Sep 17 00:00:00 2001 From: glasstiger Date: Fri, 17 Jul 2026 12:28:01 +0100 Subject: [PATCH 84/96] Add QWP failure-path coverage --- .../client/sf/cursor/PersistedSymbolDict.java | 21 +- .../qwp/client/sf/cursor/SlotLock.java | 52 +++-- .../io/questdb/client/std/FilesFacade.java | 23 ++ .../qwp/client/QwpWebSocketEncoderTest.java | 79 +++++++ .../sf/cursor/PersistedSymbolDictTest.java | 221 ++++++++++++++++++ .../sf/cursor/RecoveredFrameAnalysisTest.java | 97 ++++++++ .../qwp/client/sf/cursor/SlotLockTest.java | 68 ++++++ .../test/tools/DelegatingFilesFacade.java | 6 +- 8 files changed, 533 insertions(+), 34 deletions(-) create mode 100644 core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/RecoveredFrameAnalysisTest.java diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java index 96e26df9..fbc22784 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java @@ -172,8 +172,9 @@ public final class PersistedSymbolDict implements QuietCloseable { // tests inject a fault facade to exercise recovery I/O failures (a truncate // that cannot drop a torn tail, a short write) without a real broken disk. private final FilesFacade ff; - // Production writes directly into segmented append mappings. Custom facades retain the - // positioned-write path so fault tests can inject short writes through ff.write. + // Production writes directly into segmented append mappings. Wrapping facades retain the + // positioned-write path by default so fault tests can inject short writes through ff.write; + // mmap-specific fault facades opt in through FilesFacade.isMmapAllowed(). private final boolean mappedAppend; // True only when recovery parsed the file through a temporary read-only // mmap instead of allocating a second native buffer as large as the file. @@ -210,7 +211,7 @@ private PersistedSymbolDict( ) { this.ff = ff; this.fd = fd; - this.mappedAppend = ff == FilesFacade.INSTANCE; + this.mappedAppend = ff.isMmapAllowed(); this.mappedRecoveryInput = mappedRecoveryInput; this.appendOffset = appendOffset; this.size = size; @@ -492,7 +493,7 @@ public synchronized void close() { loadedEntriesLen = 0; } if (appendMapAddr != 0L) { - Files.munmap(appendMapAddr, appendMapCapacity, MemoryTag.MMAP_DEFAULT); + ff.munmap(appendMapAddr, appendMapCapacity, MemoryTag.MMAP_DEFAULT); appendMapAddr = 0L; appendMapCapacity = 0L; appendMapOffset = 0L; @@ -639,13 +640,13 @@ private static PersistedSymbolDict openExisting(FilesFacade ff, String filePath, return null; } int len = (int) fileLen; // open() bounds fileLen to [HEADER_SIZE, Integer.MAX_VALUE) - boolean mappedInput = ff == FilesFacade.INSTANCE; + boolean mappedInput = ff.isMmapAllowed(); long inputAddr = 0L; long entriesAddr = 0L; int entriesLen = 0; try { if (mappedInput) { - inputAddr = Files.mmap(fd, fileLen, 0L, Files.MAP_RO, MemoryTag.MMAP_DEFAULT); + inputAddr = ff.mmap(fd, fileLen, 0L, Files.MAP_RO, MemoryTag.MMAP_DEFAULT); if (inputAddr == Files.FAILED_MMAP_ADDRESS) { inputAddr = 0L; throw new IllegalStateException("could not mmap symbol dictionary for recovery"); @@ -675,7 +676,7 @@ private static PersistedSymbolDict openExisting(FilesFacade ff, String filePath, copyRecoveredEntries(inputAddr, scan.validLen, entriesAddr, entriesLen); } if (mappedInput) { - Files.munmap(inputAddr, fileLen, MemoryTag.MMAP_DEFAULT); + ff.munmap(inputAddr, fileLen, MemoryTag.MMAP_DEFAULT); } else { Unsafe.free(inputAddr, len, MemoryTag.NATIVE_DEFAULT); } @@ -695,7 +696,7 @@ private static PersistedSymbolDict openExisting(FilesFacade ff, String filePath, } catch (Throwable t) { if (inputAddr != 0L) { if (mappedInput) { - Files.munmap(inputAddr, fileLen, MemoryTag.MMAP_DEFAULT); + ff.munmap(inputAddr, fileLen, MemoryTag.MMAP_DEFAULT); } else { Unsafe.free(inputAddr, len, MemoryTag.NATIVE_DEFAULT); } @@ -923,9 +924,9 @@ private void ensureAppendMap(long required) { appendMapAddr = 0L; appendMapCapacity = 0L; appendMapOffset = 0L; - Files.munmap(oldAddr, oldCapacity, MemoryTag.MMAP_DEFAULT); + ff.munmap(oldAddr, oldCapacity, MemoryTag.MMAP_DEFAULT); } - long newAddr = Files.mmap( + long newAddr = ff.mmap( fd, newCapacity, newOffset, Files.MAP_RW, MemoryTag.MMAP_DEFAULT); if (newAddr == Files.FAILED_MMAP_ADDRESS) { throw new IllegalStateException("could not mmap append region for " diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SlotLock.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SlotLock.java index 0ec9f9f5..7902538c 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SlotLock.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SlotLock.java @@ -26,9 +26,11 @@ import io.questdb.client.std.Compat; import io.questdb.client.std.Files; +import io.questdb.client.std.FilesFacade; import io.questdb.client.std.MemoryTag; import io.questdb.client.std.QuietCloseable; import io.questdb.client.std.Unsafe; +import org.jetbrains.annotations.TestOnly; import java.nio.charset.StandardCharsets; import java.nio.file.Path; @@ -65,10 +67,12 @@ public final class SlotLock implements QuietCloseable { private static final String LOCK_FILE_NAME = ".lock"; private static final String LOCK_PID_FILE_NAME = ".lock.pid"; private static final String LOGICAL_LOCK_DIR_NAME = ".slot-locks"; + private final FilesFacade ff; private final String slotDir; private int fd; - private SlotLock(String slotDir, int fd) { + private SlotLock(FilesFacade ff, String slotDir, int fd) { + this.ff = ff; this.slotDir = slotDir; this.fd = fd; } @@ -83,10 +87,10 @@ private SlotLock(String slotDir, int fd) { */ public static SlotLock acquire(String slotDir) { validateSlotDir(slotDir); - ensureDirectory(slotDir, "slot dir"); + ensureDirectory(FilesFacade.INSTANCE, slotDir, "slot dir"); String lockPath = slotDir + "/" + LOCK_FILE_NAME; String pidPath = slotDir + "/" + LOCK_PID_FILE_NAME; - return acquireAt(slotDir, lockPath, pidPath); + return acquireAt(FilesFacade.INSTANCE, slotDir, lockPath, pidPath); } /** @@ -102,6 +106,12 @@ public static SlotLock acquire(String slotDir) { * the fresh directory through the old pathname. */ public static SlotLock acquireLogical(String slotDir) { + return acquireLogical(FilesFacade.INSTANCE, slotDir); + } + + /** Facade-aware variant used to exercise logical-lock I/O failures. */ + @TestOnly + public static SlotLock acquireLogical(FilesFacade ff, String slotDir) { validateSlotDir(slotDir); Path slotPath = Paths.get(slotDir); Path parentPath = slotPath.getParent(); @@ -113,44 +123,44 @@ public static SlotLock acquireLogical(String slotDir) { String parentDir = parentPath.toString(); String slotName = slotNamePath.toString(); String logicalLockDir = parentDir + "/" + LOGICAL_LOCK_DIR_NAME; - ensureDirectory(logicalLockDir, "logical slot lock dir"); + ensureDirectory(ff, logicalLockDir, "logical slot lock dir"); String lockPath = logicalLockDir + "/" + slotName + ".lock"; String pidPath = logicalLockDir + "/" + slotName + ".lock.pid"; - return acquireAt(slotDir, lockPath, pidPath); + return acquireAt(ff, slotDir, lockPath, pidPath); } - private static SlotLock acquireAt(String slotDir, String lockPath, String pidPath) { - int fd = Files.openRW(lockPath); + private static SlotLock acquireAt(FilesFacade ff, String slotDir, String lockPath, String pidPath) { + int fd = ff.openRW(lockPath); if (fd < 0) { throw new IllegalStateException( "could not open slot lock file: " + lockPath); } boolean ok = false; try { - int rc = Files.lock(fd); + int rc = ff.lock(fd); if (rc != 0) { String holder = readHolder(pidPath); throw new IllegalStateException( "sf slot already in use by another process [slot=" + slotDir + ", holder=" + holder + "]"); } - writePid(pidPath); + writePid(ff, pidPath); ok = true; - return new SlotLock(slotDir, fd); + return new SlotLock(ff, slotDir, fd); } finally { if (!ok) { - Files.close(fd); + ff.close(fd); } } } - private static void ensureDirectory(String path, String description) { - if (!Files.exists(path)) { - int rc = Files.mkdir(path, Files.DIR_MODE_DEFAULT); + private static void ensureDirectory(FilesFacade ff, String path, String description) { + if (!ff.exists(path)) { + int rc = ff.mkdir(path, Files.DIR_MODE_DEFAULT); // Multiple senders may create the shared parent lock directory // concurrently. Treat EEXIST as success, just as the builder does // for the SF root itself. - if (rc != 0 && !Files.exists(path)) { + if (rc != 0 && !ff.exists(path)) { throw new IllegalStateException( "could not create " + description + ": " + path + " rc=" + rc); } @@ -174,7 +184,7 @@ public void close() { // file or the .lock.pid sidecar — a stale PID is harmless (next // acquirer overwrites .lock.pid on success). if (fd >= 0) { - Files.close(fd); + ff.close(fd); fd = -1; } } @@ -204,7 +214,7 @@ private static String readHolder(String pidPath) { } } - private static void writePid(String pidPath) { + private static void writePid(FilesFacade ff, String pidPath) { long pid; try { pid = Compat.currentPid(); @@ -212,25 +222,25 @@ private static void writePid(String pidPath) { // Diagnostic-only — never block lock acquisition on it. pid = -1L; } - int wfd = Files.openRW(pidPath); + int wfd = ff.openRW(pidPath); if (wfd < 0) { // Diagnostic-only — never block lock acquisition on it. return; } try { - Files.truncate(wfd, 0L); + ff.truncate(wfd, 0L); byte[] payload = (pid + "\n").getBytes(StandardCharsets.UTF_8); long addr = Unsafe.malloc(payload.length, MemoryTag.NATIVE_DEFAULT); try { for (int i = 0; i < payload.length; i++) { Unsafe.getUnsafe().putByte(addr + i, payload[i]); } - Files.write(wfd, addr, payload.length, 0L); + ff.write(wfd, addr, payload.length, 0L); } finally { Unsafe.free(addr, payload.length, MemoryTag.NATIVE_DEFAULT); } } finally { - Files.close(wfd); + ff.close(wfd); } } } diff --git a/core/src/main/java/io/questdb/client/std/FilesFacade.java b/core/src/main/java/io/questdb/client/std/FilesFacade.java index 1b408cf4..c5e6afc7 100644 --- a/core/src/main/java/io/questdb/client/std/FilesFacade.java +++ b/core/src/main/java/io/questdb/client/std/FilesFacade.java @@ -85,6 +85,16 @@ public interface FilesFacade { int fsync(int fd); + /** + * Whether callers should use this facade's mmap path. The production facade + * returns {@code true}; wrapping fault facades retain positioned I/O unless + * they explicitly opt in, preserving their ability to inject short reads and + * writes. + */ + default boolean isMmapAllowed() { + return this == INSTANCE; + } + long length(int fd); /** @@ -107,6 +117,19 @@ public interface FilesFacade { int mkdir(String path, int mode); + /** + * Maps a file region. Kept on the facade so mmap failures can be injected + * without relying on platform-specific filesystem behavior. + */ + default long mmap(int fd, long len, long offset, int flags, int memoryTag) { + return Files.mmap(fd, len, offset, flags, memoryTag); + } + + /** Releases a region returned by {@link #mmap(int, long, long, int, int)}. */ + default void munmap(long address, long len, int memoryTag) { + Files.munmap(address, len, memoryTag); + } + int openCleanRW(String path); /** diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketEncoderTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketEncoderTest.java index 5acc0cf9..44f711a3 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketEncoderTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/QwpWebSocketEncoderTest.java @@ -1285,6 +1285,69 @@ public void testReset() throws Exception { }); } + @Test + public void testSplitMessageValidatesArgumentsAndSizeOverflow() throws Exception { + assertMemoryLeak(() -> { + try (QwpWebSocketEncoder encoder = new QwpWebSocketEncoder(); + MicrobatchBuffer target = new MicrobatchBuffer(64); + QwpTableBuffer table = new QwpTableBuffer("alpha")) { + GlobalSymbolDictionary dict = new GlobalSymbolDictionary(); + int aapl = dict.getOrAddSymbol("AAPL"); + dict.getOrAddSymbol("GOOG"); + table.getOrCreateColumn("sym", TYPE_SYMBOL, false) + .addSymbolWithGlobalId("AAPL", aapl); + table.nextRow(); + + encoder.beginMessage(1, dict, -1, 1); + int tableBodyOffset = encoder.getBuffer().getPosition(); + encoder.addTable(table); + int tableBodyLength = encoder.getBuffer().getPosition() - tableBodyOffset; + encoder.finishMessage(); + + target.setBufferPos(1); + assertFailure(IllegalStateException.class, + "split message target is not empty", + () -> encoder.copySplitMessage( + target, tableBodyOffset, tableBodyLength, false, -1, 1)); + Assert.assertEquals("invalid target must not be modified", 1, target.getBufferPos()); + target.reset(); + + assertFailure(IllegalArgumentException.class, + "table body slice is outside the staged message", + () -> encoder.copySplitMessage( + target, tableBodyOffset - 1, tableBodyLength, false, -1, 1)); + assertFailure(IllegalArgumentException.class, + "table body slice is outside the staged message", + () -> encoder.copySplitMessage( + target, tableBodyOffset, -1, false, -1, 1)); + assertFailure(IllegalArgumentException.class, + "table body slice is outside the staged message", + () -> encoder.copySplitMessage( + target, tableBodyOffset, tableBodyLength + 1, false, -1, 1)); + Assert.assertEquals("invalid slices must not write the target", 0, target.getBufferPos()); + + assertFailure(IllegalStateException.class, + "split delta does not match the staged message" + + " [stagedStart=0, stagedCount=2, splitStart=1, splitCount=1]", + () -> encoder.copySplitMessage( + target, tableBodyOffset, tableBodyLength, false, 0, 1)); + Assert.assertEquals("delta mismatch must not write the target", 0, target.getBufferPos()); + + assertFailure(IllegalArgumentException.class, + "tableBodyLength must be non-negative", + () -> encoder.getSplitMessageSize(-1, -1, 1)); + long overflowSize = (long) HEADER_SIZE + + 1 // delta start 0 + + 1 // delta count 2 + + encoder.getDeltaEntriesLen() + + Integer.MAX_VALUE; + assertFailure(OutOfMemoryError.class, + "split QWP message size overflow: " + overflowSize, + () -> encoder.getSplitMessageSize(Integer.MAX_VALUE, -1, 1)); + } + }); + } + @Test public void testSplitMessageCopiesStagedTableBodies() throws Exception { assertMemoryLeak(() -> { @@ -1388,6 +1451,22 @@ public void testVersionByteInHeader() throws Exception { }); } + private static void assertFailure( + Class expectedType, + String expectedMessage, + Runnable action + ) { + Throwable thrown = null; + try { + action.run(); + } catch (Throwable t) { + thrown = t; + } + Assert.assertNotNull("expected " + expectedType.getSimpleName(), thrown); + Assert.assertEquals(expectedType, thrown.getClass()); + Assert.assertEquals(expectedMessage, thrown.getMessage()); + } + private static final class Cursor { private long address; diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java index 0a5c941d..7b4f6d60 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java @@ -55,6 +55,54 @@ public class PersistedSymbolDictTest { @Rule public final TemporaryFolder temporaryFolder = TemporaryFolder.builder().assureDeletion().build(); + @Test + public void testAppendAllocateFailureClosesDescriptorWithoutNativeLeak() throws Exception { + assertMemoryLeak(() -> { + Path dir = newFolder("qwp-symdict-allocate-failure"); + IoFailureFacade ff = new IoFailureFacade(IoFailure.ALLOCATE); + try (PersistedSymbolDict dict = PersistedSymbolDict.openClean(ff, dir.toString())) { + Assert.assertNotNull(dict); + try { + dict.appendSymbol("A"); + Assert.fail("expected append allocation failure"); + } catch (IllegalStateException expected) { + Assert.assertEquals( + "could not grow mmap append region for .symbol-dict" + + " [required=16, fileSize=" + APPEND_MAP_CAPACITY + ']', + expected.getMessage()); + } + Assert.assertEquals("failed append must not advance the dictionary", 0, dict.size()); + } + ff.assertAllOpenedDescriptorsClosed(); + Assert.assertEquals(1, ff.allocateCalls); + Assert.assertEquals(0, ff.mmapCalls); + }); + } + + @Test + public void testAppendMmapFailureClosesDescriptorWithoutNativeLeak() throws Exception { + assertMemoryLeak(() -> { + Path dir = newFolder("qwp-symdict-mmap-append-failure"); + IoFailureFacade ff = new IoFailureFacade(IoFailure.MMAP); + try (PersistedSymbolDict dict = PersistedSymbolDict.openClean(ff, dir.toString())) { + Assert.assertNotNull(dict); + try { + dict.appendSymbol("A"); + Assert.fail("expected append mmap failure"); + } catch (IllegalStateException expected) { + Assert.assertEquals( + "could not mmap append region for .symbol-dict" + + " [offset=0, capacity=" + APPEND_MAP_CAPACITY + ']', + expected.getMessage()); + } + Assert.assertEquals("failed append must not advance the dictionary", 0, dict.size()); + } + ff.assertAllOpenedDescriptorsClosed(); + Assert.assertEquals(1, ff.allocateCalls); + Assert.assertEquals(1, ff.mmapCalls); + }); + } + @Test public void testAppendPersistsAcrossReopen() throws Exception { assertMemoryLeak(() -> { @@ -620,6 +668,52 @@ public void testOpenCleanDiscardsSurvivingDictionary() throws Exception { }); } + @Test + public void testOpenFailuresDoNotCreateOrDestroyDictionaryFiles() throws Exception { + assertMemoryLeak(() -> { + Path freshDir = newFolder("qwp-symdict-fresh-open-failure"); + IoFailureFacade freshFf = new IoFailureFacade(IoFailure.OPEN_CLEAN); + Assert.assertNull(PersistedSymbolDict.openClean(freshFf, freshDir.toString())); + Assert.assertEquals(1, freshFf.openCleanAttempts); + freshFf.assertAllOpenedDescriptorsClosed(); + Assert.assertFalse(Files.exists(freshDir.resolve(PersistedSymbolDict.FILE_NAME))); + + Path existingDir = newFolder("qwp-symdict-existing-open-failure"); + try (PersistedSymbolDict seed = PersistedSymbolDict.open(existingDir.toString())) { + Assert.assertNotNull(seed); + seed.appendSymbol("AAPL"); + } + Path file = existingDir.resolve(PersistedSymbolDict.FILE_NAME); + byte[] before = Files.readAllBytes(file); + IoFailureFacade existingFf = new IoFailureFacade(IoFailure.OPEN_EXISTING); + Assert.assertNull(PersistedSymbolDict.open(existingFf, existingDir.toString())); + Assert.assertEquals(1, existingFf.openRwAttempts); + existingFf.assertAllOpenedDescriptorsClosed(); + Assert.assertArrayEquals("failed recovery open must preserve the load-bearing file", + before, Files.readAllBytes(file)); + }); + } + + @Test + public void testRecoveryMmapFailureClosesDescriptorAndPreservesFile() throws Exception { + assertMemoryLeak(() -> { + Path dir = newFolder("qwp-symdict-recovery-mmap-failure"); + try (PersistedSymbolDict seed = PersistedSymbolDict.open(dir.toString())) { + Assert.assertNotNull(seed); + seed.appendSymbol("AAPL"); + } + Path file = dir.resolve(PersistedSymbolDict.FILE_NAME); + byte[] before = Files.readAllBytes(file); + + IoFailureFacade ff = new IoFailureFacade(IoFailure.MMAP); + Assert.assertNull(PersistedSymbolDict.open(ff, dir.toString())); + ff.assertAllOpenedDescriptorsClosed(); + Assert.assertEquals(1, ff.mmapCalls); + Assert.assertArrayEquals("failed recovery mmap must preserve the load-bearing file", + before, Files.readAllBytes(file)); + }); + } + @Test public void testRemoveOrphanDeletesFile() throws Exception { assertMemoryLeak(() -> { @@ -634,6 +728,37 @@ public void testRemoveOrphanDeletesFile() throws Exception { }); } + @Test + public void testShortHeaderWriteClosesDescriptorAndRemovesStub() throws Exception { + assertMemoryLeak(() -> { + Path dir = newFolder("qwp-symdict-short-header"); + IoFailureFacade ff = new IoFailureFacade(IoFailure.HEADER_WRITE); + Assert.assertNull(PersistedSymbolDict.openClean(ff, dir.toString())); + ff.assertAllOpenedDescriptorsClosed(); + Assert.assertFalse("headerless stub must be removed", + Files.exists(dir.resolve(PersistedSymbolDict.FILE_NAME))); + }); + } + + @Test + public void testShortRecoveryReadClosesDescriptorAndPreservesFile() throws Exception { + assertMemoryLeak(() -> { + Path dir = newFolder("qwp-symdict-short-read"); + try (PersistedSymbolDict seed = PersistedSymbolDict.open(dir.toString())) { + Assert.assertNotNull(seed); + seed.appendSymbol("AAPL"); + } + Path file = dir.resolve(PersistedSymbolDict.FILE_NAME); + byte[] before = Files.readAllBytes(file); + + IoFailureFacade ff = new IoFailureFacade(IoFailure.READ); + Assert.assertNull(PersistedSymbolDict.open(ff, dir.toString())); + ff.assertAllOpenedDescriptorsClosed(); + Assert.assertArrayEquals("short recovery read must preserve the load-bearing file", + before, Files.readAllBytes(file)); + }); + } + @Test public void testTooLargeToReopenDegradesWithoutOpeningTheFile() throws Exception { // A dictionary at or past Integer.MAX_VALUE cannot be read into one int-sized @@ -811,6 +936,102 @@ private Path newFolder(String name) throws IOException { return temporaryFolder.newFolder(name).toPath(); } + private enum IoFailure { + ALLOCATE, + HEADER_WRITE, + MMAP, + OPEN_CLEAN, + OPEN_EXISTING, + READ + } + + private static final class IoFailureFacade extends DelegatingFilesFacade { + private final IoFailure failure; + private int allocateCalls; + private int closeCalls; + private int mmapCalls; + private int openCleanAttempts; + private int openRwAttempts; + private int openedDescriptors; + + private IoFailureFacade(IoFailure failure) { + this.failure = failure; + } + + @Override + public boolean allocate(int fd, long size) { + allocateCalls++; + return failure != IoFailure.ALLOCATE && super.allocate(fd, size); + } + + @Override + public int close(int fd) { + closeCalls++; + return super.close(fd); + } + + @Override + public boolean isMmapAllowed() { + return failure == IoFailure.ALLOCATE || failure == IoFailure.MMAP; + } + + @Override + public long mmap(int fd, long len, long offset, int flags, int memoryTag) { + mmapCalls++; + if (failure == IoFailure.MMAP) { + return io.questdb.client.std.Files.FAILED_MMAP_ADDRESS; + } + return io.questdb.client.std.Files.mmap(fd, len, offset, flags, memoryTag); + } + + @Override + public int openCleanRW(String path) { + openCleanAttempts++; + if (failure == IoFailure.OPEN_CLEAN) { + return -1; + } + int fd = super.openCleanRW(path); + if (fd >= 0) { + openedDescriptors++; + } + return fd; + } + + @Override + public int openRW(String path) { + openRwAttempts++; + if (failure == IoFailure.OPEN_EXISTING) { + return -1; + } + int fd = super.openRW(path); + if (fd >= 0) { + openedDescriptors++; + } + return fd; + } + + @Override + public long read(int fd, long addr, long len, long offset) { + if (failure == IoFailure.READ && len > 0) { + return super.read(fd, addr, len - 1, offset); + } + return super.read(fd, addr, len, offset); + } + + @Override + public long write(int fd, long addr, long len, long offset) { + if (failure == IoFailure.HEADER_WRITE && offset == 0 && len == HEADER_SIZE) { + return super.write(fd, addr, len - 1, offset); + } + return super.write(fd, addr, len, offset); + } + + private void assertAllOpenedDescriptorsClosed() { + Assert.assertEquals("every successfully opened fd must be closed", + openedDescriptors, closeCalls); + } + } + /** * Fails every {@link #truncate(int, long)} -- reproducing a host where the * torn/stale-tail truncate cannot succeed (read-only remount, Windows share diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/RecoveredFrameAnalysisTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/RecoveredFrameAnalysisTest.java new file mode 100644 index 00000000..171df57d --- /dev/null +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/RecoveredFrameAnalysisTest.java @@ -0,0 +1,97 @@ +/*+***************************************************************************** + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2026 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +package io.questdb.client.test.cutlass.qwp.client.sf.cursor; + +import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine; +import io.questdb.client.cutlass.qwp.protocol.QwpConstants; +import io.questdb.client.std.MemoryTag; +import io.questdb.client.std.ObjList; +import io.questdb.client.std.Unsafe; +import org.junit.Assert; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.nio.file.Path; + +import static io.questdb.client.test.tools.TestUtils.assertMemoryLeak; + +public class RecoveredFrameAnalysisTest { + + @Rule + public final TemporaryFolder temporaryFolder = TemporaryFolder.builder().assureDeletion().build(); + + @Test + public void testTruncatedSymbolBytesMarkRecoveredDeltaAsGap() throws Exception { + assertMalformedDeltaMarksGap(new byte[]{0, 1, 3, 'x'}); + } + + @Test + public void testUnterminatedDeltaCountMarksRecoveredDeltaAsGap() throws Exception { + assertMalformedDeltaMarksGap(new byte[]{0, (byte) 0x80}); + } + + @Test + public void testUnterminatedDeltaStartMarksRecoveredDeltaAsGap() throws Exception { + assertMalformedDeltaMarksGap(new byte[]{(byte) 0x80}); + } + + @Test + public void testUnterminatedSymbolLengthMarksRecoveredDeltaAsGap() throws Exception { + assertMalformedDeltaMarksGap(new byte[]{0, 1, (byte) 0x80}); + } + + private void assertMalformedDeltaMarksGap(byte[] deltaSection) throws Exception { + assertMemoryLeak(() -> { + Path slot = temporaryFolder.newFolder("qwp-malformed-recovery").toPath(); + int payloadLen = QwpConstants.HEADER_SIZE + deltaSection.length; + long payload = Unsafe.malloc(payloadLen, MemoryTag.NATIVE_DEFAULT); + try { + Unsafe.getUnsafe().setMemory(payload, payloadLen, (byte) 0); + Unsafe.getUnsafe().putInt(payload, QwpConstants.MAGIC_MESSAGE); + Unsafe.getUnsafe().putByte( + payload + QwpConstants.HEADER_OFFSET_FLAGS, + QwpConstants.FLAG_DELTA_SYMBOL_DICT); + for (int i = 0; i < deltaSection.length; i++) { + Unsafe.getUnsafe().putByte(payload + QwpConstants.HEADER_SIZE + i, deltaSection[i]); + } + try (CursorSendEngine writer = new CursorSendEngine(slot.toString(), 4_096)) { + Assert.assertEquals(0L, writer.appendBlocking(payload, payloadLen)); + } + } finally { + Unsafe.free(payload, payloadLen, MemoryTag.NATIVE_DEFAULT); + } + + try (CursorSendEngine recovered = new CursorSendEngine(slot.toString(), 4_096)) { + ObjList symbols = new ObjList<>(); + Assert.assertEquals("malformed recovered delta must be fail-clean", + -1L, recovered.collectReplaySymbolsAbove(0, symbols)); + Assert.assertEquals("malformed delta must not recover partial symbols", 0, symbols.size()); + Assert.assertEquals("the malformed frame must be visited during recovery", + 1L, recovered.recoveryFramesVisited()); + } + }); + } +} diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SlotLockTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SlotLockTest.java index d38ef9e5..9489ee82 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SlotLockTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SlotLockTest.java @@ -26,6 +26,7 @@ import io.questdb.client.cutlass.qwp.client.sf.cursor.SlotLock; import io.questdb.client.std.Files; +import io.questdb.client.test.tools.DelegatingFilesFacade; import io.questdb.client.test.tools.TestUtils; import org.junit.After; import org.junit.Before; @@ -34,6 +35,7 @@ import java.nio.file.Paths; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -129,6 +131,36 @@ public void testLogicalLockRemainsContendedAcrossSlotRenameAndRecreate() throws }); } + @Test + public void testLogicalLockRejectsInvalidPaths() throws Exception { + TestUtils.assertMemoryLeak(() -> { + assertLogicalPathRejected(null, "slotDir must not be empty"); + assertLogicalPathRejected("", "slotDir must not be empty"); + assertLogicalPathRejected("slot", + "slotDir must contain a parent and slot name: slot"); + }); + } + + @Test + public void testLogicalLockReportsLockDirectoryCreationFailure() throws Exception { + TestUtils.assertMemoryLeak(() -> { + String slot = Paths.get(parentDir, "mkdir-failure").toString(); + String lockDir = Paths.get(parentDir, ".slot-locks").toString(); + LockDirectoryFailureFacade ff = new LockDirectoryFailureFacade(lockDir); + try { + SlotLock.acquireLogical(ff, slot); + fail("expected logical lock directory creation failure"); + } catch (IllegalStateException expected) { + assertEquals("could not create logical slot lock dir: " + lockDir + " rc=-1", + expected.getMessage()); + } + assertEquals("lock file must not be opened after directory creation fails", + 0, ff.openRwCalls); + assertFalse("failed mkdir must not leave the lock directory behind", + Files.exists(lockDir)); + }); + } + @Test public void testTwoDifferentSlotsCoexist() throws Exception { TestUtils.assertMemoryLeak(() -> { @@ -168,6 +200,17 @@ private static void rmDir(String dir) { Files.remove(dir); } + private static void assertLogicalPathRejected(String slotDir, String expectedMessage) { + Throwable thrown = null; + try { + SlotLock.acquireLogical(slotDir); + } catch (Throwable t) { + thrown = t; + } + assertTrue("expected IllegalArgumentException", thrown instanceof IllegalArgumentException); + assertEquals(expectedMessage, thrown.getMessage()); + } + private static boolean isDir(String path) { // Cheap heuristic: directories have a readable findFirst handle. long find = Files.findFirst(path); @@ -175,4 +218,29 @@ private static boolean isDir(String path) { Files.findClose(find); return true; } + + private static final class LockDirectoryFailureFacade extends DelegatingFilesFacade { + private final String lockDir; + private int openRwCalls; + + private LockDirectoryFailureFacade(String lockDir) { + this.lockDir = lockDir; + } + + @Override + public boolean exists(String path) { + return !lockDir.equals(path) && super.exists(path); + } + + @Override + public int mkdir(String path, int mode) { + return lockDir.equals(path) ? -1 : super.mkdir(path, mode); + } + + @Override + public int openRW(String path) { + openRwCalls++; + return super.openRW(path); + } + } } diff --git a/core/src/test/java/io/questdb/client/test/tools/DelegatingFilesFacade.java b/core/src/test/java/io/questdb/client/test/tools/DelegatingFilesFacade.java index 516800c4..63a1b132 100644 --- a/core/src/test/java/io/questdb/client/test/tools/DelegatingFilesFacade.java +++ b/core/src/test/java/io/questdb/client/test/tools/DelegatingFilesFacade.java @@ -33,9 +33,9 @@ * implausible file length -- so a test can drive a real I/O failure path without a * real broken disk. *

      - * {@code FilesFacade} has no default methods, so the delegation boilerplate is - * unavoidable; it lives here once rather than being stamped into each test that - * needs a fault seam. + * {@code FilesFacade} deliberately leaves most operations abstract, so the + * delegation boilerplate lives here once rather than being stamped into each + * test that needs a fault seam. */ public class DelegatingFilesFacade implements FilesFacade { @Override From 0428fd8162c8288cc9e052a09a56fd72c0a3f20d Mon Sep 17 00:00:00 2001 From: glasstiger Date: Fri, 17 Jul 2026 12:43:04 +0100 Subject: [PATCH 85/96] Clean up QWP recovery scaffolding Remove the unused symbol-mirror append helper. Reuse shared wire parsing and deletion-assured temporary directory fixtures in recovery tests. --- .../sf/cursor/CursorWebSocketSendLoop.java | 27 -- .../qwp/client/DeltaDictRecoveryTest.java | 15 +- .../qwp/client/SelfSufficientFramesTest.java | 339 ++++++++--------- ...CursorWebSocketSendLoopMirrorLeakTest.java | 354 +++++++++--------- 4 files changed, 331 insertions(+), 404 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java index cfde65e6..410853a8 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java @@ -43,7 +43,6 @@ import io.questdb.client.std.CharSequenceLongHashMap; import io.questdb.client.std.MemoryTag; import io.questdb.client.std.QuietCloseable; -import io.questdb.client.std.str.Utf8s; import io.questdb.client.std.Unsafe; import org.jetbrains.annotations.TestOnly; import org.slf4j.Logger; @@ -2460,32 +2459,6 @@ private void accumulateSentDict(long payloadAddr, int payloadLen, int deltaStart sentDictIndexedCount = sentDictCount; } - /** - * Appends one symbol to the sent-dictionary mirror in wire form - * ({@code [len varint][utf8]}), advancing {@code sentDictCount}. Seeds the mirror at - * construction; {@link #accumulateSentDict} extends it from live frames thereafter. - */ - private void appendSymbolToMirror(CharSequence symbol) { - if (forceMirrorSeedFailureForTest) { - // Simulate a native realloc OOM on the seed path (see the field). - throw new LineSenderException("simulated mirror seed allocation failure (test only)"); - } - int utf8Len = Utf8s.utf8Bytes(symbol); - int wireLen = NativeBufferWriter.varintSize(utf8Len) + utf8Len; - ensureSentDictCapacity((long) sentDictBytesLen + wireLen); - ensureSentDictEntryEndsCapacity((long) sentDictCount + 1L); - long p = NativeBufferWriter.writeVarint(sentDictBytesAddr + sentDictBytesLen, utf8Len); - if (utf8Len > 0) { - Utf8s.strCpyUtf8(symbol, p, utf8Len); - } - sentDictBytesLen += wireLen; - Unsafe.getUnsafe().putInt( - sentDictEntryEndsAddr + (long) sentDictCount * Integer.BYTES, - sentDictBytesLen); - sentDictCount++; - sentDictIndexedCount = sentDictCount; - } - private void ensureSentDictCapacity(long required) { if (sentDictBytesCapacity >= required) { return; diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java index 8979036c..6b7f2201 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java @@ -35,10 +35,11 @@ import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer; import io.questdb.client.test.tools.DelegatingFilesFacade; import io.questdb.client.test.tools.TestUtils; -import org.junit.After; import org.junit.Assert; import org.junit.Before; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.TemporaryFolder; import java.io.IOException; import java.nio.ByteBuffer; @@ -71,17 +72,15 @@ public class DeltaDictRecoveryTest { private static final int DISTINCT_SYMBOLS = 8; private static final int ROWS = 40; + + @Rule + public final TemporaryFolder temporaryFolder = TemporaryFolder.builder().assureDeletion().build(); + private String sfDir; @Before public void setUp() { - sfDir = Paths.get(System.getProperty("java.io.tmpdir"), - "qdb-delta-recov-" + System.nanoTime()).toString(); - } - - @After - public void tearDown() { - TestUtils.removeTmpDirRec(sfDir); + sfDir = temporaryFolder.getRoot().toPath().resolve("qdb-delta-recovery").toString(); } @Test diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java index 0c88bc11..be1f1744 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/SelfSufficientFramesTest.java @@ -31,7 +31,9 @@ import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer; import io.questdb.client.test.tools.TestUtils; import org.junit.Assert; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.TemporaryFolder; import java.io.IOException; import java.nio.ByteBuffer; @@ -63,57 +65,56 @@ public class SelfSufficientFramesTest { /** First byte of the symbol-dict delta payload after the 12-byte QWP header. */ private static final int DELTA_START_OFFSET = 12; + @Rule + public final TemporaryFolder temporaryFolder = TemporaryFolder.builder().assureDeletion().build(); + @Test public void testFileModeShipsMonotonicDeltaAndPersistsDict() throws Exception { // File-backed SF also ships monotonic deltas now: batch 2 carries only // "beta" (deltaStart=1). The dictionary is durably kept in .symbol-dict // so a recovered/orphan-drained slot can rebuild it. - Path sfDir = Files.createTempDirectory("qwp-sf-selfsufficient"); - try { - assertMemoryLeak(() -> { - CapturingHandler handler = new CapturingHandler(); - try (TestWebSocketServer server = new TestWebSocketServer(handler)) { - int port = server.getPort(); - server.start(); - Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); - - String config = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";"; - // The engine places slot files under sf_dir/ (default "default"). - Path dictFile = sfDir.resolve("default").resolve(".symbol-dict"); - try (Sender sender = Sender.fromConfig(config)) { - sender.table("foo").symbol("s", "alpha").longColumn("v", 1L).atNow(); - sender.flush(); - waitFor(() -> handler.batches.size() >= 1, 5_000); + Path sfDir = temporaryFolder.newFolder("qwp-sf-selfsufficient").toPath(); + assertMemoryLeak(() -> { + CapturingHandler handler = new CapturingHandler(); + try (TestWebSocketServer server = new TestWebSocketServer(handler)) { + int port = server.getPort(); + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); - sender.table("foo").symbol("s", "beta").longColumn("v", 2L).atNow(); - sender.flush(); - waitFor(() -> handler.batches.size() >= 2, 5_000); - - // Check the persisted dictionary while the sender is live: a - // fully-drained close intentionally unlinks it (slot cleanup). - Assert.assertTrue("persisted dictionary file exists", Files.exists(dictFile)); - byte[] dict = Files.readAllBytes(dictFile); - Assert.assertTrue("dictionary retains alpha", containsUtf8(dict, "alpha")); - Assert.assertTrue("dictionary retains beta", containsUtf8(dict, "beta")); - } + String config = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";"; + // The engine places slot files under sf_dir/ (default "default"). + Path dictFile = sfDir.resolve("default").resolve(".symbol-dict"); + try (Sender sender = Sender.fromConfig(config)) { + sender.table("foo").symbol("s", "alpha").longColumn("v", 1L).atNow(); + sender.flush(); + waitFor(() -> handler.batches.size() >= 1, 5_000); + + sender.table("foo").symbol("s", "beta").longColumn("v", 2L).atNow(); + sender.flush(); + waitFor(() -> handler.batches.size() >= 2, 5_000); - Assert.assertEquals("expected 2 captured batches", 2, handler.batches.size()); - byte[] b1 = handler.batches.get(0); - byte[] b2 = handler.batches.get(1); - - Assert.assertEquals("batch 1 deltaStart must be 0", - 0, readVarint(b1, DELTA_START_OFFSET)); - Assert.assertEquals("batch 1 deltaCount must be 1", 1, readVarint(b1, DELTA_START_OFFSET + 1)); - // batch 2 ships ONLY beta as a delta from id 1. - Assert.assertEquals("batch 2 deltaStart must be 1 (monotonic)", - 1, readVarint(b2, DELTA_START_OFFSET)); - Assert.assertEquals("batch 2 deltaCount must be 1 (only the new symbol)", - 1, readVarint(b2, DELTA_START_OFFSET + 1)); + // Check the persisted dictionary while the sender is live: a + // fully-drained close intentionally unlinks it (slot cleanup). + Assert.assertTrue("persisted dictionary file exists", Files.exists(dictFile)); + byte[] dict = Files.readAllBytes(dictFile); + Assert.assertTrue("dictionary retains alpha", containsUtf8(dict, "alpha")); + Assert.assertTrue("dictionary retains beta", containsUtf8(dict, "beta")); } - }); - } finally { - rmDir(sfDir); - } + + Assert.assertEquals("expected 2 captured batches", 2, handler.batches.size()); + byte[] b1 = handler.batches.get(0); + byte[] b2 = handler.batches.get(1); + + Assert.assertEquals("batch 1 deltaStart must be 0", + 0, readVarint(b1, DELTA_START_OFFSET)); + Assert.assertEquals("batch 1 deltaCount must be 1", 1, readVarint(b1, DELTA_START_OFFSET + 1)); + // batch 2 ships ONLY beta as a delta from id 1. + Assert.assertEquals("batch 2 deltaStart must be 1 (monotonic)", + 1, readVarint(b2, DELTA_START_OFFSET)); + Assert.assertEquals("batch 2 deltaCount must be 1 (only the new symbol)", + 1, readVarint(b2, DELTA_START_OFFSET + 1)); + } + }); } @Test @@ -126,54 +127,50 @@ public void testDiskModeFallsBackToFullDictWhenPersistedDictUnopenable() throws // server -- the full-dict frame is the safe degradation. Force the open // failure by planting a DIRECTORY where the dictionary file belongs: // openRW / openCleanRW on a directory fails, so open() returns null. - Path sfDir = Files.createTempDirectory("qwp-sf-fallback"); + Path sfDir = temporaryFolder.newFolder("qwp-sf-fallback").toPath(); Path dictPath = sfDir.resolve("default").resolve(".symbol-dict"); Files.createDirectories(dictPath); // a directory, not a file Files.createFile(dictPath.resolve("blocker")); // non-empty: cannot be unlinked/rmdir'd - try { - assertMemoryLeak(() -> { - CapturingHandler handler = new CapturingHandler(); - try (TestWebSocketServer server = new TestWebSocketServer(handler)) { - int port = server.getPort(); - server.start(); - Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); - - String config = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";"; - try (Sender sender = Sender.fromConfig(config)) { - sender.table("foo").symbol("s", "alpha").longColumn("v", 1L).atNow(); - sender.flush(); - waitFor(() -> handler.batches.size() >= 1, 5_000); + assertMemoryLeak(() -> { + CapturingHandler handler = new CapturingHandler(); + try (TestWebSocketServer server = new TestWebSocketServer(handler)) { + int port = server.getPort(); + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); - sender.table("foo").symbol("s", "beta").longColumn("v", 2L).atNow(); - sender.flush(); - waitFor(() -> handler.batches.size() >= 2, 5_000); - } + String config = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";"; + try (Sender sender = Sender.fromConfig(config)) { + sender.table("foo").symbol("s", "alpha").longColumn("v", 1L).atNow(); + sender.flush(); + waitFor(() -> handler.batches.size() >= 1, 5_000); - // The planted directory is untouched -- the dictionary never - // opened, so delta encoding stayed disabled. - Assert.assertTrue("planted .symbol-dict directory must remain (open failed)", - Files.isDirectory(dictPath)); - - Assert.assertEquals("expected 2 captured batches", 2, handler.batches.size()); - byte[] b1 = handler.batches.get(0); - byte[] b2 = handler.batches.get(1); - - // Full-dict fallback: BOTH batches start at id 0, and batch 2 - // re-ships the WHOLE dictionary (alpha + beta), NOT a monotonic - // delta (which would be deltaStart=1, deltaCount=1 as above). - Assert.assertEquals("batch 1 deltaStart must be 0", - 0, readVarint(b1, DELTA_START_OFFSET)); - Assert.assertEquals("batch 1 deltaCount must be 1", - 1, readVarint(b1, DELTA_START_OFFSET + 1)); - Assert.assertEquals("batch 2 deltaStart must be 0 (full-dict fallback, not monotonic)", - 0, readVarint(b2, DELTA_START_OFFSET)); - Assert.assertEquals("batch 2 deltaCount must be 2 (whole dictionary re-shipped)", - 2, readVarint(b2, DELTA_START_OFFSET + 1)); + sender.table("foo").symbol("s", "beta").longColumn("v", 2L).atNow(); + sender.flush(); + waitFor(() -> handler.batches.size() >= 2, 5_000); } - }); - } finally { - rmDir(sfDir); - } + + // The planted directory is untouched -- the dictionary never + // opened, so delta encoding stayed disabled. + Assert.assertTrue("planted .symbol-dict directory must remain (open failed)", + Files.isDirectory(dictPath)); + + Assert.assertEquals("expected 2 captured batches", 2, handler.batches.size()); + byte[] b1 = handler.batches.get(0); + byte[] b2 = handler.batches.get(1); + + // Full-dict fallback: BOTH batches start at id 0, and batch 2 + // re-ships the WHOLE dictionary (alpha + beta), NOT a monotonic + // delta (which would be deltaStart=1, deltaCount=1 as above). + Assert.assertEquals("batch 1 deltaStart must be 0", + 0, readVarint(b1, DELTA_START_OFFSET)); + Assert.assertEquals("batch 1 deltaCount must be 1", + 1, readVarint(b1, DELTA_START_OFFSET + 1)); + Assert.assertEquals("batch 2 deltaStart must be 0 (full-dict fallback, not monotonic)", + 0, readVarint(b2, DELTA_START_OFFSET)); + Assert.assertEquals("batch 2 deltaCount must be 2 (whole dictionary re-shipped)", + 2, readVarint(b2, DELTA_START_OFFSET + 1)); + } + }); } private static boolean containsUtf8(byte[] haystack, String needle) { @@ -375,56 +372,52 @@ public void testFileModeSplitPersistsDictBeforeAFAILEDPublish() throws Exception // attempted. sf_max_bytes=64 then makes every frame larger than the segment, so // appendBlocking fails with PAYLOAD_TOO_LARGE -- deterministically, no // backpressure timing needed. - Path sfDir = Files.createTempDirectory("qwp-sf-split-persist-fail"); - try { - assertMemoryLeak(() -> { - CapturingHandler handler = new CapturingHandler(); - try (TestWebSocketServer server = new TestWebSocketServer(handler)) { - server.setAdvertisedMaxBatchSize(150); // forces the two-table batch to split - int port = server.getPort(); - server.start(); - Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); - - String config = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir - + ";sf_max_bytes=64;"; - String pad = TestUtils.repeat("x", 60); - Sender sender = Sender.fromConfig(config); + Path sfDir = temporaryFolder.newFolder("qwp-sf-split-persist-fail").toPath(); + assertMemoryLeak(() -> { + CapturingHandler handler = new CapturingHandler(); + try (TestWebSocketServer server = new TestWebSocketServer(handler)) { + server.setAdvertisedMaxBatchSize(150); // forces the two-table batch to split + int port = server.getPort(); + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); + + String config = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + + ";sf_max_bytes=64;"; + String pad = TestUtils.repeat("x", 60); + Sender sender = Sender.fromConfig(config); + try { + sender.table("t1").symbol("s", "alpha").stringColumn("p", pad).longColumn("v", 1L).atNow(); + sender.table("t2").symbol("s", "bravo").stringColumn("p", pad).longColumn("v", 2L).atNow(); + try { + sender.flush(); + Assert.fail("a split frame larger than the segment must fail to publish"); + } catch (LineSenderException expected) { + // PAYLOAD_TOO_LARGE -- the publish, not the pre-flight. + } + Assert.assertEquals("the publish must fail, so no frame reaches the server", + 0, handler.batches.size()); + + // The write-ahead already ran: both of the batch's new symbols are + // durable even though the frame that references them never + // published. Move the persist after sealAndSwapBuffer and this is 0. + ObjList persisted = + ((QwpWebSocketSender) sender).getPersistedSymbolsForTest(); + Assert.assertNotNull(persisted); + Assert.assertEquals("the split path must persist its new symbols BEFORE " + + "publishing the frame that references them", 2, persisted.size()); + Assert.assertEquals("alpha", persisted.getQuick(0)); + Assert.assertEquals("bravo", persisted.getQuick(1)); + } finally { try { - sender.table("t1").symbol("s", "alpha").stringColumn("p", pad).longColumn("v", 1L).atNow(); - sender.table("t2").symbol("s", "bravo").stringColumn("p", pad).longColumn("v", 2L).atNow(); - try { - sender.flush(); - Assert.fail("a split frame larger than the segment must fail to publish"); - } catch (LineSenderException expected) { - // PAYLOAD_TOO_LARGE -- the publish, not the pre-flight. - } - Assert.assertEquals("the publish must fail, so no frame reaches the server", - 0, handler.batches.size()); - - // The write-ahead already ran: both of the batch's new symbols are - // durable even though the frame that references them never - // published. Move the persist after sealAndSwapBuffer and this is 0. - ObjList persisted = - ((QwpWebSocketSender) sender).getPersistedSymbolsForTest(); - Assert.assertNotNull(persisted); - Assert.assertEquals("the split path must persist its new symbols BEFORE " - + "publishing the frame that references them", 2, persisted.size()); - Assert.assertEquals("alpha", persisted.getQuick(0)); - Assert.assertEquals("bravo", persisted.getQuick(1)); - } finally { - try { - sender.close(); - } catch (LineSenderException ignored) { - // close() re-flushes the still-buffered oversized rows and fails - // again; expected, and not what this test asserts. close() still - // runs its resource cleanup, so no native memory leaks. - } + sender.close(); + } catch (LineSenderException ignored) { + // close() re-flushes the still-buffered oversized rows and fails + // again; expected, and not what this test asserts. close() still + // runs its resource cleanup, so no native memory leaks. } } - }); - } finally { - rmDir(sfDir); - } + } + }); } @Test @@ -437,43 +430,39 @@ public void testFileModeSplitPersistsDictBeforePublish() throws Exception { // recovered / orphan-drained slot can rebuild what the delta frames reference. // The other split tests run in memory mode, so this is the only coverage of the // split x persist path with a live PersistedSymbolDict. - Path sfDir = Files.createTempDirectory("qwp-sf-split-persist"); - try { - assertMemoryLeak(() -> { - CapturingHandler handler = new CapturingHandler(); - try (TestWebSocketServer server = new TestWebSocketServer(handler)) { - server.setAdvertisedMaxBatchSize(150); // forces the two-table batch to split - int port = server.getPort(); - server.start(); - Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); - - String config = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";"; - Path dictFile = sfDir.resolve("default").resolve(".symbol-dict"); - String pad = new String(new char[60]).replace('\0', 'x'); - try (Sender sender = Sender.fromConfig(config)) { - // Two tables, two new symbols, ONE flush -> the combined message - // exceeds cap 150 and splits into two frames. - sender.table("t1").symbol("s", "alpha").stringColumn("p", pad).longColumn("v", 1L).atNow(); - sender.table("t2").symbol("s", "bravo").stringColumn("p", pad).longColumn("v", 2L).atNow(); - sender.flush(); - waitFor(() -> handler.batches.size() >= 2, 5_000); - - // Check .symbol-dict while the sender is live: a fully-drained - // close would unlink it. The split's first-frame write-ahead - // persist must have recorded BOTH new symbols. - Assert.assertTrue("persisted dictionary file exists", Files.exists(dictFile)); - byte[] dict = Files.readAllBytes(dictFile); - Assert.assertTrue("split-flush persist must record alpha", containsUtf8(dict, "alpha")); - Assert.assertTrue("split-flush persist must record bravo", containsUtf8(dict, "bravo")); - } + Path sfDir = temporaryFolder.newFolder("qwp-sf-split-persist").toPath(); + assertMemoryLeak(() -> { + CapturingHandler handler = new CapturingHandler(); + try (TestWebSocketServer server = new TestWebSocketServer(handler)) { + server.setAdvertisedMaxBatchSize(150); // forces the two-table batch to split + int port = server.getPort(); + server.start(); + Assert.assertTrue(server.awaitStart(5, TimeUnit.SECONDS)); - Assert.assertEquals("the oversized two-table batch must split into 2 frames", - 2, handler.batches.size()); + String config = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";"; + Path dictFile = sfDir.resolve("default").resolve(".symbol-dict"); + String pad = new String(new char[60]).replace('\0', 'x'); + try (Sender sender = Sender.fromConfig(config)) { + // Two tables, two new symbols, ONE flush -> the combined message + // exceeds cap 150 and splits into two frames. + sender.table("t1").symbol("s", "alpha").stringColumn("p", pad).longColumn("v", 1L).atNow(); + sender.table("t2").symbol("s", "bravo").stringColumn("p", pad).longColumn("v", 2L).atNow(); + sender.flush(); + waitFor(() -> handler.batches.size() >= 2, 5_000); + + // Check .symbol-dict while the sender is live: a fully-drained + // close would unlink it. The split's first-frame write-ahead + // persist must have recorded BOTH new symbols. + Assert.assertTrue("persisted dictionary file exists", Files.exists(dictFile)); + byte[] dict = Files.readAllBytes(dictFile); + Assert.assertTrue("split-flush persist must record alpha", containsUtf8(dict, "alpha")); + Assert.assertTrue("split-flush persist must record bravo", containsUtf8(dict, "bravo")); } - }); - } finally { - rmDir(sfDir); - } + + Assert.assertEquals("the oversized two-table batch must split into 2 frames", + 2, handler.batches.size()); + } + }); } @Test @@ -546,21 +535,7 @@ public void testSplitPreflightAdvancesBaselineSoLaterFramesArentSizedWithTheDelt } private static int readVarint(byte[] buf, int offset) { - // Simple unsigned varint decode — sufficient for small values. - int result = 0; - int shift = 0; - while (offset < buf.length) { - int b = buf[offset++] & 0xFF; - result |= (b & 0x7F) << shift; - if ((b & 0x80) == 0) return result; - shift += 7; - if (shift > 28) throw new IllegalStateException("varint too long"); - } - throw new IllegalStateException("varint truncated"); - } - - private static void rmDir(Path dir) { - TestUtils.removeTmpDirRec(dir == null ? null : dir.toString()); + return QwpWireTestUtils.readVarint(buf, new int[]{offset}); } private static void waitFor(BoolCondition cond, long timeoutMillis) { diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopMirrorLeakTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopMirrorLeakTest.java index c2902e11..63ad5d51 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopMirrorLeakTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopMirrorLeakTest.java @@ -30,9 +30,10 @@ import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorWebSocketSendLoop; import io.questdb.client.cutlass.qwp.client.sf.cursor.PersistedSymbolDict; import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer; -import io.questdb.client.test.tools.TestUtils; import org.junit.Assert; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.TemporaryFolder; import java.io.IOException; import java.lang.reflect.Field; @@ -58,59 +59,58 @@ public class CursorWebSocketSendLoopMirrorLeakTest { private static final int DISTINCT_SYMBOLS = 8; private static final int ROWS = 40; + @Rule + public final TemporaryFolder temporaryFolder = TemporaryFolder.builder().assureDeletion().build(); + @Test public void testSeededMirrorFreedWhenLoopClosedWithoutStart() throws Exception { - Path sfDir = Files.createTempDirectory("qwp-mirror-leak"); - try { - // Populate a slot with delta frames + a non-empty .symbol-dict, then - // abandon it (silent server, close-fast) -- outside assertMemoryLeak, - // because a full Sender+server round trip is not net-zero on its own. - populateRecoverableSlot(sfDir); + Path sfDir = temporaryFolder.newFolder("qwp-mirror-leak").toPath(); + // Populate a slot with delta frames + a non-empty .symbol-dict, then + // abandon it (silent server, close-fast) -- outside assertMemoryLeak, + // because a full Sender+server round trip is not net-zero on its own. + populateRecoverableSlot(sfDir); - Path slot = sfDir.resolve("default"); - Assert.assertTrue("populate must leave a persisted dictionary", - Files.exists(slot.resolve(".symbol-dict"))); + Path slot = sfDir.resolve("default"); + Assert.assertTrue("populate must leave a persisted dictionary", + Files.exists(slot.resolve(".symbol-dict"))); - // Only the recovery construct + close is leak-checked: the engine - // recovers (loading the dict), the loop ctor seeds the mirror from it, - // and close() -- with NO start() -- must free every native allocation. - // Pre-fix the seeded mirror leaks here and this assertion fails. - assertMemoryLeak(() -> { - try (CursorSendEngine engine = new CursorSendEngine(slot.toString(), 4096)) { - PersistedSymbolDict pd = engine.getPersistedSymbolDict(); - Assert.assertNotNull("disk-mode engine must open a persisted dict", pd); - Assert.assertTrue("recovery must load the persisted symbols (seeds the mirror)", - pd.size() > 0 && pd.loadedEntriesLen() > 0); - long persistedAddr = pd.loadedEntriesAddr(); + // Only the recovery construct + close is leak-checked: the engine + // recovers (loading the dict), the loop ctor seeds the mirror from it, + // and close() -- with NO start() -- must free every native allocation. + // Pre-fix the seeded mirror leaks here and this assertion fails. + assertMemoryLeak(() -> { + try (CursorSendEngine engine = new CursorSendEngine(slot.toString(), 4096)) { + PersistedSymbolDict pd = engine.getPersistedSymbolDict(); + Assert.assertNotNull("disk-mode engine must open a persisted dict", pd); + Assert.assertTrue("recovery must load the persisted symbols (seeds the mirror)", + pd.size() > 0 && pd.loadedEntriesLen() > 0); + long persistedAddr = pd.loadedEntriesAddr(); - CursorWebSocketSendLoop loop = new CursorWebSocketSendLoop( - null, engine, 0, 1_000_000L, - () -> { - throw new IOException("no reconnect in this test"); - }, - 0, 0, 1); - // Close without start(): the ctor-seeded mirror is this - // thread's to free, since the I/O loop never ran. - Assert.assertTrue("precondition: the ctor seeded a non-empty mirror", - readInt(loop, "sentDictCount") > 0); - Assert.assertEquals("foreground loop must take the persisted buffer without copying", - persistedAddr, readLong(loop, "sentDictBytesAddr")); - Assert.assertEquals("ownership transfer must clear the persisted pointer", - 0L, pd.loadedEntriesAddr()); - Assert.assertTrue("foreground mirror must own the transferred buffer", - readBoolean(loop, "sentDictBytesOwned")); - loop.close(); - // close() must reset sentDictCount alongside freeing the buffer, - // so the mirror stays all-or-nothing: a hypothetical post-close - // start() (no closed guard) cannot read a stale count against a - // freed buffer and drive a null-mirror catch-up. - Assert.assertEquals("close() must reset sentDictCount to 0", - 0, readInt(loop, "sentDictCount")); - } - }); - } finally { - rmDir(sfDir); - } + CursorWebSocketSendLoop loop = new CursorWebSocketSendLoop( + null, engine, 0, 1_000_000L, + () -> { + throw new IOException("no reconnect in this test"); + }, + 0, 0, 1); + // Close without start(): the ctor-seeded mirror is this + // thread's to free, since the I/O loop never ran. + Assert.assertTrue("precondition: the ctor seeded a non-empty mirror", + readInt(loop, "sentDictCount") > 0); + Assert.assertEquals("foreground loop must take the persisted buffer without copying", + persistedAddr, readLong(loop, "sentDictBytesAddr")); + Assert.assertEquals("ownership transfer must clear the persisted pointer", + 0L, pd.loadedEntriesAddr()); + Assert.assertTrue("foreground mirror must own the transferred buffer", + readBoolean(loop, "sentDictBytesOwned")); + loop.close(); + // close() must reset sentDictCount alongside freeing the buffer, + // so the mirror stays all-or-nothing: a hypothetical post-close + // start() (no closed guard) cannot read a stale count against a + // freed buffer and drive a null-mirror catch-up. + Assert.assertEquals("close() must reset sentDictCount to 0", + 0, readInt(loop, "sentDictCount")); + } + }); } @Test @@ -126,170 +126,154 @@ public void testRecycledLoopReSeedsMirrorFromPersistedDict() throws Exception { // bogus "resend required" terminal. Borrowing the entries leaves the // dictionary intact for the engine's lifetime without making another native // copy, so every recycled loop can re-seed. - Path sfDir = Files.createTempDirectory("qwp-mirror-reseed"); - try { - populateRecoverableSlot(sfDir); - Path slot = sfDir.resolve("default"); - assertMemoryLeak(() -> { - try (CursorSendEngine engine = new CursorSendEngine(slot.toString(), 4096)) { - PersistedSymbolDict pd = engine.getPersistedSymbolDict(); - Assert.assertNotNull(pd); - int dictSize = pd.size(); - Assert.assertTrue("recovery must load a non-empty dictionary", dictSize > 0); - long persistedAddr = pd.loadedEntriesAddr(); + Path sfDir = temporaryFolder.newFolder("qwp-mirror-reseed").toPath(); + populateRecoverableSlot(sfDir); + Path slot = sfDir.resolve("default"); + assertMemoryLeak(() -> { + try (CursorSendEngine engine = new CursorSendEngine(slot.toString(), 4096)) { + PersistedSymbolDict pd = engine.getPersistedSymbolDict(); + Assert.assertNotNull(pd); + int dictSize = pd.size(); + Assert.assertTrue("recovery must load a non-empty dictionary", dictSize > 0); + long persistedAddr = pd.loadedEntriesAddr(); - // Session 1 seeds its mirror from the persisted dictionary. - CursorWebSocketSendLoop loop1 = newRecoveryLoop(engine); - try { - Assert.assertEquals("session-1 mirror must seed from the persisted dict", - dictSize, readInt(loop1, "sentDictCount")); - Assert.assertEquals("orphan session must borrow the persisted bytes", - persistedAddr, readLong(loop1, "sentDictBytesAddr")); - Assert.assertFalse("borrowed orphan mirror must not own the persisted bytes", - readBoolean(loop1, "sentDictBytesOwned")); - } finally { - loop1.close(); - } - Assert.assertEquals("closing a borrowed loop must leave the engine prefix alive", - persistedAddr, pd.loadedEntriesAddr()); + // Session 1 seeds its mirror from the persisted dictionary. + CursorWebSocketSendLoop loop1 = newRecoveryLoop(engine); + try { + Assert.assertEquals("session-1 mirror must seed from the persisted dict", + dictSize, readInt(loop1, "sentDictCount")); + Assert.assertEquals("orphan session must borrow the persisted bytes", + persistedAddr, readLong(loop1, "sentDictBytesAddr")); + Assert.assertFalse("borrowed orphan mirror must not own the persisted bytes", + readBoolean(loop1, "sentDictBytesOwned")); + } finally { + loop1.close(); + } + Assert.assertEquals("closing a borrowed loop must leave the engine prefix alive", + persistedAddr, pd.loadedEntriesAddr()); - // Session 2 against the SAME engine (the drainer recycle): the - // seed must NOT have been consumed -- the mirror must re-seed to - // the full dictionary so the reconnect catch-up is complete. - CursorWebSocketSendLoop loop2 = newRecoveryLoop(engine); - try { - Assert.assertEquals("recycled session-2 mirror must re-seed from the " - + "persisted dict (pre-fix it was 0)", - dictSize, readInt(loop2, "sentDictCount")); - Assert.assertEquals(persistedAddr, readLong(loop2, "sentDictBytesAddr")); - } finally { - loop2.close(); - } + // Session 2 against the SAME engine (the drainer recycle): the + // seed must NOT have been consumed -- the mirror must re-seed to + // the full dictionary so the reconnect catch-up is complete. + CursorWebSocketSendLoop loop2 = newRecoveryLoop(engine); + try { + Assert.assertEquals("recycled session-2 mirror must re-seed from the " + + "persisted dict (pre-fix it was 0)", + dictSize, readInt(loop2, "sentDictCount")); + Assert.assertEquals(persistedAddr, readLong(loop2, "sentDictBytesAddr")); + } finally { + loop2.close(); } - }); - } finally { - rmDir(sfDir); - } + } + }); } @Test public void testCtorFreesSeededMirrorWhenFrameSeedThrows() throws Exception { // C1 regression: the constructor seeds the recovery mirror in TWO steps -- a // malloc from the persisted dictionary's intact prefix, then an extension from - // the surviving frames' own deltas (appendSymbolToMirror). If that second step + // the recovered suffix retained by the engine. If that second step // throws (a native realloc OOM, or the MAX_SENT_DICT_BYTES ceiling), the throw // leaves the constructor with the object unpublished, so neither // ensureConnected's catch nor BackgroundDrainer's finally can ever close() it -- // and the already-malloc'd prefix mirror leaks. The constructor must free it on // the throw. Pre-fix, the mirror leaks here and assertMemoryLeak fails. - Path sfDir = Files.createTempDirectory("qwp-mirror-ctor-throw"); - try { - // A torn-dict SUBSET: three delta frames a@0,b@1,c@2 survive on disk, but the - // .symbol-dict is rewritten to hold only [a,b] (a host-crash tail tear). On - // recovery pd.size()==2 seeds (mallocs) the mirror, then the frame-seed - // rebuilds c@2 from the surviving frame -- the append the fault interrupts. - populateThreeFrameSlot(sfDir); - Path slot = sfDir.resolve("default"); - replacePersistedDictionaryWithTwoSymbolPrefix(slot); + Path sfDir = temporaryFolder.newFolder("qwp-mirror-ctor-throw").toPath(); + // A torn-dict SUBSET: three delta frames a@0,b@1,c@2 survive on disk, but the + // .symbol-dict is rewritten to hold only [a,b] (a host-crash tail tear). On + // recovery pd.size()==2 seeds (mallocs) the mirror, then the frame-seed + // copies c@2 from the recovered suffix -- the operation the fault interrupts. + populateThreeFrameSlot(sfDir); + Path slot = sfDir.resolve("default"); + replacePersistedDictionaryWithTwoSymbolPrefix(slot); - assertMemoryLeak(() -> { - try (CursorSendEngine engine = new CursorSendEngine(slot.toString(), 4096)) { - PersistedSymbolDict pd = engine.getPersistedSymbolDict(); - Assert.assertNotNull("recovery must open the torn subset dict", pd); - Assert.assertEquals("prefix seed must malloc a 2-entry mirror", 2, pd.size()); - Assert.assertTrue("the frame-seed path must run (frames out-reach the dict)", - engine.recoveredMaxSymbolDeltaStart() > 0L); + assertMemoryLeak(() -> { + try (CursorSendEngine engine = new CursorSendEngine(slot.toString(), 4096)) { + PersistedSymbolDict pd = engine.getPersistedSymbolDict(); + Assert.assertNotNull("recovery must open the torn subset dict", pd); + Assert.assertEquals("prefix seed must malloc a 2-entry mirror", 2, pd.size()); + Assert.assertTrue("the frame-seed path must run (frames out-reach the dict)", + engine.recoveredMaxSymbolDeltaStart() > 0L); - setMirrorSeedFault(true); - try { - new CursorWebSocketSendLoop( - null, engine, 0, 1_000_000L, - () -> { - throw new IOException("no reconnect in this test"); - }, - 0, 0, 1); - Assert.fail("ctor must propagate the injected mirror-seed failure"); - } catch (LineSenderException expected) { - Assert.assertTrue("unexpected message: " + expected.getMessage(), - expected.getMessage().contains("simulated mirror seed allocation failure")); - } finally { - setMirrorSeedFault(false); - } - Assert.assertTrue("failed foreground construction must leave the persisted prefix owned", - pd.loadedEntriesAddr() != 0L); - Assert.assertTrue("failed construction must retain the recovered suffix for retry", - engine.recoverySymbolNativeCapacity() > 0); - // The outer assertMemoryLeak proves the prefix-seeded mirror the ctor - // malloc'd was freed on the throw -- pre-fix it leaks here. + setMirrorSeedFault(true); + try { + new CursorWebSocketSendLoop( + null, engine, 0, 1_000_000L, + () -> { + throw new IOException("no reconnect in this test"); + }, + 0, 0, 1); + Assert.fail("ctor must propagate the injected mirror-seed failure"); + } catch (LineSenderException expected) { + Assert.assertTrue("unexpected message: " + expected.getMessage(), + expected.getMessage().contains("simulated mirror seed allocation failure")); + } finally { + setMirrorSeedFault(false); } - }); - } finally { - rmDir(sfDir); - } + Assert.assertTrue("failed foreground construction must leave the persisted prefix owned", + pd.loadedEntriesAddr() != 0L); + Assert.assertTrue("failed construction must retain the recovered suffix for retry", + engine.recoverySymbolNativeCapacity() > 0); + // The outer assertMemoryLeak proves the prefix-seeded mirror the ctor + // malloc'd was freed on the throw -- pre-fix it leaks here. + } + }); } @Test public void testForegroundLoopReleasesRecoveredSuffixAfterSeeding() throws Exception { - Path sfDir = Files.createTempDirectory("qwp-mirror-suffix-release"); - try { - populateThreeFrameSlot(sfDir); - Path slot = sfDir.resolve("default"); - replacePersistedDictionaryWithTwoSymbolPrefix(slot); + Path sfDir = temporaryFolder.newFolder("qwp-mirror-suffix-release").toPath(); + populateThreeFrameSlot(sfDir); + Path slot = sfDir.resolve("default"); + replacePersistedDictionaryWithTwoSymbolPrefix(slot); - assertMemoryLeak(() -> { - try (CursorSendEngine engine = new CursorSendEngine(slot.toString(), 4096)) { - Assert.assertTrue("recovery must retain c@2 above the persisted [a,b] prefix", - engine.recoverySymbolNativeCapacity() > 0); - CursorWebSocketSendLoop loop = new CursorWebSocketSendLoop( - null, engine, 0, 1_000_000L, - () -> { - throw new IOException("no reconnect in this test"); - }, - 0, 0, 1); - try { - Assert.assertEquals("foreground mirror must include prefix and recovered suffix", - 3, readInt(loop, "sentDictCount")); - Assert.assertEquals("engine must release its duplicate recovery suffix", - 0, engine.recoverySymbolNativeCapacity()); - } finally { - loop.close(); - } + assertMemoryLeak(() -> { + try (CursorSendEngine engine = new CursorSendEngine(slot.toString(), 4096)) { + Assert.assertTrue("recovery must retain c@2 above the persisted [a,b] prefix", + engine.recoverySymbolNativeCapacity() > 0); + CursorWebSocketSendLoop loop = new CursorWebSocketSendLoop( + null, engine, 0, 1_000_000L, + () -> { + throw new IOException("no reconnect in this test"); + }, + 0, 0, 1); + try { + Assert.assertEquals("foreground mirror must include prefix and recovered suffix", + 3, readInt(loop, "sentDictCount")); + Assert.assertEquals("engine must release its duplicate recovery suffix", + 0, engine.recoverySymbolNativeCapacity()); + } finally { + loop.close(); } - }); - } finally { - rmDir(sfDir); - } + } + }); } @Test public void testOrphanLoopsRetainRecoveredSuffixForRecycle() throws Exception { - Path sfDir = Files.createTempDirectory("qwp-mirror-suffix-recycle"); - try { - populateThreeFrameSlot(sfDir); - Path slot = sfDir.resolve("default"); - replacePersistedDictionaryWithTwoSymbolPrefix(slot); + Path sfDir = temporaryFolder.newFolder("qwp-mirror-suffix-recycle").toPath(); + populateThreeFrameSlot(sfDir); + Path slot = sfDir.resolve("default"); + replacePersistedDictionaryWithTwoSymbolPrefix(slot); - assertMemoryLeak(() -> { - try (CursorSendEngine engine = new CursorSendEngine(slot.toString(), 4096)) { - int suffixCapacity = engine.recoverySymbolNativeCapacity(); - Assert.assertTrue("recovery must retain c@2 above the persisted [a,b] prefix", - suffixCapacity > 0); - for (int session = 0; session < 2; session++) { - CursorWebSocketSendLoop loop = newRecoveryLoop(engine); - try { - Assert.assertEquals("orphan mirror must include prefix and recovered suffix", - 3, readInt(loop, "sentDictCount")); - Assert.assertEquals("orphan engine must retain suffix for the next session", - suffixCapacity, engine.recoverySymbolNativeCapacity()); - } finally { - loop.close(); - } + assertMemoryLeak(() -> { + try (CursorSendEngine engine = new CursorSendEngine(slot.toString(), 4096)) { + int suffixCapacity = engine.recoverySymbolNativeCapacity(); + Assert.assertTrue("recovery must retain c@2 above the persisted [a,b] prefix", + suffixCapacity > 0); + for (int session = 0; session < 2; session++) { + CursorWebSocketSendLoop loop = newRecoveryLoop(engine); + try { + Assert.assertEquals("orphan mirror must include prefix and recovered suffix", + 3, readInt(loop, "sentDictCount")); + Assert.assertEquals("orphan engine must retain suffix for the next session", + suffixCapacity, engine.recoverySymbolNativeCapacity()); + } finally { + loop.close(); } } - }); - } finally { - rmDir(sfDir); - } + } + }); } // Constructs a recovery send loop but does NOT start it: the ctor seeds the @@ -385,10 +369,6 @@ private static void setMirrorSeedFault(boolean value) throws Exception { f.setBoolean(null, value); } - private static void rmDir(Path dir) throws IOException { - TestUtils.removeTmpDirRec(dir == null ? null : dir.toString()); - } - private static class SilentHandler implements TestWebSocketServer.WebSocketServerHandler { @Override public void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] data) { From 9cc550f75a9a99e1d24d218ec5e60fdc488cba05 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Fri, 17 Jul 2026 14:28:09 +0100 Subject: [PATCH 86/96] Surface startup endpoint-policy failures endpointPolicyFailureIsTerminal() latched a terminal only for an ORPHAN drainer, so a FOREGROUND sender retried auth, non-421 upgrade and durable-ack capability failures forever -- including before it had ever reached the server. An operator whose credentials are wrong then watched a mute sender buffer into store-and-forward until it filled and misreported the cause as "out of space". Two pre-existing tests pin that startup contract and went red: QuestDBFacadeCallbacksTest#testFacadeErrorHandlerReceivesAsyncIngestError asserts the errorHandler receives the async auth terminal, and QuestDBFacadeDrainerListenerTest asserts close() rethrows the foreground's latched capability-gap terminal. Key the terminal off hasEverConnected as well. Connectivity problems are the caller's problem during startup: SYNC/OFF startup reports them by throwing from build(); ASYNC has no caller left to throw at, so the latched terminal reaches the user through SenderErrorHandler and the close() rethrow. The constructor already seeds hasEverConnected = client != null, so SYNC/OFF arrives past initialization and never latches, and swapClient makes the flag sticky: once the wire has been up, store-and-forward owns the buffered data and every endpoint-policy failure is a transient to ride out (Invariant B), unchanged. A transport failure (a dead port) never consulted this predicate and still retries forever. Rewrite the three tests that asserted the opposite so they pin the restored contract, and cross-link testAsyncNoServerRetriesForeverNoTerminal as its transport-half twin. Reverting the predicate fails both pre-existing tests plus the two rewritten async-initial ones, while the post-start tests stay green. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../sf/cursor/CursorWebSocketSendLoop.java | 30 +++++++++- .../qwp/client/InitialConnectAsyncTest.java | 43 +++++++++----- ...SendLoopForegroundReconnectPolicyTest.java | 58 ++++++++++++------- 3 files changed, 93 insertions(+), 38 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java index 410853a8..3e9738bf 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java @@ -1772,8 +1772,31 @@ private void connectLoop(Throwable initial, String phase, long paceFirstAttemptM phase, elapsedMs, attempts, lastMsg); } + /** + * Whether an endpoint-policy failure (auth, non-421 upgrade, durable-ack + * capability gap) latches a terminal instead of retrying under Invariant B. + *

      + * Two callers own a terminal, for different reasons: + *

        + *
      • An ORPHAN drainer: a sanctioned terminal that hands the slot back to + * its quarantine owner.
        • + *
      • A sender still INITIALIZING ({@code !hasEverConnected}): connectivity + * problems are the caller's problem during startup and must reach it, so + * an operator learns their credentials are wrong instead of watching a + * silent sender buffer forever. SYNC/OFF startup reports them by throwing + * from {@code build()}; ASYNC startup has no caller left to throw at, so + * the latched terminal reaches the user through {@code SenderErrorHandler} + * and a {@code close()} rethrow. The constructor seeds + * {@code hasEverConnected = client != null}, so SYNC/OFF (which is handed a + * live client) is already past initialization here and never latches.
        • + *
      + * Once a FOREGROUND sender has reached the server even once, initialization is + * over and store-and-forward owns the buffered data: every endpoint-policy + * failure is then a transient the drainer must ride out, never a producer-fatal + * terminal (Invariant B). + */ private boolean endpointPolicyFailureIsTerminal() { - return reconnectPolicy == ReconnectPolicy.ORPHAN; + return reconnectPolicy == ReconnectPolicy.ORPHAN || !hasEverConnected; } /** @@ -2287,7 +2310,10 @@ private void swapClient(WebSocketClient newClient) { this.client = newClient; // Sticky: once the wire is up, we've reached the server at least once // for this sender's lifetime. Exposed to the owning sender for - // connection-state observability; reconnect policy does not depend on it. + // connection-state observability, and it ends initialization: from here + // on endpointPolicyFailureIsTerminal() stops latching endpoint-policy + // failures on a foreground sender and rides them out instead, because + // store-and-forward now owns the buffered data (Invariant B). this.hasEverConnected = true; if (old != null) { try { diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/InitialConnectAsyncTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/InitialConnectAsyncTest.java index f7fd0c63..02d69efe 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/InitialConnectAsyncTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/InitialConnectAsyncTest.java @@ -58,10 +58,21 @@ public class InitialConnectAsyncTest { @Test - public void testAsyncAuthFailureRetriesForeverNoTerminal() throws Exception { - // Server returns HTTP 401 on every upgrade attempt. Once ASYNC build() - // has returned, store-and-forward owns the buffered rows: the I/O thread - // must keep retrying rather than terminalizing the producer. + public void testAsyncAuthFailureSurfacesTerminal() throws Exception { + // Server returns HTTP 401 on every upgrade attempt. A rejection by ENDPOINT + // POLICY before the sender has ever reached the server is a startup problem, + // not a transient, so it must reach the caller: an operator with the wrong + // credentials has to learn that, rather than watch a mute sender buffer into + // SF until it fills and misreports the cause as "out of space". SYNC/OFF + // startup reports it by throwing from build(); ASYNC has no caller left to + // throw at, so it arrives on the SenderErrorHandler instead. + // + // Contrast testAsyncNoServerRetriesForeverNoTerminal: a dead port is a + // TRANSPORT failure -- genuinely transient -- and retries forever even during + // startup. And once the wire has been up even once, initialization is over + // and store-and-forward owns the data, so the same 401 becomes a transient to + // ride out (CursorWebSocketSendLoopForegroundReconnectPolicyTest + // #testPostStartAuthFailureRetriesUntilCredentialsRecover). try (Always401Fixture fixture = new Always401Fixture()) { fixture.start(); int port = fixture.getPort(); @@ -79,16 +90,17 @@ public void testAsyncAuthFailureRetriesForeverNoTerminal() throws Exception { QwpWebSocketSender wss = (QwpWebSocketSender) sender; awaitAtLeastOneConnectAttempt(wss); - sender.table("foo").longColumn("v", 1L).atNow(); - sender.flush(); - - Assert.assertFalse( - "async 401 rejection must stay inside the retry loop", - inbox.await(1_500, TimeUnit.MILLISECONDS)); - Assert.assertNull("no terminal SenderError may be delivered for an async 401", - inbox.get()); + Assert.assertTrue( + "an async 401 must surface a terminal to the errorHandler", + inbox.await(5, TimeUnit.SECONDS)); + SenderError err = inbox.get(); + Assert.assertNotNull("a SenderError must be delivered for an async 401", err); + Assert.assertEquals(SenderError.Policy.TERMINAL, err.getAppliedPolicy()); + Assert.assertEquals(SenderError.Category.SECURITY_ERROR, err.getCategory()); + Assert.assertTrue("the terminal must name the upgrade rejection, got: " + + err.getServerMessage(), + err.getServerMessage().contains("ws-upgrade-failed")); Assert.assertFalse("no upgrade has succeeded yet", wss.wasEverConnected()); - awaitReconnectAttemptsAdvance(wss); } finally { closeQuietly(sender); } @@ -102,8 +114,9 @@ public void testAsyncNoServerRetriesForeverNoTerminal() throws Exception { // (it may appear; the data is safe in SF), so the I/O thread retries // forever. reconnect_max_duration_millis is IGNORED as a give-up deadline: // no SenderError lands, the sender stays usable, and wasEverConnected() - // stays false. Endpoint-policy failures are covered by - // testAsyncAuthFailureRetriesForeverNoTerminal. + // stays false. This is the TRANSPORT half of the startup contract; the + // endpoint-POLICY half, which does surface, is + // testAsyncAuthFailureSurfacesTerminal. int port = TestPorts.findUnusedPort(); ErrorInbox inbox = new ErrorInbox(); String cfg = "ws::addr=localhost:" + port diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopForegroundReconnectPolicyTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopForegroundReconnectPolicyTest.java index 73e3b634..cafd6a54 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopForegroundReconnectPolicyTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopForegroundReconnectPolicyTest.java @@ -55,15 +55,17 @@ public class CursorWebSocketSendLoopForegroundReconnectPolicyTest { public final TemporaryFolder sfDir = TemporaryFolder.builder().assureDeletion().build(); @Test - public void testAsyncInitialAuthFailureRetriesUntilCredentialsRecover() throws Exception { - assertAsyncInitialForegroundRecovers(false, - () -> new QwpAuthFailedException(401, "localhost", 1)); + public void testAsyncInitialAuthFailureSurfacesTerminalToTheCaller() throws Exception { + assertAsyncInitialForegroundSurfacesTerminal(false, + () -> new QwpAuthFailedException(401, "localhost", 1), + "ws-upgrade-failed"); } @Test - public void testAsyncInitialDurableAckMismatchRetriesUntilCapabilityRecovers() throws Exception { - assertAsyncInitialForegroundRecovers(true, - () -> new QwpDurableAckMismatchException("localhost", 1, "primary")); + public void testAsyncInitialDurableAckMismatchSurfacesTerminalToTheCaller() throws Exception { + assertAsyncInitialForegroundSurfacesTerminal(true, + () -> new QwpDurableAckMismatchException("localhost", 1, "primary"), + "durable-ack-mismatch"); } @Test @@ -78,9 +80,18 @@ public void testPostStartDurableAckMismatchRetriesUntilCapabilityRecovers() thro () -> new QwpDurableAckMismatchException("localhost", 1, "primary")); } - private void assertAsyncInitialForegroundRecovers( + /** + * An endpoint-policy failure before the sender has EVER reached the server is a + * startup problem, so it must reach the caller rather than retry silently: an + * operator with wrong credentials has to learn that, not watch a mute sender + * buffer forever. ASYNC startup has no caller left to throw at, so the terminal + * is latched for {@code SenderErrorHandler} delivery and the {@code close()} + * rethrow. Post-start the opposite holds -- see {@link #assertForegroundRecovers}. + */ + private void assertAsyncInitialForegroundSurfacesTerminal( boolean durableAck, - FailureSupplier failureSupplier + FailureSupplier failureSupplier, + String expectedDetail ) throws Exception { TestUtils.assertMemoryLeak(() -> { DropFirstConnectionHandler handler = new DropFirstConnectionHandler(durableAck, false); @@ -111,20 +122,18 @@ private void assertAsyncInitialForegroundRecovers( appendFrame(engine, (byte) 1); loop.start(); - await(() -> factory.attempts() >= 2 || loop.getTerminalError() != null, - "async foreground did not retry the endpoint-policy failure"); - Assert.assertNull( - "async initial endpoint-policy failures must not stop the producer", - loop.getTerminalError()); + await(() -> loop.getTerminalError() != null, + "an async initial endpoint-policy failure must latch a terminal"); + Assert.assertTrue("the terminal must name the endpoint-policy failure, got: " + + loop.getTerminalError().getMessage(), + loop.getTerminalError().getMessage().contains(expectedDetail)); + // Never reached the server, so the terminal is a startup verdict. Assert.assertFalse(loop.hasEverConnected()); - - long target = appendFrame(engine, (byte) 2); - factory.allowConnect(); - await(() -> engine.ackedFsn() >= target || loop.getTerminalError() != null, - "async foreground did not deliver after endpoint policy recovered"); - Assert.assertNull(loop.getTerminalError()); - Assert.assertTrue(loop.hasEverConnected()); - Assert.assertEquals(target, engine.ackedFsn()); + // Latched on the FIRST rejection: retrying an endpoint-policy + // failure at startup is exactly the silent-buffering regression + // this pins. Attempts cannot climb once running flips false. + Assert.assertEquals("a startup endpoint-policy failure must not be retried", + 1, factory.attempts()); } finally { factory.allowConnect(); loop.close(); @@ -133,6 +142,13 @@ private void assertAsyncInitialForegroundRecovers( }); } + /** + * The post-start twin of {@link #assertAsyncInitialForegroundSurfacesTerminal}: + * handing the loop a live client seeds {@code hasEverConnected}, so the sender is + * past initialization and store-and-forward owns the buffered data. Every + * endpoint-policy failure is then a transient to ride out, never a terminal + * (Invariant B). + */ private void assertForegroundRecovers(boolean durableAck, FailureSupplier failureSupplier) throws Exception { TestUtils.assertMemoryLeak(() -> { DropFirstConnectionHandler handler = new DropFirstConnectionHandler(durableAck, true); From 68aa31a5798b11b32ccffd947e77fd6d16246118 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Fri, 17 Jul 2026 14:28:26 +0100 Subject: [PATCH 87/96] Pin the quarantine test on the rename failure testQuarantineRenameFailurePreservesOriginalSlot counted the slot's .sfa files BEFORE build(), then asserted the count was unchanged after the failed rename. Recovery legitimately unlinks an empty never-rotated hot-spare on the way in (SegmentRing drops a recovered segment whose frameCount() is 0 with no torn tail), and whether SegmentManager had provisioned one by then is a race -- so the assertion measured that race rather than the rename. It failed 3 of 3 standalone runs, and inverted (2 vs 3) under load. Snapshot inside the quarantineAfterCloseHook instead: after the engine closed, before the rename. That is exactly what "a failed rename preserves the slot" means, and it is immune to the hot-spare race because both counts observe the post-recovery state. Assert the snapshot is non-zero as well, so the equality cannot pass vacuously on an empty slot. The product was never at fault: the reclaimed segment holds no frames, and close() unlinks nothing -- instrumenting quarantineTornSlot shows the same segments before and after it. Mutating that method to delete a segment during the failed rename still fails the test (expected 2, was 1), so it continues to catch real destruction. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../qwp/client/DeltaDictRecoveryTest.java | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java index 6b7f2201..1845e7bb 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java @@ -471,7 +471,16 @@ public void testQuarantineRenameFailurePreservesOriginalSlot() throws Exception assertMemoryLeak(() -> { writeAndTearUnreplayableSlot(); java.nio.file.Path slot = Paths.get(sfDir, "default"); - int segmentCount = countSegmentFiles(slot); + // Snapshot when quarantine STARTS (after the engine closed, before the + // rename), not before build(). Recovery legitimately unlinks an empty + // never-rotated hot-spare on the way in (SegmentRing: frameCount()==0 and + // no torn tail), and whether SegmentManager provisioned one is a race -- + // so a pre-build count measures that race, not this test's subject. What + // the rename failure must preserve is whatever the slot holds at the + // moment quarantine begins. + AtomicInteger atQuarantineStart = new AtomicInteger(-1); + Sender.LineSenderBuilder.setQuarantineAfterCloseHookForTest( + () -> atQuarantineStart.set(countSegmentFiles(slot))); Sender.LineSenderBuilder.setQuarantineFilesFacadeForTest(new DelegatingFilesFacade() { @Override public int rename(String oldPath, String newPath) { @@ -503,11 +512,16 @@ public int rename(String oldPath, String newPath) { } } finally { Sender.LineSenderBuilder.setQuarantineFilesFacadeForTest(null); + Sender.LineSenderBuilder.setQuarantineAfterCloseHookForTest(null); } Assert.assertTrue("rename failure must preserve the original slot directory", java.nio.file.Files.isDirectory(slot)); + // Guards the equality below against passing vacuously on an empty slot: + // 0 == 0 would "preserve every segment" while holding nothing. + Assert.assertTrue("quarantine must have started with data-bearing segments, saw: " + + atQuarantineStart.get(), atQuarantineStart.get() > 0); Assert.assertEquals("rename failure must preserve every segment", - segmentCount, countSegmentFiles(slot)); + atQuarantineStart.get(), countSegmentFiles(slot)); Assert.assertFalse("failed rename must not leave a quarantine directory", java.nio.file.Files.exists(Paths.get(sfDir, "default.unreplayable-0"))); }); From 5ded812da18f35de7bdcfe11dcb441705b337019 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Fri, 17 Jul 2026 14:47:19 +0100 Subject: [PATCH 88/96] Cut symbol-dict overhead and fix two blind guards Five review findings on the delta symbol dictionary. The three performance ones all sit on paths that scale with symbol cardinality -- the workload this feature exists for introduces a new symbol per ROW -- and the two guards were pinning nothing. RecoveredFrameAnalysis.foldDelta issued one bounds check, one capacity check and a ~12-byte copyMemory per SYMBOL. runningCoverage is loop-invariant (it only advances after the walk) and id ascends, so "id >= runningCoverage" is a step predicate: the entries a frame contributes are always ONE contiguous run at the tail of its delta section. Note where the run starts and copy it once. Recovery walks the whole backlog, so this traded millions of stub-dispatched small copies for one bulk copy per frame. It also drops the partial prefix the per-entry version left behind on a markGap bail-out; that residue was already unreachable (a gap pins coverage() at -1, and a later self-sufficient frame resets the buffer), so not writing it is equivalent and leaves less state to reason about. accumulateSentDict already solved the same problem this way one file over. Crc32c.updateUnsafe read each 8-byte slice-by-8 block with eight getByte calls: 16 loads per block where 9 do. Consume it with two getInt loads. getInt is an Unsafe intrinsic exactly as getByte is, so the catchable InternalError this method exists for is unaffected; the loop becomes little-endian, which costs nothing because the formats it checksums are already little-endian throughout. This is the whole recovery CRC. The differential test against the native intrinsic (8 alignments x 28 lengths x 3 seeds, plus the chaining fuzz) proves it bit-identical. The sent-dict entry-ends index was maintained incrementally by accumulateSentDict and built eagerly by the constructor, yet sendDictCatchUp is its ONLY reader. That put a store per new symbol on the per-frame I/O path and retained 4 bytes per symbol for the connection's whole life, to serve a reconnect that may never come. Build it lazily instead: ensureSentDictEntryIndex() already notices sentDictIndexedCount falling behind and rebuilds, at most once per reconnect, dwarfed by shipping the same bytes over the wire. A connection that never reconnects no longer allocates the index at all. testCtorFreesSeededMirrorWhenFrameSeedThrows could not fail. The seam threw BEFORE ensureSentDictCapacity -- the call that copy-on-writes the borrowed prefix into a loop-OWNED allocation -- so the cleanup it guards, gated on sentDictBytesOwned, freed nothing. Deleting that cleanup left the test green. Move the seam past the grow: deleting the cleanup now fails with a 4096-byte NATIVE_DEFAULT leak. The test's premise comment described the old copy-based design and is corrected. testPersistFailureSurfacesAsLineSenderException drove code production never runs. FilesFacade.isMmapAllowed() defaults to (this == INSTANCE), so installing any fault facade silently swapped the dictionary onto the positioned-write fallback -- the act of injecting the fault replaced the code under test. Fault a refused ff.allocate on the mmap append window instead, with the facade opting into isMmapAllowed(): that is both the real shape of ENOSPC here and the path production takes. Reverting the wrap now escapes as "could not grow mmap append region", from the mapped path. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../qwp/client/QwpWebSocketSender.java | 19 +++---- .../sf/cursor/CursorWebSocketSendLoop.java | 50 ++++++++++++------- .../sf/cursor/RecoveredFrameAnalysis.java | 41 ++++++++++++--- .../java/io/questdb/client/std/Crc32c.java | 36 +++++++------ .../qwp/client/DeltaDictRecoveryTest.java | 47 +++++++++++------ ...WebSocketSendLoopCatchUpAlignmentTest.java | 15 +++--- ...CursorWebSocketSendLoopMirrorLeakTest.java | 25 ++++++---- 7 files changed, 155 insertions(+), 78 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index 2b1971e4..f06ea87e 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -3905,15 +3905,16 @@ private void persistNewSymbolsBeforePublish() { pd.appendSymbols(globalSymbolDictionary, from, currentBatchMaxSymbolId); } } catch (Throwable t) { - // A short write (disk full / quota) to the persisted dictionary throws - // a low-level IllegalStateException. Surface it as a LineSenderException - // -- like every other flush-path failure, e.g. the cursor append in - // sealAndSwapBuffer -- so a caller catching LineSenderException around - // flush() also catches a disk-full during the write-ahead persist. The - // persist ran before publish and pd.size() did not advance on the short - // write, so the still-buffered rows re-persist the same range - // idempotently on retry. A JVM Error is never a persist failure; let it - // propagate. + // A failed write to the persisted dictionary throws a low-level + // IllegalStateException: in production that is ff.allocate refusing to grow + // the mmap append window (how a full disk / exhausted quota surfaces there), + // and behind an injected facade a short positioned write. Surface it as a + // LineSenderException -- like every other flush-path failure, e.g. the cursor + // append in sealAndSwapBuffer -- so a caller catching LineSenderException + // around flush() also catches a disk-full during the write-ahead persist. The + // persist ran before publish and pd.size() did not advance on the failure, so + // the still-buffered rows re-persist the same range idempotently on retry. + // A JVM Error is never a persist failure; let it propagate. if (t instanceof Error) { throw (Error) t; } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java index 3e9738bf..5619d1e9 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java @@ -197,12 +197,14 @@ public final class CursorWebSocketSendLoop implements QuietCloseable { * Throttle "reconnect attempt N failed" WARN logs to one per 5 s. */ private static final long RECONNECT_LOG_THROTTLE_NANOS = 5_000_000_000L; - // Test seam: when true, recovery mirror seeding throws before growing, - // simulating the native realloc OOM (or MAX_SENT_DICT_BYTES ceiling) that - // ensureSentDictCapacity can raise while the constructor seeds the recovery - // mirror. Lets a test prove the constructor frees the already-malloc'd mirror on - // such a throw rather than leaking it. Production never sets it; volatile only so - // a test thread's write is visible to the loop under test. + // Test seam: when true, recovery mirror seeding throws immediately AFTER + // ensureSentDictCapacity has grown (and therefore taken ownership of) the mirror, + // standing in for the copyRecoveredSymbolSuffix-adjacent failure that leaves a + // freshly malloc'd mirror with no owner. It sits after the grow deliberately: the + // constructor's cleanup only frees an OWNED mirror, so a seam before the grow + // would leave nothing to free and the leak guard would pass with the cleanup + // deleted. Production never sets it; volatile only so a test thread's write is + // visible to the loop under test. @TestOnly static volatile boolean forceMirrorSeedFailureForTest; // Pre-converted to nanos for the comparison in sendDurableAckKeepaliveIfDue. @@ -784,11 +786,19 @@ public CursorWebSocketSendLoop(WebSocketClient client, CursorSendEngine engine, int suffixLen = engine.recoveredSymbolSuffixLen(baseline); int suffixCount = engine.recoveredSymbolSuffixCount(baseline); if (suffixLen > 0) { + ensureSentDictCapacity((long) sentDictBytesLen + suffixLen); if (forceMirrorSeedFailureForTest) { + // Throw AFTER the grow, never before it. ensureSentDictCapacity + // is what copy-on-writes a borrowed prefix into a loop-OWNED + // allocation (sentDictBytesOwned = true), and the catch's + // releaseSentDictBytes() is gated on exactly that flag. A seam + // in front of the grow fires while the mirror is still borrowed + // from PersistedSymbolDict, so the cleanup frees nothing and the + // guard below passes even with the cleanup deleted -- it would + // stop pinning the leak it exists for. throw new LineSenderException( "simulated mirror seed allocation failure (test only)"); } - ensureSentDictCapacity((long) sentDictBytesLen + suffixLen); engine.copyRecoveredSymbolSuffix( baseline, sentDictBytesAddr + sentDictBytesLen); sentDictBytesLen += suffixLen; @@ -796,9 +806,10 @@ public CursorWebSocketSendLoop(WebSocketClient client, CursorSendEngine engine, } } } - if (sentDictCount > 0) { - rebuildSentDictEntryIndex(); - } + // The entry-ends index is NOT built here. sendDictCatchUp is its only + // reader and builds it on demand, so a recovered slot that drains without + // ever reconnecting -- the normal case -- never pays the O(n) walk nor + // retains the 4-bytes-per-symbol index. } catch (Throwable t) { releaseSentDictBytes(); throw t; @@ -2446,7 +2457,16 @@ private void accumulateSentDict(long payloadAddr, int payloadLen, int deltaStart if (deltaCount <= 0 || deltaStart > sentDictCount || deltaEnd <= sentDictCount) { return; } - ensureSentDictEntryIndex(); + // Deliberately does NOT touch the entry-ends index. Only sendDictCatchUp reads + // it, so maintaining it here would put a store per new symbol on the per-frame + // I/O path -- and on the workload this feature targets (a new symbol per ROW) + // that is a store per row, plus 4 bytes per symbol retained for the connection's + // whole life, to serve a reconnect that may never happen. sendDictCatchUp calls + // ensureSentDictEntryIndex(), which notices sentDictIndexedCount has fallen + // behind sentDictCount and rebuilds; that O(n) walk runs at most once per + // reconnect and is dwarfed by shipping the same n bytes over the wire. A + // connection that never reconnects now never allocates the index at all. + // // Walk past the already-held prefix [deltaStart, sentDictCount), then copy // the new tail [sentDictCount, deltaEnd). int skip = sentDictCount - deltaStart; @@ -2459,8 +2479,8 @@ private void accumulateSentDict(long payloadAddr, int payloadLen, int deltaStart } long regionStart = p; long newCount = deltaEnd - sentDictCount; - long requiredEntryCount = (long) sentDictCount + newCount; - ensureSentDictEntryEndsCapacity(requiredEntryCount); + // Walk the new tail only to find where it ends; the entry boundaries are not + // recorded here (see above -- the catch-up rebuilds them on demand). for (long i = 0; i < newCount; i++) { long len = readVarintAt(p, limit); p = varintEnd + len; @@ -2469,9 +2489,6 @@ private void accumulateSentDict(long payloadAddr, int payloadLen, int deltaStart // than corrupt the mirror. return; } - Unsafe.getUnsafe().putInt( - sentDictEntryEndsAddr + ((long) sentDictCount + i) * Integer.BYTES, - sentDictBytesLen + (int) (p - regionStart)); } int regionBytes = (int) (p - regionStart); // long sum: sentDictBytesLen + regionBytes can exceed Integer.MAX_VALUE on @@ -2482,7 +2499,6 @@ private void accumulateSentDict(long payloadAddr, int payloadLen, int deltaStart Unsafe.getUnsafe().copyMemory(regionStart, sentDictBytesAddr + sentDictBytesLen, regionBytes); sentDictBytesLen += regionBytes; sentDictCount += (int) newCount; - sentDictIndexedCount = sentDictCount; } private void ensureSentDictCapacity(long required) { diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/RecoveredFrameAnalysis.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/RecoveredFrameAnalysis.java index ed709d16..25d2d431 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/RecoveredFrameAnalysis.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/RecoveredFrameAnalysis.java @@ -221,8 +221,14 @@ public void close() { releaseRawStorage(); } - private void appendRaw(long entryStart, int entryLen) { - long required = (long) runningRawLen + entryLen; + /** + * Appends one contiguous run of {@code count} wire entries -- {@code [len][utf8]} + * repeated, exactly as a delta section carries them -- in a single copy. Callers + * pass a whole frame's new-symbol suffix at once; see {@link #foldDelta} for why + * that suffix is always contiguous. + */ + private void appendRaw(long addr, int len, int count) { + long required = (long) runningRawLen + len; if (required > MAX_RAW_BYTES) { throw new IllegalStateException("recovered symbol dictionary suffix exceeds maximum size " + "[required=" + required + ", max=" + MAX_RAW_BYTES + ']'); @@ -235,9 +241,9 @@ private void appendRaw(long entryStart, int entryLen) { rawAddr = Unsafe.realloc(rawAddr, rawCapacity, (int) newCapacity, MemoryTag.NATIVE_DEFAULT); rawCapacity = (int) newCapacity; } - Unsafe.getUnsafe().copyMemory(entryStart, rawAddr + runningRawLen, entryLen); - runningRawLen += entryLen; - runningRawCount++; + Unsafe.getUnsafe().copyMemory(addr, rawAddr + runningRawLen, len); + runningRawLen += len; + runningRawCount += count; } private void foldDelta(long fsn, long p, long limit) { @@ -292,7 +298,24 @@ private void foldDelta(long fsn, long p, long limit) { return; } + // runningCoverage is loop-invariant here -- it only advances after the walk -- + // and id ascends from deltaStart, so `id >= runningCoverage` is a step + // predicate: the entries this frame contributes are always ONE contiguous run + // at the tail of its delta section. Note where that run starts and copy it in a + // single memcpy once the walk succeeds, rather than paying a bound check, a + // capacity check and a ~12-byte copyMemory per symbol. Recovery walks the whole + // backlog, and the workload this feature exists for introduces a new symbol per + // ROW, so that is millions of stub-dispatched small copies where one bulk copy + // per frame does the job. + // + // Deferring the copy past the markGap bail-outs also drops the partial prefix + // the per-entry version used to leave behind. That residue was already + // unreachable -- a gap pins coverage() at -1, and a later self-sufficient frame + // resets runningRawLen/runningRawCount -- so not writing it is equivalent, and + // leaves less state to reason about. long id = deltaStart; + long suffixStart = 0L; + int suffixCount = 0; for (long i = 0; i < deltaCount; i++, id++) { symbolEntriesVisited++; long entryStart = p; @@ -310,9 +333,15 @@ private void foldDelta(long fsn, long p, long limit) { } p += symbolLen; if (id >= runningCoverage) { - appendRaw(entryStart, (int) (p - entryStart)); + if (suffixCount == 0) { + suffixStart = entryStart; + } + suffixCount++; } } + if (suffixCount > 0) { + appendRaw(suffixStart, (int) (p - suffixStart), suffixCount); + } if (deltaEnd > runningCoverage) { runningCoverage = deltaEnd; } diff --git a/core/src/main/java/io/questdb/client/std/Crc32c.java b/core/src/main/java/io/questdb/client/std/Crc32c.java index 3f05ee84..738efb1b 100644 --- a/core/src/main/java/io/questdb/client/std/Crc32c.java +++ b/core/src/main/java/io/questdb/client/std/Crc32c.java @@ -73,6 +73,14 @@ private Crc32c() { * mmap access fault into a catchable {@link InternalError}; the native * {@link #update} path cannot provide that guarantee because a SIGBUS raised * inside JNI aborts the JVM. + *

      + * The hot loop consumes each 8-byte block with two {@code getInt}s rather than + * eight {@code getByte}s -- 9 loads per block instead of 16, for the same table + * work. {@code getInt} is an {@link Unsafe} intrinsic exactly as {@code getByte} + * is, so the fault-catchability above is unaffected. It does make the loop + * little-endian, which costs nothing here: the segment and dictionary formats this + * checksums are already little-endian throughout (their headers are read back with + * {@code getInt}), and the native twin this must agree with asserts the same. * * @param seed previous CRC value, or {@link #INIT} to start fresh * @param addr off-heap address of at least {@code len} readable bytes @@ -84,22 +92,18 @@ public static int updateUnsafe(int seed, long addr, long len) { int crc = ~seed; int[] table = CRC32C_TABLE; while (len >= Long.BYTES) { - int b0 = Unsafe.getUnsafe().getByte(addr) & 0xFF; - int b1 = Unsafe.getUnsafe().getByte(addr + 1) & 0xFF; - int b2 = Unsafe.getUnsafe().getByte(addr + 2) & 0xFF; - int b3 = Unsafe.getUnsafe().getByte(addr + 3) & 0xFF; - int b4 = Unsafe.getUnsafe().getByte(addr + 4) & 0xFF; - int b5 = Unsafe.getUnsafe().getByte(addr + 5) & 0xFF; - int b6 = Unsafe.getUnsafe().getByte(addr + 6) & 0xFF; - int b7 = Unsafe.getUnsafe().getByte(addr + 7) & 0xFF; - crc = table[7 * 256 + ((crc ^ b0) & 0xFF)] - ^ table[6 * 256 + (((crc >>> 8) ^ b1) & 0xFF)] - ^ table[5 * 256 + (((crc >>> 16) ^ b2) & 0xFF)] - ^ table[4 * 256 + (((crc >>> 24) ^ b3) & 0xFF)] - ^ table[3 * 256 + b4] - ^ table[2 * 256 + b5] - ^ table[256 + b6] - ^ table[b7]; + // Little-endian: lo holds bytes 0..3 and hi bytes 4..7, ascending from the + // low byte, so the per-byte table lookups below stay in wire order. + int lo = Unsafe.getUnsafe().getInt(addr) ^ crc; + int hi = Unsafe.getUnsafe().getInt(addr + Integer.BYTES); + crc = table[7 * 256 + (lo & 0xFF)] + ^ table[6 * 256 + ((lo >>> 8) & 0xFF)] + ^ table[5 * 256 + ((lo >>> 16) & 0xFF)] + ^ table[4 * 256 + (lo >>> 24)] + ^ table[3 * 256 + (hi & 0xFF)] + ^ table[2 * 256 + ((hi >>> 8) & 0xFF)] + ^ table[256 + ((hi >>> 16) & 0xFF)] + ^ table[hi >>> 24]; addr += Long.BYTES; len -= Long.BYTES; } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java index 1845e7bb..b7daef8f 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java @@ -822,12 +822,20 @@ public void testCommitMessageDoesNotShipUnpersistedLeakedSymbol() throws Excepti public void testPersistFailureSurfacesAsLineSenderException() throws Exception { // A dictionary write that fails mid-flush -- a full disk, an exhausted quota -- // must reach the caller as a LineSenderException, like every other flush-path - // failure. PersistedSymbolDict throws a low-level IllegalStateException on a short - // write; persistNewSymbolsBeforePublish wraps it. Without the wrap the raw - // IllegalStateException sails straight past every user's - // `catch (LineSenderException)` around flush() and takes the application down. + // failure. PersistedSymbolDict throws a low-level IllegalStateException when it + // cannot grow its append window; persistNewSymbolsBeforePublish wraps it. + // Without the wrap the raw IllegalStateException sails straight past every + // user's `catch (LineSenderException)` around flush() and takes the application + // down. // - // Nothing could reach that translation before: PersistedSymbolDict has + // The fault is a REFUSED ff.allocate on the mmap append window, with the facade + // opting into isMmapAllowed(): that is both the real shape of ENOSPC here and + // the path production actually runs. A facade that inherits the default + // isMmapAllowed() (this == INSTANCE -> false) silently swaps the dictionary onto + // the positioned-write fallback, which production never executes -- so a + // short-write fault would prove this translation on dead code. + // + // Nothing could reach the translation at all before: PersistedSymbolDict has // facade-aware overloads, but CursorSendEngine called only the // FilesFacade.INSTANCE ones, so no test could inject a dictionary I/O fault // through the real producer path. The engine now takes a FilesFacade for exactly @@ -842,7 +850,7 @@ public void testPersistFailureSurfacesAsLineSenderException() throws Exception { String slot = Paths.get(sfDir, "default").toString(); Assert.assertEquals(0, io.questdb.client.std.Files.mkdir(sfDir, io.questdb.client.std.Files.DIR_MODE_DEFAULT)); - ShortDictWriteFacade ff = new ShortDictWriteFacade(); + FullDiskDictFacade ff = new FullDiskDictFacade(); // The engine owns the dictionary; the fault facade reaches only it, so the // segment files still write normally and the ONLY failure is the persist. CursorSendEngine engine = new CursorSendEngine( @@ -851,7 +859,7 @@ public void testPersistFailureSurfacesAsLineSenderException() throws Exception { Sender sender = QwpWebSocketSender.connect( "localhost", port, null, 0, 0, 0L, null, false, engine); try { - ff.armed = true; // the next dictionary append short-writes + ff.armed = true; // the next dictionary append cannot grow its window sender.table("m").symbol("s", "boom").longColumn("v", 1L).atNow(); try { sender.flush(); @@ -1553,16 +1561,27 @@ public void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] dat * fills mid-flush. Offset 0 is left alone so the file header still writes -- only the * entry append fails, which is the path under test. */ - private static final class ShortDictWriteFacade extends DelegatingFilesFacade { + /** + * Refuses to grow the symbol dictionary's mmap append window -- the production + * shape of a full disk, since PersistedSymbolDict reserves that window through + * {@code ff.allocate} and ENOSPC surfaces there as a refusal. + *

      + * {@code isMmapAllowed()} opts IN deliberately. The inherited default is + * {@code this == INSTANCE}, so any wrapping facade otherwise routes the dictionary + * down the positioned-write fallback that production never executes -- injecting a + * fault would then silently replace the code under test. + */ + private static final class FullDiskDictFacade extends DelegatingFilesFacade { boolean armed; @Override - public long write(int fd, long addr, long len, long offset) { - if (armed && offset > 0 && len > 1) { - armed = false; - return INSTANCE.write(fd, addr, len - 1, offset); - } - return INSTANCE.write(fd, addr, len, offset); + public boolean allocate(int fd, long size) { + return !armed && INSTANCE.allocate(fd, size); + } + + @Override + public boolean isMmapAllowed() { + return true; } } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java index 00a0f230..ac6dad09 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java @@ -170,8 +170,12 @@ public void testSplitCatchUpReusesEntryIndexAndStagesOnlyPrefixAcrossReconnects( CursorWebSocketSendLoop loop = newLoop(engine, client); try { seedMirror(loop, TestUtils.repeat("x", 3_000), TestUtils.repeat("y", 3_000)); - assertEquals("the mirror is indexed once when it is built", - 1, loop.catchUpEntryIndexBuildCount()); + // Building the mirror must NOT index it: the index serves only the + // catch-up, so paying for it on the per-frame send path -- and + // retaining it -- would tax every connection for a reconnect that + // may never come. + assertEquals("accumulating the mirror must not build the entry index", + 0, loop.catchUpEntryIndexBuildCount()); invokeSetWireBaselineWithCatchUp(loop, 0L); assertEquals("the small cap must split the dictionary", 2, client.framesSent); @@ -972,10 +976,9 @@ private static void seedMirror(CursorWebSocketSendLoop loop, String... symbols) setIntField(loop, "sentDictBytesCapacity", total); setIntField(loop, "sentDictBytesLen", total); setIntField(loop, "sentDictCount", symbols.length); - Method index = CursorWebSocketSendLoop.class.getDeclaredMethod( - "rebuildSentDictEntryIndex"); - index.setAccessible(true); - index.invoke(loop); + // Leave sentDictIndexedCount at 0: the mirror carries its own entry lengths, and + // the entry-ends index is built lazily by the catch-up. Pre-building it here + // would hide that from every test that seeds this way. } private static void setField(Object target, String name, long value) throws Exception { diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopMirrorLeakTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopMirrorLeakTest.java index 63ad5d51..e480dd9f 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopMirrorLeakTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopMirrorLeakTest.java @@ -170,19 +170,24 @@ public void testRecycledLoopReSeedsMirrorFromPersistedDict() throws Exception { @Test public void testCtorFreesSeededMirrorWhenFrameSeedThrows() throws Exception { - // C1 regression: the constructor seeds the recovery mirror in TWO steps -- a - // malloc from the persisted dictionary's intact prefix, then an extension from - // the recovered suffix retained by the engine. If that second step - // throws (a native realloc OOM, or the MAX_SENT_DICT_BYTES ceiling), the throw - // leaves the constructor with the object unpublished, so neither - // ensureConnected's catch nor BackgroundDrainer's finally can ever close() it -- - // and the already-malloc'd prefix mirror leaks. The constructor must free it on - // the throw. Pre-fix, the mirror leaks here and assertMemoryLeak fails. + // C1 regression: the constructor seeds the recovery mirror in TWO steps. It + // first BORROWS the persisted dictionary's intact prefix by reference, then + // extends it with the recovered suffix the engine retains -- and that extension + // calls ensureSentDictCapacity, which copy-on-writes the borrowed prefix into a + // fresh loop-OWNED allocation. A throw just past that grow (a native realloc + // OOM, or the MAX_SENT_DICT_BYTES ceiling) leaves the constructor with the + // object unpublished, so neither ensureConnected's catch nor BackgroundDrainer's + // finally can ever close() it -- and the owned mirror leaks. The constructor + // must free it on the throw. Delete that free and assertMemoryLeak fails here. + // + // The fault therefore has to sit AFTER the grow. Injected before it, the mirror + // is still borrowed (sentDictBytesOwned == false), the cleanup is a no-op, and + // this test would pass with the cleanup deleted. Path sfDir = temporaryFolder.newFolder("qwp-mirror-ctor-throw").toPath(); // A torn-dict SUBSET: three delta frames a@0,b@1,c@2 survive on disk, but the // .symbol-dict is rewritten to hold only [a,b] (a host-crash tail tear). On - // recovery pd.size()==2 seeds (mallocs) the mirror, then the frame-seed - // copies c@2 from the recovered suffix -- the operation the fault interrupts. + // recovery pd.size()==2 seeds the borrowed prefix, then the frame-seed grows the + // mirror to take c@2 from the recovered suffix -- the step the fault interrupts. populateThreeFrameSlot(sfDir); Path slot = sfDir.resolve("default"); replacePersistedDictionaryWithTwoSymbolPrefix(slot); From b279eece1dd5c4c27acd75fcd853757f6039eaf4 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Fri, 17 Jul 2026 16:00:30 +0100 Subject: [PATCH 89/96] Re-fold recovery at baseline 0 when discarding the dictionary The full-dict-fallback branch discards a persisted .symbol-dict whose frames carry their whole dictionary inline (maxSymbolDeltaStart == 0 and recoveredMaxSymbolId >= size). But the recovery analysis was folded with the dictionary's size as its baseline, and every consumer of it presents the baseline it derived the same way: once pd is null, seedGlobalDictionaryFromPersisted computes baseline 0, and checkedRecoveryAnalysis rejects a baseline that disagrees with the fold. Discarding a dictionary that held entries (size > 0) therefore threw IllegalStateException("recovery symbol baseline mismatch") out of build(). That is not an UnreplayableSlotException, so build()'s quarantine handler could not set the slot aside. With a stable senderId every restart re-recovered the same slot and threw again, so the application could never construct a Sender -- it could not even buffer new rows. Exactly the outage quarantineTornSlot exists to prevent, and on a slot that is fully recoverable, since its frames need no dictionary. It takes one transient plus one crash: a session whose .symbol-dict fails to open (EIO, fd exhaustion, a Windows share lock) runs full-dict fallback and, per the never-recreate contract, leaves the previous session's populated side-file intact; if that session then crashes, this recovery opens a dictionary with size > 0 beside self-sufficient frames that out-reach it. The prior regression test covered only the size == 0 case. Re-fold at baseline 0 after the discard so producer, mirror and replay guard all anchor at 0 -- the full-dict mode the slot was written in. The new test plants a populated survivor beside full-dict frames; reverting the re-fold throws "recovery symbol baseline mismatch [expected=3, actual=0]". Co-Authored-By: Claude Opus 4.8 (1M context) --- .../client/sf/cursor/CursorSendEngine.java | 30 ++++++ .../qwp/client/DeltaDictRecoveryTest.java | 95 +++++++++++++++++++ 2 files changed, 125 insertions(+) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java index 64242e41..083fc038 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java @@ -404,6 +404,36 @@ private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager man && recoveredMaxSymbolDeltaStart == 0L) { persistedDictInProgress.close(); persistedDictInProgress = null; + // Re-fold at baseline 0. The analysis above was keyed to the + // dictionary's size, and every consumer of it presents the baseline it + // derived the same way -- seedGlobalDictionaryFromPersisted computes + // baseline 0 once pd is gone, and checkedRecoveryAnalysis rejects a + // baseline that disagrees with the fold. Discarding a dictionary that + // held entries (size > 0) therefore desynchronised the two and threw + // IllegalStateException("recovery symbol baseline mismatch") out of + // build(). That is NOT an UnreplayableSlotException, so build()'s + // quarantine handler could not catch it and set the slot aside: with a + // stable senderId every restart re-recovered the same slot and threw + // again, so the application could never construct a Sender -- it could + // not even BUFFER new rows. Exactly the outage quarantineTornSlot + // exists to prevent, on a slot that is fully recoverable (its frames + // carry their whole dictionary inline). + // + // Reachable on one transient plus one crash: a session whose + // .symbol-dict fails to open (EIO, fd exhaustion, a Windows share + // lock) falls back to full-dict frames and, per the never-recreate + // contract, leaves the previous session's populated side-file intact; + // if that session then crashes, this recovery opens a dictionary with + // size > 0 next to self-sufficient frames that out-reach it. + // + // Re-folding rather than keeping the dictionary preserves the discard's + // whole point -- the slot recovers in full-dict mode, exactly as it was + // written, with producer, mirror and replay guard all anchored at 0. + recoveredFrameAnalysisInProgress.close(); + recoveredFrameAnalysisInProgress = recovered.analyzeRecovery(0); + this.recoveredCommitBoundaryFsn = recoveredFrameAnalysisInProgress.commitBoundaryFsn(); + this.recoveredMaxSymbolId = recoveredFrameAnalysisInProgress.maxDeltaEnd() - 1L; + this.recoveredMaxSymbolDeltaStart = recoveredFrameAnalysisInProgress.maxDeltaStart(); } } else { // Fresh start with no recovered segments. Any stale diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java index b7daef8f..4e36e983 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java @@ -1352,6 +1352,101 @@ public void testTornDictTotalLossRebuildsFromSurvivingFrames() throws Exception }); } + @Test + public void testFullDictFramesRecoverBesideASurvivingPopulatedDictionary() throws Exception { + // The populated-side-file twin of testFullDictFramesRecoverInFullDictModeInsteadOf- + // Bricking. Self-sufficient frames can also be recovered next to a .symbol-dict + // that HOLDS entries, and that slot must recover exactly as the empty-side-file + // one does. + // + // It takes one transient plus one crash. A delta session leaves a populated + // .symbol-dict. The next session's open of it fails transiently (EIO, fd + // exhaustion, a Windows share lock), so that session runs full-dict fallback -- + // and PersistedSymbolDict's never-recreate contract deliberately leaves the + // older side-file intact rather than truncating the only copy of load-bearing + // state. That session writes self-sufficient frames out-reaching the stale + // dictionary, then crashes. This recovery opens the survivor: size() > 0, + // maxSymbolDeltaStart == 0, recoveredMaxSymbolId >= size(). + // + // CursorSendEngine discards the dictionary here (the frames carry their own, so + // it is not needed) -- but the recovery analysis was folded with the dictionary's + // size as its baseline, while seedGlobalDictionaryFromPersisted computes baseline + // 0 once pd is gone. checkedRecoveryAnalysis then threw + // IllegalStateException("recovery symbol baseline mismatch"), which is NOT an + // UnreplayableSlotException, so build()'s quarantine handler could not set the + // slot aside. With a stable senderId every restart re-recovered the same slot and + // threw again: the application could never construct a Sender, so it could not + // even buffer new rows -- on a slot that is perfectly recoverable. The engine now + // re-folds at baseline 0 when it discards. + assertMemoryLeak(() -> { + java.nio.file.Path slot = Paths.get(sfDir, "default"); + java.nio.file.Path dict = slot.resolve(".symbol-dict"); + // Phase 1: force full-dict fallback with a planted directory, so the frames + // land self-sufficient (deltaStart=0) and no side-file is written. + java.nio.file.Files.createDirectories(dict); + java.nio.file.Path blocker = dict.resolve("blocker"); + java.nio.file.Files.createFile(blocker); + try (TestWebSocketServer silent = new TestWebSocketServer(new SilentHandler())) { + int port = silent.getPort(); + silent.start(); + Assert.assertTrue(silent.awaitStart(5, TimeUnit.SECONDS)); + String cfg = "ws::addr=localhost:" + port + + ";sf_dir=" + sfDir + + ";close_flush_timeout_millis=0;"; + try (Sender s1 = Sender.fromConfig(cfg)) { + for (int i = 0; i < DISTINCT_SYMBOLS; i++) { + s1.table("m").symbol("s", "sym-" + i).longColumn("v", i).atNow(); + s1.flush(); + } + } + Assert.assertTrue("full-dict fallback: .symbol-dict must stay a directory", + java.nio.file.Files.isDirectory(dict)); + } + + // Swap the planted directory for the POPULATED side-file the earlier delta + // session left behind. Fewer entries than the frames reference, so the + // discard's recoveredMaxSymbolId >= size() guard fires with size() > 0 -- + // the case that bricked build(). + java.nio.file.Files.delete(blocker); + java.nio.file.Files.delete(dict); + final int survivingEntries = 3; + Assert.assertTrue(survivingEntries < DISTINCT_SYMBOLS); + try (PersistedSymbolDict survivor = PersistedSymbolDict.openClean(slot.toString())) { + Assert.assertNotNull("the survivor dictionary must open", survivor); + for (int i = 0; i < survivingEntries; i++) { + survivor.appendSymbol("sym-" + i); + } + Assert.assertEquals(survivingEntries, survivor.size()); + } + + // Phase 2: recover. build() must SUCCEED, and the self-sufficient frames must + // replay gap-free in full-dict mode -- the stale dictionary is discarded, not + // trusted, so it cannot misattribute an id. + DictReconstructingHandler handler = new DictReconstructingHandler(); + try (TestWebSocketServer good = new TestWebSocketServer(handler)) { + int port = good.getPort(); + good.start(); + Assert.assertTrue(good.awaitStart(5, TimeUnit.SECONDS)); + String cfg = "ws::addr=localhost:" + port + ";sf_dir=" + sfDir + ";"; + try (Sender ignored = Sender.fromConfig(cfg)) { // must NOT throw + long deadline = System.currentTimeMillis() + 5_000; + while (System.currentTimeMillis() < deadline + && handler.maxDictSize() < DISTINCT_SYMBOLS) { + Thread.sleep(20); + } + } + Assert.assertFalse("a discarded dictionary leaves full-dict mode, which is " + + "catch-up-free", handler.sawCatchUpFrame); + List reconstructed = handler.dictSnapshot(); + Assert.assertEquals("reconstructed dictionary size", + DISTINCT_SYMBOLS, reconstructed.size()); + for (int i = 0; i < DISTINCT_SYMBOLS; i++) { + Assert.assertEquals("dictionary id " + i, "sym-" + i, reconstructed.get(i)); + } + } + }); + } + @Test public void testFullDictFramesRecoverInFullDictModeInsteadOfBricking() throws Exception { // M1 regression (counterpoint to testTornDictTotalLossFailsCleanOnResume): a From ded2edc64a3fe059801053468d01d467ee7e23cf Mon Sep 17 00:00:00 2001 From: glasstiger Date: Fri, 17 Jul 2026 16:00:55 +0100 Subject: [PATCH 90/96] Close store-and-forward recovery and drainer gaps Four fixes across the recovery, orphan-scan and drainer paths, all on inputs a real cluster can produce. Bound the oversized-batch throw. flushPendingRowsSplit's all-or-nothing pre-flight throws when a single table's frame exceeds the server cap, and keeps the batch buffered so a rejected flush does not silently drop the caller's rows -- correct, and better than the old behaviour that discarded every table's rows while reporting failure. But a batch that can never fit the reachable cap then re-rejects on every flush(), growing unbounded if the caller keeps appending. The batch cannot drain until a larger-cap node returns or the sender is closed, so the message now says so, rather than reading as a transient a caller would retry into an ever-larger batch. Stop OrphanScanner counting the legacy-reader guards as data. CursorSendEngine plants .qwp-v2-guard-a/b.sfa in every disk-mode slot, before recovery or any segment, and its failure path does not unlink them. A construction that then throws (ENOSPC on the initial segment) leaves a slot holding only guards, which the scanner reported as an orphan with unacked data: a drainer adopted it, failed its own build under the same disk pressure, and dropped a permanent .failed sentinel plus an ERROR on a slot that never held a byte. Exclude the guards by name (SegmentRing's own predicate, now shared) -- a real segment beside them still counts. Saturate the drainer's capability-gap budget. reconnect_max_duration_millis is validated only as > 0, so a large value (Long.MAX_VALUE is the natural "never give up") wrapped the raw millis*1_000_000 multiply negative, and the OR-gated elapsed check then quarantined the slot on the FIRST capability-gap sweep, skipping the whole 16-sweep settle budget -- asking for more tolerance bought none. Convert with TimeUnit.toNanos, which clamps at Long.MAX_VALUE. The keepalive-interval and poison-escalation windows in CursorWebSocketSendLoop's constructor had the identical raw multiply -- the poison one is the exact twin of the cap-gap dwell the PR already hardened -- and a negative there escalates a transient to a producer-fatal terminal on strike count alone; convert both the same way. Make the legacy-reader barrier durable and stop re-truncating it. Its content never changes, yet it was rewritten via openCleanRW (truncate) on every engine open and never synced, so every open re-opened a window in which a host crash leaves a zero-filled guard beside already-durable v2 segments -- and a rolled-back v1 reader then skips the guard (bad magic) and the segments (bad version) alike, reads the slot empty, and truncates the unacked log. Verify an existing guard and rewrite only a missing or damaged one, and msync the guard on creation (33 bytes, once per slot lifetime) so it is durable before the data it protects. The guard frame is not byte-reproducible, so the test discriminates kept from rewritten by byte-identity across a reopen and asserts only semantic validity after a repair. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../qwp/client/QwpWebSocketSender.java | 15 ++++- .../client/sf/cursor/BackgroundDrainer.java | 13 ++++- .../sf/cursor/CursorWebSocketSendLoop.java | 12 +++- .../qwp/client/sf/cursor/MmapSegment.java | 9 +++ .../qwp/client/sf/cursor/OrphanScanner.java | 12 +++- .../qwp/client/sf/cursor/SegmentRing.java | 57 ++++++++++++++++--- .../sf/cursor/CursorSendEngineTest.java | 57 +++++++++++++++++++ .../client/sf/cursor/OrphanScannerTest.java | 28 +++++++++ 8 files changed, 191 insertions(+), 12 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index f06ea87e..8baf5004 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -3702,10 +3702,23 @@ private void flushPendingRowsSplit( splitFrameBodyBytes.getQuick(bodyIdx), simBaseline, currentBatchMaxSymbolId); bodyIdx++; if (messageSize > cap) { + // The batch stays BUFFERED: this throw precedes every publish, and a + // rejected flush must not silently discard the caller's rows (see + // SelfSufficientFramesTest#testOversizedTableSplitStrandsNothing). So the + // next flush() re-encodes and re-rejects the same batch until either the + // reachable cap grows -- a failover to a larger-cap node, which is the + // case retaining the rows exists to survive -- or the sender is closed, + // which discards them. It cannot drain against a cap this table will + // never fit, so say that here rather than let a caller read the repeat + // rejections as a transient and keep appending to a batch that only + // grows. throw new LineSenderException("single table batch too large for server batch cap") .put(" [table=").put(tableName) .put(", messageSize=").put(messageSize) - .put(", serverMaxBatchSize=").put(cap).put(']'); + .put(", serverMaxBatchSize=").put(cap).put(']') + .put("; the batch is retained for retry and every flush() will " + + "reject it again until a larger-cap node is reached -- " + + "close the sender to discard it, or produce smaller batches"); } // Mirror advanceSentMaxSymbolId: once the first frame ships the batch's // new ids, the remaining frames carry an empty delta above the baseline. diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/BackgroundDrainer.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/BackgroundDrainer.java index fc5116fa..97130cb7 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/BackgroundDrainer.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/BackgroundDrainer.java @@ -36,6 +36,7 @@ import org.slf4j.LoggerFactory; import java.util.concurrent.ThreadLocalRandom; +import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.LockSupport; /** @@ -287,7 +288,17 @@ public WebSocketClient connectWithDurableAckRetry() { // Timestamp of the previous capability-gap sweep; 0 = the next gap // charges nothing because a fresh episode is starting. long lastCapabilityGapNanos = 0L; - final long capabilityGapBudgetNanos = reconnectMaxDurationMillis * 1_000_000L; + // Saturate rather than multiply: reconnect_max_duration_millis is validated only + // as > 0, so a large value (Long.MAX_VALUE is the natural way to ask for "never + // give up") wraps a raw multiply NEGATIVE. capabilityGapElapsedNanos then clears + // the budget on the FIRST capability-gap sweep -- 0 >= a negative -- and, because + // this gate is an OR with the attempt cap, nothing else holds it back: the slot + // quarantines immediately, skipping the whole 16-sweep settle budget. Asking for + // more tolerance would buy exactly none. TimeUnit clamps at Long.MAX_VALUE, which + // is the intended "effectively unbounded". CursorWebSocketSendLoop's dwell + // conversion guards the same way for the same reason. + final long capabilityGapBudgetNanos = + TimeUnit.MILLISECONDS.toNanos(reconnectMaxDurationMillis); // Observability-only counter for the transient all-replica window; // never consulted for escalation (Invariant B). int roleRejectAttempts = 0; diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java index 5619d1e9..55240e93 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java @@ -839,12 +839,20 @@ public CursorWebSocketSendLoop(WebSocketClient client, CursorSendEngine engine, this.reconnectInitialBackoffMillis = reconnectInitialBackoffMillis; this.reconnectMaxBackoffMillis = reconnectMaxBackoffMillis; this.durableAckMode = durableAckMode; + // Saturate, never multiply raw -- the same hazard the cap-gap dwell above + // guards. A raw multiply wraps a large millisecond value NEGATIVE, and both of + // these read as "elapsed >= window", so a negative makes the gate trivially + // true: the keepalive would ping on every loop pass, and the poison detector + // would escalate on the strike count alone -- exactly the transient-to-terminal + // false positive the dwell exists to stop (Invariant B). Both keys are + // user-supplied and validated only as >= 0, so asking for a very long window is + // legal, and asking for a longer one must never buy a shorter one. this.durableAckKeepaliveIntervalNanos = durableAckKeepaliveIntervalMillis > 0 - ? durableAckKeepaliveIntervalMillis * 1_000_000L + ? TimeUnit.MILLISECONDS.toNanos(durableAckKeepaliveIntervalMillis) : 0L; this.maxHeadFrameRejections = maxHeadFrameRejections; this.poisonMinEscalationWindowNanos = poisonMinEscalationWindowMillis > 0 - ? poisonMinEscalationWindowMillis * 1_000_000L + ? TimeUnit.MILLISECONDS.toNanos(poisonMinEscalationWindowMillis) : 0L; // SYNC/OFF startup hands a live client to the constructor, so we // already know we reached the server at least once. ASYNC startup diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java index 4d216f8f..bc1d2da3 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java @@ -194,6 +194,15 @@ static void createLegacyReaderGuard(String path, long baseSeq) { if (segment.tryAppend(payload, 1) != HEADER_SIZE) { throw new MmapSegmentException("could not append legacy-reader guard frame: " + path); } + // Unlike the segment log this file guards, the guard IS msync'd. It is a + // rollback barrier, so it has to be durable before the data it protects + // becomes durable -- a guard still sitting in the page cache when the host + // dies leaves zeroes on disk beside written-back v2 segments, and a + // rolled-back v1 reader skips the guard (bad magic) and the segments (bad + // version) alike, sees an empty slot, and truncates the log. It is 33 bytes + // written once per slot lifetime, not per flush, so the sync costs nothing + // measurable and buys the barrier the durability its whole purpose assumes. + segment.msync(); } finally { if (payload != 0L) { Unsafe.free(payload, 1, MemoryTag.NATIVE_DEFAULT); diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/OrphanScanner.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/OrphanScanner.java index 833960ec..49b70674 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/OrphanScanner.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/OrphanScanner.java @@ -327,7 +327,17 @@ private static boolean hasAnySegmentFile(String slotPath) { while (rc > 0) { String name = Files.utf8ToString(Files.findName(find)); rc = Files.findNext(find); - if (name != null && name.endsWith(".sfa")) { + // The legacy-reader barriers are named .sfa on purpose (a rolled-back v1 + // reader must not skip them), but they are a barrier, not data. Counting + // them here would call a slot that never held a byte an orphan with + // unacked data: CursorSendEngine plants them FIRST, before recovery or + // any segment is created, and its cleanup does not unlink them, so a + // construction that fails afterwards (ENOSPC on the initial segment) + // leaves a directory holding nothing else. A drainer would then adopt it, + // fail its own build under the same disk pressure, and quarantine an + // empty slot with a permanent .failed sentinel plus an ERROR for an + // operator to chase. + if (name != null && name.endsWith(".sfa") && !SegmentRing.isLegacyReaderGuard(name)) { return true; } } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java index 45aaebb3..aafaabbb 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java @@ -349,12 +349,47 @@ public long ackedFsn() { * logical ring. */ static void installLegacyReaderBarrier(String sfDir) { - MmapSegment.createLegacyReaderGuard( - sfDir + '/' + LEGACY_READER_GUARD_A, - Long.MAX_VALUE - 4L); - MmapSegment.createLegacyReaderGuard( - sfDir + '/' + LEGACY_READER_GUARD_B, - Long.MAX_VALUE - 1L); + installLegacyReaderGuardIfDamaged(sfDir + '/' + LEGACY_READER_GUARD_A, Long.MAX_VALUE - 4L); + installLegacyReaderGuardIfDamaged(sfDir + '/' + LEGACY_READER_GUARD_B, Long.MAX_VALUE - 1L); + } + + /** + * Verifies the guard at {@code path} and rewrites it only when it is missing or + * damaged. + *

      + * A guard's content never changes, so re-creating one that is already intact buys + * nothing -- and costs the only window in which the barrier can be lost. Creation + * goes through {@code openCleanRW} (which truncates) and does not fsync, so an + * unconditional rewrite on every engine open re-opens that window every time. A host + * crash inside it leaves a zero-filled guard next to v2 segment data that is already + * durable; a rolled-back v1 reader then skips the guard (bad magic) AND the v2 + * segments (bad version), concludes the slot is empty, and truncates the unacked log + * the barrier exists to protect. Keeping an intact guard means the steady state never + * enters that window at all. + */ + private static void installLegacyReaderGuardIfDamaged(String path, long baseSeq) { + if (isIntactLegacyReaderGuard(path, baseSeq)) { + return; + } + MmapSegment.createLegacyReaderGuard(path, baseSeq); + } + + private static boolean isIntactLegacyReaderGuard(String path, long baseSeq) { + if (!Files.exists(path)) { + return false; + } + try (MmapSegment guard = MmapSegment.openExisting(path)) { + // Anything short of the exact shape createLegacyReaderGuard writes -- a + // zero-filled survivor of a torn creation included -- is rewritten. + return guard != null + && guard.version() == MmapSegment.LEGACY_VERSION + && guard.baseSeq() == baseSeq + && guard.frameCount() == 1 + && guard.tornTailBytes() == 0L; + } catch (Throwable t) { + LOG.warn("legacy-reader guard {} could not be verified ({}); rewriting it", path, t.toString()); + return false; + } } /** @@ -945,7 +980,15 @@ private static void sortByBaseSeq(ObjList list, int lo, int hi) { } } - private static boolean isLegacyReaderGuard(String name) { + /** + * Whether {@code name} is one of the reserved rollback-barrier files this class + * plants in every disk-mode slot. They carry the {@code .sfa} suffix so a rolled-back + * v1 reader cannot skip them, which means every directory walk that reasons about + * "does this slot hold data" has to exclude them -- they are a barrier, not a log. + * Package-private so {@link OrphanScanner} shares this one definition rather than + * matching the names itself. + */ + static boolean isLegacyReaderGuard(String name) { return LEGACY_READER_GUARD_A.equals(name) || LEGACY_READER_GUARD_B.equals(name); } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineTest.java index 0bdddb3e..0b7bf989 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineTest.java @@ -47,6 +47,7 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.LockSupport; +import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; @@ -298,6 +299,62 @@ public void testRecoveryAdvancesAckedFsnPastWatermark() throws Exception { }); } + @Test + public void testLegacyReaderGuardsSurviveReopenAndAreRepairedWhenDamaged() throws Exception { + // The barrier is only worth what it is worth on disk. Creating it truncates + // (openCleanRW) and, before this, did not sync -- and it ran unconditionally on + // EVERY engine open, so every open destroyed a durable guard and re-dirtied it. + // A host crash inside that writeback window leaves a zero-filled guard beside v2 + // segments that ARE already durable; a rolled-back v1 reader then skips the guard + // (bad magic) and the segments (bad version) alike, reads the slot as empty, and + // truncates the unacked log the guard was planted to protect. The guard's content + // never changes, so an intact one is verified and kept, and only a damaged one is + // rewritten -- the steady state never re-enters the window. + // + // Two guards created with identical arguments are NOT byte-identical (they + // differ in the frame header region), so a KEPT guard is byte-identical to + // itself across a reopen while a REWRITTEN one is not -- which is exactly the + // discriminator this needs. The repair leg then only asserts semantic validity, + // since a rebuilt guard legitimately differs byte-for-byte from the original. + TestUtils.assertMemoryLeak(() -> { + long buf = Unsafe.malloc(16, MemoryTag.NATIVE_DEFAULT); + try { + try (CursorSendEngine engine = new CursorSendEngine(tmpDir, 4096)) { + assertEquals(0L, engine.appendBlocking(buf, 16)); + } + String guardA = tmpDir + "/.qwp-v2-guard-a.sfa"; + byte[] planted = java.nio.file.Files.readAllBytes(java.nio.file.Paths.get(guardA)); + assertTrue("the guard must carry bytes", planted.length > 0); + + // Reopening the slot must leave an intact guard untouched. Under the old + // unconditional re-create this rewrites it, and because the frame is not + // byte-reproducible the bytes then differ -- so this comparison fails. + try (CursorSendEngine reopened = new CursorSendEngine(tmpDir, 4096)) { + assertEquals("recovery must adopt the one real frame, not a guard", + 0L, reopened.publishedFsn()); + } + assertArrayEquals("an intact guard must survive a reopen untouched", + planted, java.nio.file.Files.readAllBytes(java.nio.file.Paths.get(guardA))); + + // Zero it -- the shape a torn creation leaves -- and it must be rebuilt to + // a valid guard, otherwise a slot that crashed mid-plant stays unprotected. + java.nio.file.Files.write(java.nio.file.Paths.get(guardA), new byte[planted.length]); + try (CursorSendEngine repaired = new CursorSendEngine(tmpDir, 4096)) { + assertEquals("recovery must adopt the one real frame, not a guard", + 0L, repaired.publishedFsn()); + } + try (MmapSegment guard = MmapSegment.openExisting(guardA)) { + assertNotNull("a damaged guard must be rewritten to a valid one", guard); + assertEquals(MmapSegment.LEGACY_VERSION, guard.version()); + assertEquals(1L, guard.frameCount()); + assertEquals("a rewritten guard must not read as torn", 0L, guard.tornTailBytes()); + } + } finally { + Unsafe.free(buf, 16, MemoryTag.NATIVE_DEFAULT); + } + }); + } + @Test public void testV2DiskSlotForcesLegacyReaderToFailClosed() throws Exception { TestUtils.assertMemoryLeak(() -> { diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/OrphanScannerTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/OrphanScannerTest.java index a39bef67..c198afc5 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/OrphanScannerTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/OrphanScannerTest.java @@ -89,6 +89,34 @@ public void testSlotWithSfaIsAnOrphan() throws Exception { }); } + @Test + public void testSlotHoldingOnlyLegacyReaderGuardsIsNotAnOrphan() throws Exception { + TestUtils.assertMemoryLeak(() -> { + // The rollback barriers are named .sfa deliberately -- a rolled-back v1 + // reader must not skip them -- but they are a barrier, not data. A slot + // holding nothing else has never had a byte written to it and must not be + // adopted: CursorSendEngine plants the guards FIRST, before recovery or the + // initial segment, and its failure path does not unlink them, so an ENOSPC + // partway through construction leaves exactly this directory. Counting the + // guards sent a drainer to adopt it, which fails under the same disk + // pressure and quarantines the empty slot with a permanent .failed sentinel. + String slot = sfDir + "/guards-only"; + assertEquals(0, Files.mkdir(slot, Files.DIR_MODE_DEFAULT)); + touchFile(slot + "/.qwp-v2-guard-a.sfa"); + touchFile(slot + "/.qwp-v2-guard-b.sfa"); + + ObjList orphans = OrphanScanner.scan(sfDir, "default"); + assertEquals(0, orphans.size()); + + // ...but a real segment alongside them still is: the exclusion must be by + // guard name, not "ignore dot-prefixed .sfa" or "ignore the first two". + touchFile(slot + "/sf-0001.sfa"); + orphans = OrphanScanner.scan(sfDir, "default"); + assertEquals(1, orphans.size()); + assertEquals(slot, orphans.get(0)); + }); + } + @Test public void testEmptySlotDirIsNotAnOrphan() throws Exception { TestUtils.assertMemoryLeak(() -> { From 16be7646b04dddb7625ac847340eefa60e364878 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Fri, 17 Jul 2026 23:43:40 +0100 Subject: [PATCH 91/96] Fix connectWithRetry budget overflow connectWithRetry set its deadline as startNanos + maxDurationMillis * 1_000_000L, a raw millis-to-nanos multiply. A large reconnect_max_duration_millis -- Long.MAX_VALUE is the natural "retry until the server boots" value, and the builder setter enforces no upper bound -- wraps that product negative, so the pre-condition retry loop runs zero iterations. The SYNC initial connect then throws "no attempts made" without ever calling the reconnect factory: the exact opposite of the requested patience. The prior commit hardened this same overflow class in the drainer's capabilityGapBudgetNanos and the loop dwells via TimeUnit.MILLISECONDS.toNanos, but missed connectWithRetry, the primary consumer of the value. connectWithRetry now compares elapsed time against a saturating budget instead of an absolute deadline: toNanos alone still overflows the startNanos + budget sum once it saturates to Long.MAX_VALUE, whereas the bounded nanoTime difference stays correct. A new regression test drives connectWithRetry with a Long.MAX_VALUE budget and asserts at least one attempt runs; it fails on the old code with "failed after 0ms / 0 attempts". Co-Authored-By: Claude Opus 4.8 (1M context) --- .../sf/cursor/CursorWebSocketSendLoop.java | 15 +++++-- ...ursorWebSocketSendLoopZeroBackoffTest.java | 42 +++++++++++++++++++ 2 files changed, 54 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java index 55240e93..431eca59 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java @@ -951,12 +951,21 @@ public static WebSocketClient connectWithRetry( String contextLabel ) { long startNanos = System.nanoTime(); - long deadlineNanos = startNanos + maxDurationMillis * 1_000_000L; + // Saturating budget, compared against ELAPSED rather than an absolute + // startNanos+budget deadline. A large reconnect_max_duration_millis (e.g. + // Long.MAX_VALUE, the natural "retry until the server boots") must not collapse + // the budget to zero: a raw maxDurationMillis*1_000_000L wraps NEGATIVE -- the + // same overflow the drainer's capabilityGapBudgetNanos and the loop dwells were + // hardened against -- so the pre-condition loop below would make ZERO attempts. + // Even TimeUnit.toNanos saturated to Long.MAX_VALUE would overflow an absolute + // startNanos+budget sum; comparing the bounded nanoTime difference against the + // budget stays correct at saturation. + long budgetNanos = TimeUnit.MILLISECONDS.toNanos(maxDurationMillis); long backoffMillis = initialBackoffMillis; int attempts = 0; long lastLogNanos = 0L; Throwable lastError = null; - while (System.nanoTime() < deadlineNanos) { + while (System.nanoTime() - startNanos < budgetNanos) { attempts++; try { WebSocketClient c = factory.reconnect(); @@ -1011,7 +1020,7 @@ public static WebSocketClient connectWithRetry( // this > 0. Mirrors BackgroundDrainer's sweep-loop jitter guard. long jitter = ThreadLocalRandom.current().nextLong(Math.max(1L, backoffMillis)); long sleepMillis = backoffMillis + jitter; - long remainingMillis = (deadlineNanos - System.nanoTime()) / 1_000_000L; + long remainingMillis = (budgetNanos - (System.nanoTime() - startNanos)) / 1_000_000L; if (remainingMillis <= 0) { break; } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopZeroBackoffTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopZeroBackoffTest.java index a8fc8882..dfa6497a 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopZeroBackoffTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopZeroBackoffTest.java @@ -33,6 +33,7 @@ import org.junit.rules.TemporaryFolder; import java.io.IOException; +import java.util.concurrent.atomic.AtomicInteger; /** * Regression guard for the I/O-thread silent-death hazard in @@ -135,4 +136,45 @@ public void zeroBackoffConnectFailuresKeepRetryingInsteadOfKillingIoThread() thr } } } + + /** + * Regression guard for the connect-budget overflow. A large + * {@code reconnect_max_duration_millis} -- {@code Long.MAX_VALUE} is the + * natural "retry until the server boots" value, and the builder setter + * imposes no upper bound -- must NOT collapse the budget to zero. The pre-fix + * {@code startNanos + maxDurationMillis * 1_000_000L} wrapped NEGATIVE, so the + * pre-condition retry loop ran ZERO iterations and {@code connectWithRetry} + * threw "no attempts made" without ever calling the factory: the exact + * opposite of the requested maximal patience. Post-fix the saturating + * elapsed-vs-budget comparison keeps the loop live, so at least one attempt + * is made. + *

      + * The factory throws an {@link Error} rather than a retriable transport + * failure on purpose: under a {@code Long.MAX_VALUE} budget a retriable + * failure would retry forever, whereas an Error propagates on the first + * attempt -- and it can only propagate if the loop body ran at all, which is + * precisely the signal the pre-fix overflow destroyed. + */ + @Test(timeout = 30_000) + public void connectWithRetryWithSaturatingBudgetStillMakesAttempts() { + AtomicInteger attempts = new AtomicInteger(); + try { + CursorWebSocketSendLoop.connectWithRetry( + () -> { + attempts.incrementAndGet(); + throw new LinkageError("attempt reached (test)"); + }, + Long.MAX_VALUE, // pre-fix: startNanos + MAX * 1_000_000L wraps negative + 1L, + 4L, + "test-connect-budget-overflow"); + Assert.fail("expected the factory's Error to propagate after an attempt"); + } catch (LinkageError expected) { + Assert.assertEquals("attempt reached (test)", expected.getMessage()); + } + Assert.assertEquals( + "a Long.MAX_VALUE reconnect budget must not collapse to zero attempts " + + "(pre-fix millis * 1_000_000L overflow)", + 1, attempts.get()); + } } From 042c123839fc26c527b3d10f5cc16e8e4a5df617 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Fri, 17 Jul 2026 23:59:43 +0100 Subject: [PATCH 92/96] Single-pass symbol-dict append; test window remap PersistedSymbolDict.appendSymbols encoded the mapped chunk in two passes over the id range: a first loop called Utf8s.utf8Bytes on every symbol just to size the header varint and the mmap reserve, then a second loop re-walked each symbol's UTF-8 length to write it. It now stages the entry region in the scratch buffer with a single walk per symbol and bulk-copies it into the append window after the header, dropping the redundant walk. The chunk it writes is byte-identical, so the exact-file-size "one chunk, one CRC" test still holds. A direct in-window encode cannot avoid the double walk: the back-to-back chunk format leaves no room to reserve and back-fill the header in place. The path is cold -- only a retry after a failed publish re-encodes from the dictionary; the steady state persists a frame's pre-encoded bytes through appendRawEntries. Add testMappedAppendSpansMultipleWindowsAndRecoversDenseIds: it appends a dictionary larger than one 4 MiB append window, asserts the window remaps (appendMapGrowthCount >= 2) without a positioned write, then reopens and checks every id recovers to its own symbol across the boundary. This exercises the ensureAppendMap remap arithmetic and the single-pass encode as chunks straddle the boundary, both of which the single-window happy path left untested. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../client/sf/cursor/PersistedSymbolDict.java | 51 +++++++++++-------- .../sf/cursor/PersistedSymbolDictTest.java | 48 +++++++++++++++++ 2 files changed, 77 insertions(+), 22 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java index fbc22784..54ba8f39 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java @@ -415,11 +415,18 @@ public synchronized void appendSymbol(CharSequence symbol) { /** * Appends the dense id range {@code [from .. to]} as one chunk with one - * checksum. Production encodes directly into a segmented append mapping, avoiding - * both the staging copy and a positioned-write syscall on every flush. The - * scratch/positioned-write fallback exists only behind an injected filesystem - * facade so short-write recovery tests retain their fault seam. Callers pass the - * dictionary and the range so the ids resolve to their symbol strings. + * checksum. This is the RE-ENCODE path: the steady-state persist ships a frame's + * pre-encoded delta bytes through {@link #appendRawEntries}, and only a retry + * after a failed publish rebuilds a range straight from the dictionary. The + * mapped path stages the entry region in the scratch buffer with a single UTF-8 + * walk per symbol, then bulk-copies it into the append window after the header, + * and still commits WITHOUT a positioned-write syscall. A direct encode into the + * window would walk each symbol's UTF-8 length twice -- the exact entriesLen sizes + * both the header varint and the mmap reserve -- and the back-to-back chunk format + * leaves no room to reserve-and-back-fill the header in place. The + * positioned-write fallback runs only behind an injected filesystem facade so + * short-write recovery tests retain their fault seam. Callers pass the dictionary + * and the range so the ids resolve to their symbol strings. *

      * Same durability and idempotency contract as {@link #appendSymbol}: no fsync, * and a short write throws WITHOUT advancing {@code size}/{@code appendOffset}, @@ -432,32 +439,32 @@ public synchronized void appendSymbols(GlobalSymbolDictionary dict, int from, in } int count = to - from + 1; if (mappedAppend) { - long entriesLenLong = 0L; + // Stage the entry region in scratch with ONE UTF-8 walk per symbol, then + // bulk-copy it into the append window after the header (see the method + // javadoc for why a direct in-window encode would have to walk each symbol + // twice). ensureScratch enforces the same MAX_SCRATCH_BYTES ceiling the old + // sizing pass did, throwing before size/appendOffset advance. + int entriesLen = 0; for (int id = from; id <= to; id++) { - int utf8Len = Utf8s.utf8Bytes(dict.getSymbol(id)); - entriesLenLong += NativeBufferWriter.varintSize(utf8Len) + (long) utf8Len; - if (entriesLenLong > MAX_SCRATCH_BYTES) { - throw new IllegalStateException("symbol dict chunk exceeds the maximum size to " - + FILE_NAME + " [required=" + entriesLenLong + ", max=" - + MAX_SCRATCH_BYTES + ']'); + CharSequence symbol = dict.getSymbol(id); + int utf8Len = Utf8s.utf8Bytes(symbol); + int wireLen = NativeBufferWriter.varintSize(utf8Len) + utf8Len; // [len][utf8] + ensureScratch((long) entriesLen + wireLen); + long q = NativeBufferWriter.writeVarint(scratchAddr + entriesLen, utf8Len); + if (utf8Len > 0) { + Utf8s.strCpyUtf8(symbol, q, utf8Len); } + entriesLen += wireLen; } - int entriesLen = (int) entriesLenLong; int hdrLen = NativeBufferWriter.varintSize(count) + NativeBufferWriter.varintSize(entriesLen); long recLen = (long) hdrLen + entriesLen + CRC_SIZE; ensureAppendMap(checkedRequiredOffset(recLen)); long recStart = appendMapAddr + appendOffset - appendMapOffset; long p = NativeBufferWriter.writeVarint(recStart, count); - p = NativeBufferWriter.writeVarint(p, entriesLen); - for (int id = from; id <= to; id++) { - CharSequence symbol = dict.getSymbol(id); - int utf8Len = Utf8s.utf8Bytes(symbol); - p = NativeBufferWriter.writeVarint(p, utf8Len); - if (utf8Len > 0) { - Utf8s.strCpyUtf8(symbol, p, utf8Len); - p += utf8Len; - } + NativeBufferWriter.writeVarint(p, entriesLen); + if (entriesLen > 0) { + Unsafe.getUnsafe().copyMemory(scratchAddr, recStart + hdrLen, entriesLen); } commitMappedChunk(recStart, hdrLen, entriesLen, count); return; diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java index 7b4f6d60..3efc76d7 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/PersistedSymbolDictTest.java @@ -452,6 +452,54 @@ public void testMappedAppendUsesMultiMegabyteWindowsWithoutPositionedWrites() th }); } + @Test + public void testMappedAppendSpansMultipleWindowsAndRecoversDenseIds() throws Exception { + // A dictionary larger than one 4 MiB append window must remap the window + // mid-append (appendMapGrowthCount >= 2) and still recover every symbol in + // dense id order across the boundary. Guards two paths the single-window + // happy path leaves uncovered: the remap arithmetic in ensureAppendMap, and + // the single-pass appendSymbols encode as its chunks straddle the boundary. + assertMemoryLeak(() -> { + final int n = 640; + final String pad = TestUtils.repeat("x", 8192); // ~8 KiB per symbol -> >4 MiB total + GlobalSymbolDictionary dict = new GlobalSymbolDictionary(); + for (int i = 0; i < n; i++) { + dict.getOrAddSymbol("sym-" + i + "-" + pad); // distinct, id == i + } + Assert.assertEquals(n, dict.size()); + + Path dir = newFolder("qwp-symdict"); + try (PersistedSymbolDict d = PersistedSymbolDict.open(dir.toString())) { + Assert.assertNotNull(d); + // Multi-symbol batches so the appendSymbols encode loop is exercised, + // not just the single-symbol path. + for (int from = 0; from < n; from += 8) { + int to = Math.min(from + 8, n) - 1; + d.appendSymbols(dict, from, to); + } + Assert.assertEquals(n, d.size()); + Assert.assertEquals("production mmap appends must not use positioned writes", + 0L, d.appendWriteCount()); + Assert.assertTrue( + "a dictionary larger than the 4 MiB window must remap at least once (saw " + + d.appendMapGrowthCount() + ")", + d.appendMapGrowthCount() >= 2); + } + + // Fresh open: every id must resolve to its own symbol, including the ids + // whose chunks straddled the 4 MiB window boundary. + try (PersistedSymbolDict re = PersistedSymbolDict.open(dir.toString())) { + Assert.assertNotNull(re); + Assert.assertEquals(n, re.size()); + ObjList got = re.readLoadedSymbols(); + for (int i = 0; i < n; i++) { + Assert.assertEquals("symbol at id " + i + " must survive the window boundary", + dict.getSymbol(i), got.getQuick(i)); + } + } + }); + } + private static int varintSize(int v) { int n = 1; while ((v >>>= 7) != 0) { From 54e21f8209f8e2598d5978e4956d562aaa4824e2 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Sat, 18 Jul 2026 00:26:11 +0100 Subject: [PATCH 93/96] Delete dead recovery symbol-walk methods Production recovery runs one ordered fold (SegmentRing.analyzeRecovery -> MmapSegment.scanRecovery -> RecoveredFrameAnalysis) and seeds the producer through CursorSendEngine.addRecoveredSymbolsTo. The older per-metric walk -- collectReplaySymbolsAbove, maxSymbolDeltaEnd and maxSymbolDeltaStart on MmapSegment and SegmentRing, plus the CursorSendEngine.collectReplaySymbolsAbove wrapper -- had no production caller and diverged from the fold: it lacked the full-dict gap reset, the unacked-gap distinction and commit-boundary checkpointing, so the tests routing through it verified an algorithm production never runs. Delete all seven methods and the now-orphaned RecoveredFrameAnalysis.appendDecodedSymbols helper, then rewire the two tests that reached the walk (RecoveredFrameAnalysisTest, CursorWebSocketSendLoopOrphanTailTest) to the production addRecoveredSymbolsTo path. That is behavior-preserving -- same coverage, same decoded symbols -- and stronger, since the tests now drive the real seeding method. Drop the imports the deletion left unused. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../client/sf/cursor/CursorSendEngine.java | 57 +--- .../qwp/client/sf/cursor/MmapSegment.java | 257 ------------------ .../sf/cursor/RecoveredFrameAnalysis.java | 16 +- .../qwp/client/sf/cursor/SegmentRing.java | 74 ----- ...CursorWebSocketSendLoopOrphanTailTest.java | 23 +- .../sf/cursor/RecoveredFrameAnalysisTest.java | 6 +- 6 files changed, 20 insertions(+), 413 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java index 083fc038..fcfb2a45 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java @@ -25,7 +25,6 @@ package io.questdb.client.cutlass.qwp.client.sf.cursor; import io.questdb.client.cutlass.qwp.client.GlobalSymbolDictionary; -import io.questdb.client.cutlass.qwp.protocol.QwpConstants; import io.questdb.client.std.Compat; import io.questdb.client.std.Files; import io.questdb.client.std.FilesFacade; @@ -375,14 +374,14 @@ private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager man // .symbol-dict no longer holds, resuming would re-use it. The walk is // bounded to recoveredCommitBoundaryFsn so the aborted orphan-deferred // tail -- retired without ever being transmitted -- does not inflate - // this and over-reject an otherwise-recoverable slot. maxSymbolDeltaEnd + // this and over-reject an otherwise-recoverable slot. maxDeltaEnd() // returns 0 when no such frame carries a symbol, yielding -1 here. // Computed before the I/O loop or producer append; single-threaded. this.recoveredMaxSymbolId = recoveredFrameAnalysisInProgress.maxDeltaEnd() - 1L; // Full-dict-fallback recovery. When the persisted .symbol-dict is a // SUBSET of the ids the surviving frames reference // (recoveredMaxSymbolId >= its size) YET every such frame is - // self-sufficient (maxSymbolDeltaStart == 0 -- a full-dict frame that + // self-sufficient (maxDeltaStart() == 0 -- a full-dict frame that // re-registers its dictionary from id 0), the slot was written in // full-dict fallback: the dictionary never opened when writing, so no // side-file exists and this recovery opened a FRESH EMPTY one. Those @@ -392,7 +391,7 @@ private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager man // was written. Without this the sender's seed-time guard would treat the // empty dictionary as a host-crash tear and brick build(), even though // the orphan drainer drains the same frames fine. A genuine torn DELTA - // dictionary keeps a frame with deltaStart > 0 (maxSymbolDeltaStart > 0) + // dictionary keeps a frame with deltaStart > 0 (maxDeltaStart() > 0) // and is NOT discarded here: it still fails clean at seed time, since // the ids its delta frames reference cannot be rebuilt without the lost // dictionary. The recoveredMaxSymbolId >= size guard means this never @@ -741,56 +740,6 @@ public synchronized void close() { } } - /** - * Rebuilds, from the surviving frames' OWN delta sections, the symbols the upcoming - * replay will register ABOVE {@code baseline}, appending them to {@code out} in - * ascending id order. Returns the coverage the replay establishes (one past the highest - * id it registers), or {@code -1} when the frames have a genuine GAP above the baseline. - *

      - * The producer seeds its dictionary from the persisted {@code .symbol-dict} and THEN - * from this, so it is fed the same entries, in the same order, that the send loop's - * {@code accumulateSentDict} will feed its mirror from as those very frames replay. The - * producer's delta baseline and the loop's mirror coverage therefore land on the same - * number by construction, which is the invariant the torn-dictionary guard rests on. - *

      - * This is what makes a slot whose dictionary was torn -- or lost outright -- still - * recoverable when the surviving frames define the ids themselves (they carry the - * symbols in their own deltas; it is why the orphan drainer can drain such a slot). Only - * a real gap, where the ids were introduced by frames since acked and trimmed away, is - * unrecoverable -- and that is precisely when this returns -1. - *

      - * Bounded above by {@link #recoveredCommitBoundaryFsn} like {@link #recoveredMaxSymbolId}: - * frames past it are the aborted orphan-deferred tail, retired without ever being - * transmitted, so their ids never reach a server and must not inflate the baseline. - */ - public long collectReplaySymbolsAbove(int baseline, ObjList out) { - if (recoveredFrameAnalysis != null - && recoveredFrameAnalysis.baseline() == baseline) { - recoveredFrameAnalysis.appendDecodedSymbols(out); - return recoveredFrameAnalysis.coverage(); - } - if (ring == null) { - return baseline; - } - return ring.collectReplaySymbolsAbove( - QwpConstants.MAGIC_MESSAGE, - QwpConstants.HEADER_OFFSET_FLAGS, - QwpConstants.FLAG_DELTA_SYMBOL_DICT, - QwpConstants.HEADER_SIZE, - // Walk from the LOWEST frame still on disk, not from ackedFsn+1. An acked frame - // is not trimmed the instant it is acked -- trim drops whole SEALED segments -- - // so the symbols it registered are usually still sitting right there, and they - // are exactly the ids the replay set's deltas start above. Skipping them threw - // away the only surviving copy of those symbols and condemned a slot that had - // everything it needed. A slot whose registering frames really HAVE been - // trimmed still reports the gap, because then no frame on disk holds them. - 0L, - recoveredCommitBoundaryFsn, - baseline, - out - ); - } - /** * Decodes the cached recovery suffix directly into the producer's global * dictionary. Recovery always builds the analysis with the persisted diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java index bc1d2da3..406526c4 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java @@ -28,11 +28,9 @@ import io.questdb.client.std.Files; import io.questdb.client.std.FilesFacade; import io.questdb.client.std.MemoryTag; -import io.questdb.client.std.ObjList; import io.questdb.client.std.Os; import io.questdb.client.std.QuietCloseable; import io.questdb.client.std.Unsafe; -import io.questdb.client.std.str.Utf8s; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -588,121 +586,6 @@ public long findLastFrameFsnWithoutPayloadFlag(int flagsOffset, int flagMask, in return best; } - /** - * Rebuilds, from the frames' own delta sections, the symbols a replay of - * {@code [minFsnInclusive .. maxFsnInclusive]} would register ABOVE {@code coverage}, - * appending them to {@code out} in ascending id order. Returns the coverage the walk - * establishes (one past the highest id registered), or {@code -1} when a frame's delta - * starts ABOVE the coverage reached so far. - *

      - * This is the recovery-time twin of {@code CursorWebSocketSendLoop.accumulateSentDict}: - * it visits the same frames, in the same FSN order, and extracts exactly the entries - * that method will add to the send loop's mirror as those frames go back on the wire. - * The producer's dictionary is seeded from it so the producer's delta baseline and the - * loop's mirror coverage stay in lockstep BY CONSTRUCTION -- which is what makes a slot - * whose {@code .symbol-dict} was torn (or lost entirely) recoverable, as long as the - * surviving frames define the ids themselves. - *

      - * A {@code -1} return is a genuine GAP: the ids between the coverage and the frame's - * delta start were introduced by frames that were acked and trimmed away, so they lived - * ONLY in the lost dictionary and cannot be rebuilt from anything. That is the one case - * the data really must be resent, and it is exactly the condition the send loop's - * torn-dictionary guard ({@code deltaStart > sentDictCount}) would trip on at replay -- - * so the two agree, rather than the producer failing on slots the drainer drains fine. - *

      - * Same frame layout and FSN bound as {@link #maxSymbolDeltaEnd}. - */ - public long collectReplaySymbolsAbove( - int headerMagic, - int flagsOffset, - int flagDeltaMask, - int qwpHeaderSize, - long minFsnInclusive, - long maxFsnInclusive, - long coverage, - ObjList out - ) { - long off = HEADER_SIZE; - long frames = frameCount; - for (long i = 0; i < frames; i++) { - long fsn = baseSeq + i; - int payloadLen = Unsafe.getUnsafe().getInt(mmapAddress + off + 4); - long payload = mmapAddress + off + FRAME_HEADER_SIZE; - if (fsn >= minFsnInclusive - && fsn <= maxFsnInclusive - && payloadLen >= qwpHeaderSize - && payloadLen > flagsOffset - && Unsafe.getUnsafe().getInt(payload) == headerMagic - && (Unsafe.getUnsafe().getByte(payload + flagsOffset) & flagDeltaMask) != 0) { - long p = payload + qwpHeaderSize; - long limit = payload + payloadLen; - long deltaStart = 0L; - int shift = 0; - while (p < limit) { - byte b = Unsafe.getUnsafe().getByte(p++); - deltaStart |= (long) (b & 0x7F) << shift; - if ((b & 0x80) == 0) { - break; - } - shift += 7; - if (shift > 35) { - break; // corrupt run; recovered frames are CRC-valid, so defensive only - } - } - long deltaCount = 0L; - shift = 0; - while (p < limit) { - byte b = Unsafe.getUnsafe().getByte(p++); - deltaCount |= (long) (b & 0x7F) << shift; - if ((b & 0x80) == 0) { - break; - } - shift += 7; - if (shift > 35) { - break; - } - } - if (deltaStart > coverage) { - return -1L; // gap: ids [coverage, deltaStart) are unrecoverable - } - // Walk this frame's [len varint][utf8] entries. Ids below the coverage we - // already hold are skipped (a replayed frame legitimately overlaps the tip); - // ids at or above it extend the dictionary. - long id = deltaStart; - for (long k = 0; k < deltaCount; k++, id++) { - long len = 0L; - shift = 0; - while (p < limit) { - byte b = Unsafe.getUnsafe().getByte(p++); - len |= (long) (b & 0x7F) << shift; - if ((b & 0x80) == 0) { - break; - } - shift += 7; - if (shift > 35) { - break; - } - } - if (p + len > limit) { - // Malformed entry region. Do NOT guess: report it as unrecoverable - // so the caller fails clean rather than seeding a shifted dictionary. - return -1L; - } - if (id >= coverage) { - out.add(Utf8s.stringFromUtf8Bytes(p, p + len)); - } - p += len; - } - long deltaEnd = deltaStart + deltaCount; - if (deltaEnd > coverage) { - coverage = deltaEnd; - } - } - off += FRAME_HEADER_SIZE + payloadLen; - } - return coverage; - } - /** * Feeds every recovered frame in this segment to the engine's single-pass * recovery fold. Segments are visited oldest-first by {@link SegmentRing}. @@ -717,146 +600,6 @@ void scanRecovery(RecoveredFrameAnalysis analysis) { } } - /** - * Highest {@code deltaStart + deltaCount} (one past the highest symbol id) any - * delta-flagged frame in this segment carries, or {@code 0} only when NO - * delta-flagged frame is present. A frame with {@code deltaCount == 0} (a commit - * or a symbol-reusing frame -- the encoder always sets the delta flag) still - * contributes its {@code deltaStart}, i.e. the producer's baseline at encode - * time, NOT 0. That is deliberate: it anchors the torn-dict guard at that - * baseline so a dictionary torn below it is detected -- a conservative - * over-strand (it may over-reject a frame whose rows reference only surviving - * ids), never an under-strand that could silently shift the dense id map. - * Read-only walk over the recovered frames, used - * once at recovery to detect a persisted {@code .symbol-dict} torn (host crash, - * out-of-order page loss) below the ids the surviving frames still reference: if - * the max here reaches at or beyond the recovered dictionary size, a resuming - * producer -- seeded from the shorter dictionary -- would re-use ids the frames - * already define. Only frames at or below {@code maxFsnInclusive} count: frames - * above it are the aborted orphan-deferred tail, which {@code trySendOne} retires - * without ever transmitting, so their ids must not inflate the result (a resuming - * producer never reuses them on the wire). The frame layout mirrors - * {@link #findLastFrameFsnWithoutPayloadFlag}: the QWP message header - * ({@code qwpHeaderSize} bytes) is followed by the delta section - * {@code [deltaStart varint][deltaCount varint]...}. - * - * @param headerMagic the QWP message magic identifying a well-formed frame - * @param flagsOffset byte offset of the flags field within the QWP header - * @param flagDeltaMask the FLAG_DELTA_SYMBOL_DICT bit - * @param qwpHeaderSize the QWP message header size (delta section starts past it) - * @param maxFsnInclusive highest frame FSN to consider; frames above it are the - * retired orphan-deferred tail -- pass - * {@code recoveredCommitBoundaryFsn} - */ - public long maxSymbolDeltaEnd(int headerMagic, int flagsOffset, int flagDeltaMask, int qwpHeaderSize, long maxFsnInclusive) { - long maxEnd = 0L; - long off = HEADER_SIZE; - long frames = frameCount; - for (long i = 0; i < frames; i++) { - long fsn = baseSeq + i; - int payloadLen = Unsafe.getUnsafe().getInt(mmapAddress + off + 4); - long payload = mmapAddress + off + FRAME_HEADER_SIZE; - // Skip the orphan-deferred tail (fsn > maxFsnInclusive): retired, never - // sent, so its ids must not count. off still advances below. - if (fsn <= maxFsnInclusive - && payloadLen >= qwpHeaderSize - && payloadLen > flagsOffset - && Unsafe.getUnsafe().getInt(payload) == headerMagic - && (Unsafe.getUnsafe().getByte(payload + flagsOffset) & flagDeltaMask) != 0) { - long p = payload + qwpHeaderSize; - long limit = payload + payloadLen; - long deltaStart = 0L; - int shift = 0; - while (p < limit) { - byte b = Unsafe.getUnsafe().getByte(p++); - deltaStart |= (long) (b & 0x7F) << shift; - if ((b & 0x80) == 0) { - break; - } - shift += 7; - if (shift > 35) { - break; // corrupt run; recovered frames are CRC-valid, so defensive only - } - } - long deltaCount = 0L; - shift = 0; - while (p < limit) { - byte b = Unsafe.getUnsafe().getByte(p++); - deltaCount |= (long) (b & 0x7F) << shift; - if ((b & 0x80) == 0) { - break; - } - shift += 7; - if (shift > 35) { - break; - } - } - long end = deltaStart + deltaCount; - if (end > maxEnd) { - maxEnd = end; - } - } - off += FRAME_HEADER_SIZE + payloadLen; - } - return maxEnd; - } - - /** - * Highest {@code deltaStart} any delta-flagged frame at or below - * {@code maxFsnInclusive} carries, or {@code 0} when NO delta-flagged frame is - * present. A result of {@code 0} means every surviving symbol-bearing frame is - * SELF-SUFFICIENT -- it re-registers its dictionary from id 0 (a full-dict - * frame), so the slot replays with no persisted dictionary at all; a result - * {@code > 0} means at least one frame is a non-self-sufficient delta whose ids - * depend on a prior frame's (or the persisted dictionary's) registrations. - * {@link CursorSendEngine} pairs this with {@link #maxSymbolDeltaEnd} at - * recovery: when the persisted {@code .symbol-dict} is a subset of the ids the - * frames reference BUT this returns {@code 0}, the slot was written in full-dict - * fallback (the dictionary never opened when writing) and recovers in full-dict - * mode instead of failing clean; a torn DELTA dictionary has a {@code - * deltaStart > 0} frame and still fails clean. Same read-only frame walk and - * layout as {@link #maxSymbolDeltaEnd}; only frames at or below {@code - * maxFsnInclusive} count (the retired orphan-deferred tail is excluded). - */ - public long maxSymbolDeltaStart(int headerMagic, int flagsOffset, int flagDeltaMask, int qwpHeaderSize, long maxFsnInclusive) { - long maxStart = 0L; - long off = HEADER_SIZE; - long frames = frameCount; - for (long i = 0; i < frames; i++) { - long fsn = baseSeq + i; - int payloadLen = Unsafe.getUnsafe().getInt(mmapAddress + off + 4); - long payload = mmapAddress + off + FRAME_HEADER_SIZE; - // Skip the orphan-deferred tail (fsn > maxFsnInclusive): retired, never - // sent, so its baseline must not count. off still advances below. - if (fsn <= maxFsnInclusive - && payloadLen >= qwpHeaderSize - && payloadLen > flagsOffset - && Unsafe.getUnsafe().getInt(payload) == headerMagic - && (Unsafe.getUnsafe().getByte(payload + flagsOffset) & flagDeltaMask) != 0) { - long p = payload + qwpHeaderSize; - long limit = payload + payloadLen; - long deltaStart = 0L; - int shift = 0; - while (p < limit) { - byte b = Unsafe.getUnsafe().getByte(p++); - deltaStart |= (long) (b & 0x7F) << shift; - if ((b & 0x80) == 0) { - break; - } - shift += 7; - if (shift > 35) { - break; // corrupt run; recovered frames are CRC-valid, so defensive only - } - } - if (deltaStart > maxStart) { - maxStart = deltaStart; - } - } - off += FRAME_HEADER_SIZE + payloadLen; - } - return maxStart; - } - /** * Number of frames written since {@link #create} (or recovered by * {@link #openExisting}). Used by {@code SegmentRing} to compute diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/RecoveredFrameAnalysis.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/RecoveredFrameAnalysis.java index 25d2d431..0977c833 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/RecoveredFrameAnalysis.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/RecoveredFrameAnalysis.java @@ -27,7 +27,6 @@ import io.questdb.client.cutlass.qwp.client.GlobalSymbolDictionary; import io.questdb.client.cutlass.qwp.protocol.QwpConstants; import io.questdb.client.std.MemoryTag; -import io.questdb.client.std.ObjList; import io.questdb.client.std.QuietCloseable; import io.questdb.client.std.Unsafe; import io.questdb.client.std.str.Utf8s; @@ -96,15 +95,11 @@ void accept(long fsn, long payload, int payloadLen) { } } - void appendDecodedSymbols(ObjList out) { - decodeSymbols(null, out); - } - void addDecodedSymbolsTo(GlobalSymbolDictionary target) { - decodeSymbols(target, null); + decodeSymbols(target); } - private void decodeSymbols(GlobalSymbolDictionary target, ObjList out) { + private void decodeSymbols(GlobalSymbolDictionary target) { long p = rawAddr; long limit = rawAddr + committedRawLen; for (int i = 0; i < committedRawCount; i++) { @@ -118,12 +113,7 @@ private void decodeSymbols(GlobalSymbolDictionary target, ObjList out) { if (symbolLen > limit - p) { throw new IllegalStateException("truncated cached symbol dictionary suffix"); } - String symbol = Utf8s.stringFromUtf8Bytes(p, p + symbolLen); - if (target != null) { - target.addRecoveredSymbol(symbol); - } else { - out.add(symbol); - } + target.addRecoveredSymbol(Utf8s.stringFromUtf8Bytes(p, p + symbolLen)); p += symbolLen; } if (p != limit) { diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java index aafaabbb..6e438ac1 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java @@ -616,41 +616,6 @@ public synchronized long findLastFsnWithoutPayloadFlag(int flagsOffset, int flag return Math.max(best, fsn); } - /** - * Rebuilds the symbols a replay of {@code [minFsnInclusive .. maxFsnInclusive]} would - * register ABOVE {@code baseline}, appending them to {@code out} in ascending id order. - * Returns the coverage the replay establishes, or {@code -1} on a genuine gap. See - * {@link MmapSegment#collectReplaySymbolsAbove}. - *

      - * Walks the segments in ascending FSN order -- {@code sealedSegments} are held in - * baseSeq order and {@code active} is the newest (see {@code recover}) -- threading the - * coverage through, because unlike {@link #maxSymbolDeltaEnd} (a plain max) this walk is - * ORDER-DEPENDENT: each frame may only extend a dictionary the frames before it built. - */ - public synchronized long collectReplaySymbolsAbove( - int headerMagic, - int flagsOffset, - int flagDeltaMask, - int qwpHeaderSize, - long minFsnInclusive, - long maxFsnInclusive, - long baseline, - ObjList out - ) { - long coverage = baseline; - for (int i = sealedHead, n = sealedSegments.size(); i < n; i++) { - coverage = sealedSegments.get(i).collectReplaySymbolsAbove( - headerMagic, flagsOffset, flagDeltaMask, qwpHeaderSize, - minFsnInclusive, maxFsnInclusive, coverage, out); - if (coverage < 0) { - return -1L; - } - } - return active.collectReplaySymbolsAbove( - headerMagic, flagsOffset, flagDeltaMask, qwpHeaderSize, - minFsnInclusive, maxFsnInclusive, coverage, out); - } - /** * Performs the one ordered recovery fold across sealed segments and the * active segment. The returned native suffix remains owned by the caller. @@ -670,45 +635,6 @@ synchronized RecoveredFrameAnalysis analyzeRecovery(int symbolBaseline) { } } - /** - * Highest {@code deltaStart + deltaCount} any symbol-dict delta frame at or below - * {@code maxFsnInclusive} references (0 when none). See - * {@link MmapSegment#maxSymbolDeltaEnd}; used once at recovery to detect a - * persisted dictionary torn below the ids the surviving committed frames - * reference. Frames above {@code maxFsnInclusive} are the retired orphan-deferred - * tail and are excluded. - */ - public synchronized long maxSymbolDeltaEnd(int headerMagic, int flagsOffset, int flagDeltaMask, int qwpHeaderSize, long maxFsnInclusive) { - long maxEnd = 0L; - for (int i = sealedHead, n = sealedSegments.size(); i < n; i++) { - long end = sealedSegments.get(i).maxSymbolDeltaEnd(headerMagic, flagsOffset, flagDeltaMask, qwpHeaderSize, maxFsnInclusive); - if (end > maxEnd) { - maxEnd = end; - } - } - long end = active.maxSymbolDeltaEnd(headerMagic, flagsOffset, flagDeltaMask, qwpHeaderSize, maxFsnInclusive); - return Math.max(maxEnd, end); - } - - /** - * Highest {@code deltaStart} any symbol-dict delta frame at or below - * {@code maxFsnInclusive} carries (0 when none, or when every such frame is - * self-sufficient). See {@link MmapSegment#maxSymbolDeltaStart}; paired with - * {@link #maxSymbolDeltaEnd} at recovery to tell a full-dict-fallback slot - * (recoverable with no dictionary) from a torn delta dictionary (fails clean). - */ - public synchronized long maxSymbolDeltaStart(int headerMagic, int flagsOffset, int flagDeltaMask, int qwpHeaderSize, long maxFsnInclusive) { - long maxStart = 0L; - for (int i = sealedHead, n = sealedSegments.size(); i < n; i++) { - long start = sealedSegments.get(i).maxSymbolDeltaStart(headerMagic, flagsOffset, flagDeltaMask, qwpHeaderSize, maxFsnInclusive); - if (start > maxStart) { - maxStart = start; - } - } - long start = active.maxSymbolDeltaStart(headerMagic, flagsOffset, flagDeltaMask, qwpHeaderSize, maxFsnInclusive); - return Math.max(maxStart, start); - } - public MmapSegment getActive() { return active; } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopOrphanTailTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopOrphanTailTest.java index ef5861ed..26e6e124 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopOrphanTailTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopOrphanTailTest.java @@ -37,7 +37,6 @@ import io.questdb.client.network.PlainSocketFactory; import io.questdb.client.std.Files; import io.questdb.client.std.MemoryTag; -import io.questdb.client.std.ObjList; import io.questdb.client.std.Unsafe; import io.questdb.client.test.tools.TestUtils; import org.junit.After; @@ -398,12 +397,12 @@ public void testRecoveryScansFramesOnceAndReusesCachedSymbolSuffix() throws Exce try (CursorSendEngine engine = newEngine()) { assertEquals("one recovery visit per frame", 3L, engine.recoveryFramesVisited()); - ObjList first = new ObjList<>(); - ObjList second = new ObjList<>(); - assertEquals(3L, engine.collectReplaySymbolsAbove(0, first)); - assertEquals(3L, engine.collectReplaySymbolsAbove(0, second)); - assertEquals("a", first.getQuick(0)); - assertEquals("c", second.getQuick(2)); + GlobalSymbolDictionary first = new GlobalSymbolDictionary(); + GlobalSymbolDictionary second = new GlobalSymbolDictionary(); + assertEquals(3L, engine.addRecoveredSymbolsTo(0, first)); + assertEquals(3L, engine.addRecoveredSymbolsTo(0, second)); + assertEquals("a", first.getSymbol(0)); + assertEquals("c", second.getSymbol(2)); assertEquals("producer seed reads must reuse the recovery result", 3L, engine.recoveryFramesVisited()); @@ -467,11 +466,11 @@ public void testSelfSufficientFrameRepairsAckedRecoveryGap() throws Exception { try (CursorSendEngine engine = newEngine()) { assertEquals(0L, engine.ackedFsn()); - ObjList recovered = new ObjList<>(); + GlobalSymbolDictionary recovered = new GlobalSymbolDictionary(); assertEquals("the full frame must re-anchor replay coverage", - 1L, engine.collectReplaySymbolsAbove(0, recovered)); + 1L, engine.addRecoveredSymbolsTo(0, recovered)); assertEquals(1, recovered.size()); - assertEquals("a", recovered.getQuick(0)); + assertEquals("a", recovered.getSymbol(0)); } }); } @@ -488,9 +487,9 @@ public void testSelfSufficientFrameCannotHideUnackedRecoveryGap() throws Excepti appendDeltaSymbolFrame(engine, 0, 'a'); } try (CursorSendEngine engine = newEngine()) { - ObjList recovered = new ObjList<>(); + GlobalSymbolDictionary recovered = new GlobalSymbolDictionary(); assertEquals("an unacked gap remains unreplayable", - -1L, engine.collectReplaySymbolsAbove(0, recovered)); + -1L, engine.addRecoveredSymbolsTo(0, recovered)); assertEquals(0, recovered.size()); } }); diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/RecoveredFrameAnalysisTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/RecoveredFrameAnalysisTest.java index 171df57d..d6ec8151 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/RecoveredFrameAnalysisTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/RecoveredFrameAnalysisTest.java @@ -24,10 +24,10 @@ package io.questdb.client.test.cutlass.qwp.client.sf.cursor; +import io.questdb.client.cutlass.qwp.client.GlobalSymbolDictionary; import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine; import io.questdb.client.cutlass.qwp.protocol.QwpConstants; import io.questdb.client.std.MemoryTag; -import io.questdb.client.std.ObjList; import io.questdb.client.std.Unsafe; import org.junit.Assert; import org.junit.Rule; @@ -85,9 +85,9 @@ private void assertMalformedDeltaMarksGap(byte[] deltaSection) throws Exception } try (CursorSendEngine recovered = new CursorSendEngine(slot.toString(), 4_096)) { - ObjList symbols = new ObjList<>(); + GlobalSymbolDictionary symbols = new GlobalSymbolDictionary(); Assert.assertEquals("malformed recovered delta must be fail-clean", - -1L, recovered.collectReplaySymbolsAbove(0, symbols)); + -1L, recovered.addRecoveredSymbolsTo(0, symbols)); Assert.assertEquals("malformed delta must not recover partial symbols", 0, symbols.size()); Assert.assertEquals("the malformed frame must be visited during recovery", 1L, recovered.recoveryFramesVisited()); From 0a9c3d6b6be541c54b9b7b51832f5a3e0c574482 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Sat, 18 Jul 2026 01:03:34 +0100 Subject: [PATCH 94/96] De-reflect cursor tests; de-flake poison timing Two test-quality fixes in the send-loop cursor suite; no production behavior change. De-reflection. CatchUpAlignmentTest, PoisonFrameTest, TornDictGuardTest and MirrorLeakTest reached into CursorWebSocketSendLoop by reflection -- reading private constants and fields, invoking private methods, and writing private state -- so a rename broke them as a runtime ReflectiveOperationException instead of a compile error. Add @TestOnly accessors and thin ...ForTest wrappers on CursorWebSocketSendLoop, each delegating verbatim to the same private member, and make forceMirrorSeedFailureForTest public; then rewire the four tests to call them. The rewrites drive the identical production paths, and the mirror-overflow guard now catches the exact public LineSenderException type rather than unwrapping an InvocationTargetException. De-flake. PoisonFrameTest asserted absolute wall-clock bounds -- a recycle is "immediate" under 300 ms, and a recycle count fits a fixed sleep window -- that a CI GC pause or starved scheduler can trip. Replace them with predicates only a real regression fails: consecutive recycle gaps must clear 3/4 of the reconnect backoff (an unpaced loop's sub-millisecond gaps fail, while a slow test thread only yields more paced samples); the deterministic zeroProgressRecycles branch value (level 0, then 1, then reset on progress); a first*2 <= second relative bound; and the kept second >= backoff lower bound, which starvation can only lengthen. Split the dwell test into a huge-window case (threshold reached, window still open, so no escalation -- asserting poisonStrikes == 3) and a small-window case crossed by a sleep strictly longer than the window, which must escalate to the PROTOCOL_VIOLATION terminal. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../sf/cursor/CursorWebSocketSendLoop.java | 146 ++++++++- ...WebSocketSendLoopCatchUpAlignmentTest.java | 225 +++++-------- ...CursorWebSocketSendLoopMirrorLeakTest.java | 54 +--- ...ursorWebSocketSendLoopPoisonFrameTest.java | 299 +++++++++--------- ...sorWebSocketSendLoopTornDictGuardTest.java | 20 +- 5 files changed, 384 insertions(+), 360 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java index 431eca59..8d42af75 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java @@ -206,7 +206,7 @@ public final class CursorWebSocketSendLoop implements QuietCloseable { // deleted. Production never sets it; volatile only so a test thread's write is // visible to the loop under test. @TestOnly - static volatile boolean forceMirrorSeedFailureForTest; + public static volatile boolean forceMirrorSeedFailureForTest; // Pre-converted to nanos for the comparison in sendDurableAckKeepaliveIfDue. // Zero or negative disables the keepalive entirely. private final long durableAckKeepaliveIntervalNanos; @@ -1091,6 +1091,16 @@ public static boolean isTerminalCloseCode(int code) { } } + @TestOnly + public static int maxCatchUpCapGapAttempts() { + return MAX_CATCHUP_CAP_GAP_ATTEMPTS; + } + + @TestOnly + public static int maxSentDictBytes() { + return MAX_SENT_DICT_BYTES; + } + /** * Surfaces any error the I/O thread recorded. Called by the producer * thread (typically from inside its append wrapper) so failures don't @@ -2895,6 +2905,140 @@ public int catchUpFrameGrowthCount() { return catchUpFrameGrowthCount; } + @TestOnly + public int catchUpCapGapAttempts() { + return catchUpCapGapAttempts; + } + + @TestOnly + public long catchUpCapGapFirstNanos() { + return catchUpCapGapFirstNanos; + } + + @TestOnly + public long catchUpCapGapMinEscalationWindowNanos() { + return catchUpCapGapMinEscalationWindowNanos; + } + + @TestOnly + public long fsnAtZero() { + return fsnAtZero; + } + + @TestOnly + public long poisonFsn() { + return poisonFsn; + } + + @TestOnly + public int poisonStrikes() { + return poisonStrikes; + } + + @TestOnly + public long sentDictBytesAddr() { + return sentDictBytesAddr; + } + + @TestOnly + public int sentDictBytesLen() { + return sentDictBytesLen; + } + + @TestOnly + public boolean sentDictBytesOwned() { + return sentDictBytesOwned; + } + + @TestOnly + public int sentDictCount() { + return sentDictCount; + } + + @TestOnly + public int zeroProgressRecycles() { + return zeroProgressRecycles; + } + + @TestOnly + public void accumulateSentDictForTest(long payloadAddr, int payloadLen, int deltaStart) { + accumulateSentDict(payloadAddr, payloadLen, deltaStart); + } + + @TestOnly + public void connectLoopForTest(Throwable initial, String phase, long paceFirstAttemptMillis) { + connectLoop(initial, phase, paceFirstAttemptMillis); + } + + @TestOnly + public void deliverCloseForTest(int code, String reason) { + responseHandler.onClose(code, reason); + } + + @TestOnly + public void deliverResponseForTest(long payloadPtr, int payloadLen) { + responseHandler.onBinaryMessage(payloadPtr, payloadLen); + } + + @TestOnly + public void ensureSentDictCapacityForTest(long required) { + ensureSentDictCapacity(required); + } + + @TestOnly + public void positionCursorForStartForTest() { + positionCursorForStart(); + } + + @TestOnly + public void seedSentDictMirrorForTest(long addr, int bytes, int symbolCount) { + sentDictBytesAddr = addr; + sentDictBytesCapacity = bytes; + sentDictBytesLen = bytes; + sentDictCount = symbolCount; + sentDictBytesOwned = true; + } + + @TestOnly + public void sendCatchUpChunkForTest(int deltaStart, int deltaCount, long symbolsAddr, int symbolsLen) { + sendCatchUpChunk(deltaStart, deltaCount, symbolsAddr, symbolsLen); + } + + @TestOnly + public void setCatchUpCapGapFirstNanosForTest(long nanos) { + catchUpCapGapFirstNanos = nanos; + } + + @TestOnly + public void setDataFrameSentThisConnectionForTest(boolean value) { + dataFrameSentThisConnection = value; + } + + @TestOnly + public void setFsnAtZeroForTest(long value) { + fsnAtZero = value; + } + + @TestOnly + public void setNextWireSeqForTest(long value) { + nextWireSeq = value; + } + + @TestOnly + public void setRunningForTest(boolean value) { + running = value; + } + + @TestOnly + public void setWireBaselineWithCatchUpForTest(long replayStart) { + setWireBaselineWithCatchUp(replayStart); + } + + @TestOnly + public boolean trySendOneForTest() { + return trySendOne(); + } + private void ensureCatchUpFrameCapacity(int required) { if (catchUpFrameCapacity >= required) { return; diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java index ac6dad09..f21560ce 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopCatchUpAlignmentTest.java @@ -39,9 +39,6 @@ import org.junit.Before; import org.junit.Test; -import java.lang.reflect.Field; -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Arrays; @@ -118,7 +115,7 @@ public void testCatchUpFrameAckDoesNotAdvanceTrimWatermark() throws Exception { 1L, engine.ackedFsn()); // Mechanism: the catch-up frames are anchored below replayStart. assertEquals("fsnAtZero must be anchored catchUpFrames below replayStart", - replayStart - client.framesSent, readLong(loop, "fsnAtZero")); + replayStart - client.framesSent, loop.fsnAtZero()); } finally { loop.close(); // frees the seeded mirror + the stub client's buffers } @@ -154,7 +151,7 @@ public void testSplitCatchUpFramesAcksDoNotAdvanceTrimWatermark() throws Excepti assertEquals("no catch-up frame ACK may advance the trim watermark", 2L, engine.ackedFsn()); assertEquals("fsnAtZero must subtract the full split frame count", - replayStart - client.framesSent, readLong(loop, "fsnAtZero")); + replayStart - client.framesSent, loop.fsnAtZero()); } finally { loop.close(); } @@ -229,9 +226,9 @@ public void testTransientCatchUpSendFailureIsRetriableNotTerminal() throws Excep invokeSetWireBaselineWithCatchUp(loop, engine.ackedFsn() + 1L); fail("a transient catch-up send failure must raise a retriable " + "CatchUpSendException, not be swallowed into fail()/a terminal"); - } catch (InvocationTargetException e) { + } catch (RuntimeException e) { assertEquals("transient catch-up send failure must surface as CatchUpSendException", - "CatchUpSendException", e.getCause().getClass().getSimpleName()); + "CatchUpSendException", e.getClass().getSimpleName()); } // Retriable, not terminal: the producer-facing error latch stays clear. loop.checkError(); @@ -261,15 +258,12 @@ public void testAccumulateSentDictPartialOverlapExtendsMirror() throws Exception int[] frameLen = new int[1]; long frame = buildDeltaFrame(0, new String[]{"aa", "bb", "cc"}, frameLen); try { - Method m = CursorWebSocketSendLoop.class.getDeclaredMethod( - "accumulateSentDict", long.class, int.class, int.class); - m.setAccessible(true); - m.invoke(loop, frame, frameLen[0], 0); + loop.accumulateSentDictForTest(frame, frameLen[0], 0); } finally { Unsafe.free(frame, frameLen[0], MemoryTag.NATIVE_DEFAULT); } assertEquals("straddling delta must extend the mirror to all 3 ids", - 3, readInt(loop, "sentDictCount")); + 3, loop.sentDictCount()); assertEquals("mirror must hold the two new tail symbols after the " + "already-held prefix, gap-free", Arrays.asList("aa", "bb", "cc"), readMirrorSymbols(loop)); @@ -293,18 +287,15 @@ public void testCatchUpChunkFrameSizeOverflowFailsLoud() throws Exception { try (CursorSendEngine engine = newEngine()) { CursorWebSocketSendLoop loop = newLoop(engine, client); try { - Method m = CursorWebSocketSendLoop.class.getDeclaredMethod( - "sendCatchUpChunk", int.class, int.class, long.class, int.class); - m.setAccessible(true); // symbolsLen past the mirror ceiling: HEADER + varints + symbolsLen // overflows an int, so the guard must reject it before malloc. - m.invoke(loop, 0, 1, 0L, Integer.MAX_VALUE - 4); + loop.sendCatchUpChunkForTest(0, 1, 0L, Integer.MAX_VALUE - 4); fail("an overflowing catch-up frame size must fail loud, not malloc negative"); - } catch (InvocationTargetException e) { + } catch (RuntimeException e) { assertEquals("overflow must surface as CatchUpSendException", - "CatchUpSendException", e.getCause().getClass().getSimpleName()); - assertTrue("message must name the frame-size guard: " + e.getCause().getMessage(), - e.getCause().getMessage().contains("catch-up frame exceeds the maximum size")); + "CatchUpSendException", e.getClass().getSimpleName()); + assertTrue("message must name the frame-size guard: " + e.getMessage(), + e.getMessage().contains("catch-up frame exceeds the maximum size")); } finally { loop.close(); } @@ -318,9 +309,7 @@ public void testForegroundCatchUpCapGapRetriesPastOrphanBudget() throws Exceptio // quarantine budget. Drive more cap gaps than that entire budget and assert every // failure remains retriable to the I/O loop and invisible to the producer. TestUtils.assertMemoryLeak(() -> { - Field maxField = CursorWebSocketSendLoop.class.getDeclaredField("MAX_CATCHUP_CAP_GAP_ATTEMPTS"); - maxField.setAccessible(true); - int maxAttempts = maxField.getInt(null); + int maxAttempts = CursorWebSocketSendLoop.maxCatchUpCapGapAttempts(); CatchUpCapturingClient client = new CatchUpCapturingClient(160); try (CursorSendEngine engine = newEngine()) { CursorWebSocketSendLoop loop = newForegroundLoop(engine, client); @@ -330,15 +319,15 @@ public void testForegroundCatchUpCapGapRetriesPastOrphanBudget() throws Exceptio try { invokeSetWireBaselineWithCatchUp(loop, engine.ackedFsn() + 1L); fail("cap gap must raise a retriable CatchUpSendException (attempt " + i + ')'); - } catch (InvocationTargetException e) { - assertEquals("CatchUpSendException", e.getCause().getClass().getSimpleName()); + } catch (RuntimeException e) { + assertEquals("CatchUpSendException", e.getClass().getSimpleName()); } loop.checkError(); } assertEquals("foreground retries must not burn the orphan attempt budget", - 0, readInt(loop, "catchUpCapGapAttempts")); + 0, loop.catchUpCapGapAttempts()); assertEquals("foreground retries must not anchor an orphan cap-gap episode", - -1L, readLong(loop, "catchUpCapGapFirstNanos")); + -1L, loop.catchUpCapGapFirstNanos()); } finally { loop.close(); } @@ -358,9 +347,7 @@ public void testCatchUpCapGapStrikesAloneDoNotLatchWithinTheEscalationWindow() t // exists to ride out. Here we drive far MORE than the budget's strikes inside a // deliberately huge window and assert the orphan loop stays retriable. TestUtils.assertMemoryLeak(() -> { - Field maxField = CursorWebSocketSendLoop.class.getDeclaredField("MAX_CATCHUP_CAP_GAP_ATTEMPTS"); - maxField.setAccessible(true); - int maxAttempts = maxField.getInt(null); + int maxAttempts = CursorWebSocketSendLoop.maxCatchUpCapGapAttempts(); CatchUpCapturingClient client = new CatchUpCapturingClient(160); try (CursorSendEngine engine = newEngine()) { // A one-hour dwell the test cannot possibly elapse. @@ -371,28 +358,26 @@ public void testCatchUpCapGapStrikesAloneDoNotLatchWithinTheEscalationWindow() t try { invokeSetWireBaselineWithCatchUp(loop, engine.ackedFsn() + 1L); fail("cap gap must raise a retriable CatchUpSendException (attempt " + i + ')'); - } catch (InvocationTargetException e) { - assertEquals("CatchUpSendException", e.getCause().getClass().getSimpleName()); + } catch (RuntimeException e) { + assertEquals("CatchUpSendException", e.getClass().getSimpleName()); } // The producer-facing latch must stay clear on EVERY attempt, // including the ones past the strike budget. loop.checkError(); } assertTrue("the strikes really did exceed the budget", - readInt(loop, "catchUpCapGapAttempts") > maxAttempts); + loop.catchUpCapGapAttempts() > maxAttempts); // Backdate the episode anchor past the window: the very next cap gap // now satisfies BOTH conditions and latches. This pins the AND -- if // escalation ignored the wall clock the loop would already have // latched above; if it ignored the strike count it could never latch. - Field anchor = CursorWebSocketSendLoop.class.getDeclaredField("catchUpCapGapFirstNanos"); - anchor.setAccessible(true); - anchor.setLong(loop, System.nanoTime() - TimeUnit.HOURS.toNanos(2)); + loop.setCatchUpCapGapFirstNanosForTest(System.nanoTime() - TimeUnit.HOURS.toNanos(2)); try { invokeSetWireBaselineWithCatchUp(loop, engine.ackedFsn() + 1L); fail("the escalating cap gap must still raise CatchUpSendException"); - } catch (InvocationTargetException e) { - assertEquals("CatchUpSendException", e.getCause().getClass().getSimpleName()); + } catch (RuntimeException e) { + assertEquals("CatchUpSendException", e.getClass().getSimpleName()); } try { loop.checkError(); @@ -418,7 +403,7 @@ public void testCatchUpCapGapDwellConversionSaturatesInsteadOfOverflowing() thro engine, new CatchUpCapturingClient(0), maxExactMillis); try { assertEquals(maxExactMillis * 1_000_000L, - readLong(exactLoop, "catchUpCapGapMinEscalationWindowNanos")); + exactLoop.catchUpCapGapMinEscalationWindowNanos()); } finally { exactLoop.close(); } @@ -428,7 +413,7 @@ public void testCatchUpCapGapDwellConversionSaturatesInsteadOfOverflowing() thro try { assertEquals("an oversized dwell must become effectively infinite, not negative", Long.MAX_VALUE, - readLong(saturatedLoop, "catchUpCapGapMinEscalationWindowNanos")); + saturatedLoop.catchUpCapGapMinEscalationWindowNanos()); } finally { saturatedLoop.close(); } @@ -453,9 +438,7 @@ public void testCapGapEpisodeWithANegativeAnchorStillEscalates() throws Exceptio // a long-lived dev box (which is why it passed locally). Planting the negative anchor // directly pins the sentinel on ANY machine, whatever its uptime. TestUtils.assertMemoryLeak(() -> { - Field maxField = CursorWebSocketSendLoop.class.getDeclaredField("MAX_CATCHUP_CAP_GAP_ATTEMPTS"); - maxField.setAccessible(true); - int maxAttempts = maxField.getInt(null); + int maxAttempts = CursorWebSocketSendLoop.maxCatchUpCapGapAttempts(); CatchUpCapturingClient client = new CatchUpCapturingClient(160); try (CursorSendEngine engine = newEngine()) { // A one-hour dwell, against an anchor two hours back: satisfied on elapsed, @@ -468,22 +451,20 @@ public void testCapGapEpisodeWithANegativeAnchorStillEscalates() throws Exceptio try { invokeSetWireBaselineWithCatchUp(loop, engine.ackedFsn() + 1L); fail("cap gap must raise a retriable CatchUpSendException (attempt " + i + ')'); - } catch (InvocationTargetException e) { - assertEquals("CatchUpSendException", e.getCause().getClass().getSimpleName()); + } catch (RuntimeException e) { + assertEquals("CatchUpSendException", e.getClass().getSimpleName()); } loop.checkError(); // dwell unmet => retriable, whatever the count } // The episode began two hours ago on a machine booted minutes ago. - Field anchor = CursorWebSocketSendLoop.class.getDeclaredField("catchUpCapGapFirstNanos"); - anchor.setAccessible(true); - anchor.setLong(loop, -TimeUnit.HOURS.toNanos(2)); + loop.setCatchUpCapGapFirstNanosForTest(-TimeUnit.HOURS.toNanos(2)); // Both halves now hold, so this strike must latch the terminal. try { invokeSetWireBaselineWithCatchUp(loop, engine.ackedFsn() + 1L); fail("the escalating cap gap must still raise CatchUpSendException"); - } catch (InvocationTargetException e) { - assertEquals("CatchUpSendException", e.getCause().getClass().getSimpleName()); + } catch (RuntimeException e) { + assertEquals("CatchUpSendException", e.getClass().getSimpleName()); } try { loop.checkError(); @@ -514,9 +495,7 @@ public void testTransientCatchUpFailureDoesNotBurnTheCapGapBudget() throws Excep // 16-strike budget anyway. So drive MORE transients than the whole budget and // assert the counter never moves and no terminal ever latches. TestUtils.assertMemoryLeak(() -> { - Field maxField = CursorWebSocketSendLoop.class.getDeclaredField("MAX_CATCHUP_CAP_GAP_ATTEMPTS"); - maxField.setAccessible(true); - int maxAttempts = maxField.getInt(null); + int maxAttempts = CursorWebSocketSendLoop.maxCatchUpCapGapAttempts(); // A cap that FITS (no cap gap), but whose sendBinary always throws: every // failure here is transport-transient, never a capability gap. CatchUpCapturingClient client = new CatchUpCapturingClient(0, true); @@ -528,14 +507,14 @@ public void testTransientCatchUpFailureDoesNotBurnTheCapGapBudget() throws Excep try { invokeSetWireBaselineWithCatchUp(loop, engine.ackedFsn() + 1L); fail("a transient send failure must raise CatchUpSendException (attempt " + i + ')'); - } catch (InvocationTargetException e) { - assertEquals("CatchUpSendException", e.getCause().getClass().getSimpleName()); + } catch (RuntimeException e) { + assertEquals("CatchUpSendException", e.getClass().getSimpleName()); } loop.checkError(); // a transient is retriable, forever assertEquals("a transient must NOT burn the cap-gap terminal budget", - 0, readInt(loop, "catchUpCapGapAttempts")); + 0, loop.catchUpCapGapAttempts()); assertEquals("a transient must NOT anchor a cap-gap episode", - -1L, readLong(loop, "catchUpCapGapFirstNanos")); + -1L, loop.catchUpCapGapFirstNanos()); } } finally { loop.close(); @@ -565,9 +544,7 @@ public void testCatchUpCapGapRetriesUntilBudgetThenLatches() throws Exception { // smaller-cap node quarantined the orphan slot. (A successful catch-up resets the budget; // the other catch-up tests, which use a fitting cap, never trip it.) TestUtils.assertMemoryLeak(() -> { - Field maxField = CursorWebSocketSendLoop.class.getDeclaredField("MAX_CATCHUP_CAP_GAP_ATTEMPTS"); - maxField.setAccessible(true); - int maxAttempts = maxField.getInt(null); + int maxAttempts = CursorWebSocketSendLoop.maxCatchUpCapGapAttempts(); // Pin the budget against a LITERAL before deriving anything from it. The // retriable loop below is bounded by maxAttempts, so keying this test purely // off the constant under test makes it TAUTOLOGICAL: a regression of @@ -592,11 +569,11 @@ public void testCatchUpCapGapRetriesUntilBudgetThenLatches() throws Exception { try { invokeSetWireBaselineWithCatchUp(loop, engine.ackedFsn() + 1L); fail("cap gap must raise a retriable CatchUpSendException (attempt " + i + ')'); - } catch (InvocationTargetException e) { - assertEquals("CatchUpSendException", e.getCause().getClass().getSimpleName()); + } catch (RuntimeException e) { + assertEquals("CatchUpSendException", e.getClass().getSimpleName()); assertTrue("attempt " + i + " must name the catch-up cap gap: " - + e.getCause().getMessage(), - e.getCause().getMessage().contains("during catch-up")); + + e.getMessage(), + e.getMessage().contains("during catch-up")); } loop.checkError(); // under budget => retriable => no terminal } @@ -604,8 +581,8 @@ public void testCatchUpCapGapRetriesUntilBudgetThenLatches() throws Exception { try { invokeSetWireBaselineWithCatchUp(loop, engine.ackedFsn() + 1L); fail("the exhausting cap gap must still raise CatchUpSendException"); - } catch (InvocationTargetException e) { - assertEquals("CatchUpSendException", e.getCause().getClass().getSimpleName()); + } catch (RuntimeException e) { + assertEquals("CatchUpSendException", e.getClass().getSimpleName()); } try { loop.checkError(); @@ -632,9 +609,7 @@ public void testSuccessfulCatchUpResetsCapGapBudget() throws Exception { // testCatchUpCapGapRetriesUntilBudgetThenLatches only accrues gaps under one // fixed cap with no success interleaved, so it cannot pin the reset. TestUtils.assertMemoryLeak(() -> { - Field maxField = CursorWebSocketSendLoop.class.getDeclaredField("MAX_CATCHUP_CAP_GAP_ATTEMPTS"); - maxField.setAccessible(true); - int maxAttempts = maxField.getInt(null); + int maxAttempts = CursorWebSocketSendLoop.maxCatchUpCapGapAttempts(); // Same anti-tautology pin as testCatchUpCapGapRetriesUntilBudgetThenLatches. // With maxAttempts == 1 the accrual loop below would run ZERO times and the // "budget accrued to max-1" precondition would degenerate to 0 == 0, so the @@ -652,19 +627,19 @@ public void testSuccessfulCatchUpResetsCapGapBudget() throws Exception { try { invokeSetWireBaselineWithCatchUp(loop, engine.ackedFsn() + 1L); fail("cap gap must raise a retriable CatchUpSendException (attempt " + i + ')'); - } catch (InvocationTargetException e) { - assertEquals("CatchUpSendException", e.getCause().getClass().getSimpleName()); + } catch (RuntimeException e) { + assertEquals("CatchUpSendException", e.getClass().getSimpleName()); } } assertEquals("precondition: budget accrued to max-1", - maxAttempts - 1, readInt(loop, "catchUpCapGapAttempts")); + maxAttempts - 1, loop.catchUpCapGapAttempts()); // A larger-cap node returns: the whole dictionary re-registers with // no cap gap, so the settle budget must reset to 0. client.cap = 0; // no cap => the 200-char symbol fits one frame invokeSetWireBaselineWithCatchUp(loop, engine.ackedFsn() + 1L); assertEquals("a successful catch-up must reset the cap-gap settle budget", - 0, readInt(loop, "catchUpCapGapAttempts")); + 0, loop.catchUpCapGapAttempts()); // Behavioural proof the budget is genuinely fresh: max-1 more cap // gaps still latch NO terminal (they would if the counter had stayed @@ -674,8 +649,8 @@ public void testSuccessfulCatchUpResetsCapGapBudget() throws Exception { try { invokeSetWireBaselineWithCatchUp(loop, engine.ackedFsn() + 1L); fail("post-reset cap gap must be retriable (attempt " + i + ')'); - } catch (InvocationTargetException e) { - assertEquals("CatchUpSendException", e.getCause().getClass().getSimpleName()); + } catch (RuntimeException e) { + assertEquals("CatchUpSendException", e.getClass().getSimpleName()); } loop.checkError(); // fresh budget => still under max => no terminal } @@ -694,23 +669,19 @@ public void testMirrorOverflowFailsLoud() throws Exception { // (~200M+ symbols on one connection), so drive the guard directly with an // oversized required, mirroring testCatchUpChunkFrameSizeOverflowFailsLoud. TestUtils.assertMemoryLeak(() -> { - Field maxField = CursorWebSocketSendLoop.class.getDeclaredField("MAX_SENT_DICT_BYTES"); - maxField.setAccessible(true); - long overCeiling = (long) maxField.getInt(null) + 1L; + long overCeiling = (long) CursorWebSocketSendLoop.maxSentDictBytes() + 1L; CatchUpCapturingClient client = new CatchUpCapturingClient(0); try (CursorSendEngine engine = newEngine()) { CursorWebSocketSendLoop loop = newLoop(engine, client); try { - Method m = CursorWebSocketSendLoop.class.getDeclaredMethod("ensureSentDictCapacity", long.class); - m.setAccessible(true); try { - m.invoke(loop, overCeiling); + loop.ensureSentDictCapacityForTest(overCeiling); fail("a mirror capacity past MAX_SENT_DICT_BYTES must fail loud, not overflow"); - } catch (InvocationTargetException e) { + } catch (LineSenderException e) { assertEquals("overflow must surface as LineSenderException", - "LineSenderException", e.getCause().getClass().getSimpleName()); - assertTrue("message must name the mirror ceiling: " + e.getCause().getMessage(), - e.getCause().getMessage().contains("mirror exceeds the maximum size")); + "LineSenderException", e.getClass().getSimpleName()); + assertTrue("message must name the mirror ceiling: " + e.getMessage(), + e.getMessage().contains("mirror exceeds the maximum size")); } // recordFatal (not a bare throw) latched the terminal, so the loop // winds down instead of reconnecting into the same overflow. @@ -748,9 +719,7 @@ private void assertUnrelatedReconnectStateRestartsCapGapEpisode(boolean roleReje // outage must end the old episode: its wall-clock duration says nothing about // whether the cluster's batch cap remained incompatible while no node answered. TestUtils.assertMemoryLeak(() -> { - Field maxField = CursorWebSocketSendLoop.class.getDeclaredField("MAX_CATCHUP_CAP_GAP_ATTEMPTS"); - maxField.setAccessible(true); - int maxAttempts = maxField.getInt(null); + int maxAttempts = CursorWebSocketSendLoop.maxCatchUpCapGapAttempts(); assertTrue("the cap-gap settle budget must have a retriable interval", maxAttempts > 1); int[] reconnectCalls = {0}; @@ -767,12 +736,12 @@ private void assertUnrelatedReconnectStateRestartsCapGapEpisode(boolean roleReje if (call == maxAttempts) { assertEquals("precondition: consecutive cap gaps survive reconnect", maxAttempts - 1, - readInt(loopRef[0], "catchUpCapGapAttempts")); + loopRef[0].catchUpCapGapAttempts()); // Model the elapsed outage without sleeping. With the defect, // this old anchor survives the unrelated failure and the next // cap gap immediately satisfies both terminal conditions. staleAnchor[0] = System.nanoTime() - TimeUnit.HOURS.toNanos(2); - setField(loopRef[0], "catchUpCapGapFirstNanos", staleAnchor[0]); + loopRef[0].setCatchUpCapGapFirstNanosForTest(staleAnchor[0]); if (roleReject) { throw new QwpRoleMismatchException( "PRIMARY", null, "all endpoints role-rejected"); @@ -783,7 +752,7 @@ private void assertUnrelatedReconnectStateRestartsCapGapEpisode(boolean roleReje // Stop after getServerMaxBatchSize() has driven the final cap // gap, leaving its fresh episode state observable below. return new CatchUpCapturingClient(160, false, - () -> setBooleanFieldUnchecked(loopRef[0], "running", false)); + () -> loopRef[0].setRunningForTest(false)); } throw new AssertionError("unexpected reconnect call " + call); }, @@ -795,14 +764,14 @@ private void assertUnrelatedReconnectStateRestartsCapGapEpisode(boolean roleReje loopRef[0] = loop; try { seedMirror(loop, TestUtils.repeat("x", 200)); - setBooleanField(loop, "running", true); + loop.setRunningForTest(true); invokeConnectLoop(loop); loop.checkError(); assertEquals("pre-outage cap gaps must not carry into the new episode", - 1, readInt(loop, "catchUpCapGapAttempts")); + 1, loop.catchUpCapGapAttempts()); assertTrue("the post-outage cap gap must get a fresh dwell anchor", - readLong(loop, "catchUpCapGapFirstNanos") > staleAnchor[0]); + loop.catchUpCapGapFirstNanos() > staleAnchor[0]); assertEquals("test must observe gaps, the unrelated state, and a new gap", maxAttempts + 1, reconnectCalls[0]); } finally { @@ -836,17 +805,11 @@ private static long buildDeltaFrame(int deltaStart, String[] symbols, int[] outL return addr; } - private static int readInt(CursorWebSocketSendLoop loop, String name) throws Exception { - Field f = CursorWebSocketSendLoop.class.getDeclaredField(name); - f.setAccessible(true); - return f.getInt(loop); - } - // Parses the loop's native sent-dictionary mirror ([len varint][utf8]...) back // into the symbol strings a reconnect catch-up would re-register. private static List readMirrorSymbols(CursorWebSocketSendLoop loop) throws Exception { - long addr = readLong(loop, "sentDictBytesAddr"); - int len = readInt(loop, "sentDictBytesLen"); + long addr = loop.sentDictBytesAddr(); + int len = loop.sentDictBytesLen(); List out = new ArrayList<>(); long p = addr; long limit = addr + len; @@ -879,28 +842,18 @@ private static void deliverOk(CursorWebSocketSendLoop loop, long wireSeq) throws Unsafe.getUnsafe().putByte(ptr, WebSocketResponse.STATUS_OK); Unsafe.getUnsafe().putLong(ptr + 1, wireSeq); Unsafe.getUnsafe().putShort(ptr + 9, (short) 0); - Field f = CursorWebSocketSendLoop.class.getDeclaredField("responseHandler"); - f.setAccessible(true); - Object handler = f.get(loop); - Method m = handler.getClass().getDeclaredMethod("onBinaryMessage", long.class, int.class); - m.setAccessible(true); - m.invoke(handler, ptr, size); + loop.deliverResponseForTest(ptr, size); } finally { Unsafe.free(ptr, size, MemoryTag.NATIVE_DEFAULT); } } - private static void invokeSetWireBaselineWithCatchUp(CursorWebSocketSendLoop loop, long replayStart) throws Exception { - Method m = CursorWebSocketSendLoop.class.getDeclaredMethod("setWireBaselineWithCatchUp", long.class); - m.setAccessible(true); - m.invoke(loop, replayStart); + private static void invokeSetWireBaselineWithCatchUp(CursorWebSocketSendLoop loop, long replayStart) { + loop.setWireBaselineWithCatchUpForTest(replayStart); } - private static void invokeConnectLoop(CursorWebSocketSendLoop loop) throws Exception { - Method m = CursorWebSocketSendLoop.class.getDeclaredMethod( - "connectLoop", Throwable.class, String.class, long.class); - m.setAccessible(true); - m.invoke(loop, new LineSenderException("test reconnect"), "reconnect", 0L); + private static void invokeConnectLoop(CursorWebSocketSendLoop loop) { + loop.connectLoopForTest(new LineSenderException("test reconnect"), "reconnect", 0L); } private CursorWebSocketSendLoop newLoop(CursorSendEngine engine, WebSocketClient client) { @@ -947,12 +900,6 @@ private CursorSendEngine newEngine() { return new CursorSendEngine(tmpDir, 16_384); } - private static long readLong(CursorWebSocketSendLoop loop, String name) throws Exception { - Field f = CursorWebSocketSendLoop.class.getDeclaredField(name); - f.setAccessible(true); - return f.getLong(loop); - } - // Populates the loop's native sent-dictionary mirror with {@code symbols} in // the on-wire [len varint][utf8] layout, so setWireBaselineWithCatchUp sees a // non-empty dictionary to re-register. loop.close() frees it. @@ -971,42 +918,12 @@ private static void seedMirror(CursorWebSocketSendLoop loop, String... symbols) Unsafe.getUnsafe().putByte(p++, b); } } - setField(loop, "sentDictBytesAddr", addr); - setBooleanField(loop, "sentDictBytesOwned", true); - setIntField(loop, "sentDictBytesCapacity", total); - setIntField(loop, "sentDictBytesLen", total); - setIntField(loop, "sentDictCount", symbols.length); + loop.seedSentDictMirrorForTest(addr, total, symbols.length); // Leave sentDictIndexedCount at 0: the mirror carries its own entry lengths, and // the entry-ends index is built lazily by the catch-up. Pre-building it here // would hide that from every test that seeds this way. } - private static void setField(Object target, String name, long value) throws Exception { - Field f = CursorWebSocketSendLoop.class.getDeclaredField(name); - f.setAccessible(true); - f.setLong(target, value); - } - - private static void setIntField(Object target, String name, int value) throws Exception { - Field f = CursorWebSocketSendLoop.class.getDeclaredField(name); - f.setAccessible(true); - f.setInt(target, value); - } - - private static void setBooleanField(Object target, String name, boolean value) throws Exception { - Field f = CursorWebSocketSendLoop.class.getDeclaredField(name); - f.setAccessible(true); - f.setBoolean(target, value); - } - - private static void setBooleanFieldUnchecked(Object target, String name, boolean value) { - try { - setBooleanField(target, name, value); - } catch (Exception e) { - throw new AssertionError(e); - } - } - private static int varintSize(long value) { int n = 1; while (value > 0x7F) { diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopMirrorLeakTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopMirrorLeakTest.java index e480dd9f..ff8cfa50 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopMirrorLeakTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopMirrorLeakTest.java @@ -36,7 +36,6 @@ import org.junit.rules.TemporaryFolder; import java.io.IOException; -import java.lang.reflect.Field; import java.nio.file.Files; import java.nio.file.Path; import java.util.concurrent.TimeUnit; @@ -95,20 +94,20 @@ public void testSeededMirrorFreedWhenLoopClosedWithoutStart() throws Exception { // Close without start(): the ctor-seeded mirror is this // thread's to free, since the I/O loop never ran. Assert.assertTrue("precondition: the ctor seeded a non-empty mirror", - readInt(loop, "sentDictCount") > 0); + loop.sentDictCount() > 0); Assert.assertEquals("foreground loop must take the persisted buffer without copying", - persistedAddr, readLong(loop, "sentDictBytesAddr")); + persistedAddr, loop.sentDictBytesAddr()); Assert.assertEquals("ownership transfer must clear the persisted pointer", 0L, pd.loadedEntriesAddr()); Assert.assertTrue("foreground mirror must own the transferred buffer", - readBoolean(loop, "sentDictBytesOwned")); + loop.sentDictBytesOwned()); loop.close(); // close() must reset sentDictCount alongside freeing the buffer, // so the mirror stays all-or-nothing: a hypothetical post-close // start() (no closed guard) cannot read a stale count against a // freed buffer and drive a null-mirror catch-up. Assert.assertEquals("close() must reset sentDictCount to 0", - 0, readInt(loop, "sentDictCount")); + 0, loop.sentDictCount()); } }); } @@ -141,11 +140,11 @@ public void testRecycledLoopReSeedsMirrorFromPersistedDict() throws Exception { CursorWebSocketSendLoop loop1 = newRecoveryLoop(engine); try { Assert.assertEquals("session-1 mirror must seed from the persisted dict", - dictSize, readInt(loop1, "sentDictCount")); + dictSize, loop1.sentDictCount()); Assert.assertEquals("orphan session must borrow the persisted bytes", - persistedAddr, readLong(loop1, "sentDictBytesAddr")); + persistedAddr, loop1.sentDictBytesAddr()); Assert.assertFalse("borrowed orphan mirror must not own the persisted bytes", - readBoolean(loop1, "sentDictBytesOwned")); + loop1.sentDictBytesOwned()); } finally { loop1.close(); } @@ -159,8 +158,8 @@ public void testRecycledLoopReSeedsMirrorFromPersistedDict() throws Exception { try { Assert.assertEquals("recycled session-2 mirror must re-seed from the " + "persisted dict (pre-fix it was 0)", - dictSize, readInt(loop2, "sentDictCount")); - Assert.assertEquals(persistedAddr, readLong(loop2, "sentDictBytesAddr")); + dictSize, loop2.sentDictCount()); + Assert.assertEquals(persistedAddr, loop2.sentDictBytesAddr()); } finally { loop2.close(); } @@ -200,7 +199,7 @@ public void testCtorFreesSeededMirrorWhenFrameSeedThrows() throws Exception { Assert.assertTrue("the frame-seed path must run (frames out-reach the dict)", engine.recoveredMaxSymbolDeltaStart() > 0L); - setMirrorSeedFault(true); + CursorWebSocketSendLoop.forceMirrorSeedFailureForTest = true; try { new CursorWebSocketSendLoop( null, engine, 0, 1_000_000L, @@ -213,7 +212,7 @@ public void testCtorFreesSeededMirrorWhenFrameSeedThrows() throws Exception { Assert.assertTrue("unexpected message: " + expected.getMessage(), expected.getMessage().contains("simulated mirror seed allocation failure")); } finally { - setMirrorSeedFault(false); + CursorWebSocketSendLoop.forceMirrorSeedFailureForTest = false; } Assert.assertTrue("failed foreground construction must leave the persisted prefix owned", pd.loadedEntriesAddr() != 0L); @@ -244,7 +243,7 @@ public void testForegroundLoopReleasesRecoveredSuffixAfterSeeding() throws Excep 0, 0, 1); try { Assert.assertEquals("foreground mirror must include prefix and recovered suffix", - 3, readInt(loop, "sentDictCount")); + 3, loop.sentDictCount()); Assert.assertEquals("engine must release its duplicate recovery suffix", 0, engine.recoverySymbolNativeCapacity()); } finally { @@ -270,7 +269,7 @@ public void testOrphanLoopsRetainRecoveredSuffixForRecycle() throws Exception { CursorWebSocketSendLoop loop = newRecoveryLoop(engine); try { Assert.assertEquals("orphan mirror must include prefix and recovered suffix", - 3, readInt(loop, "sentDictCount")); + 3, loop.sentDictCount()); Assert.assertEquals("orphan engine must retain suffix for the next session", suffixCapacity, engine.recoverySymbolNativeCapacity()); } finally { @@ -347,33 +346,6 @@ private static void replacePersistedDictionaryWithTwoSymbolPrefix(Path slot) { } } - private static int readInt(CursorWebSocketSendLoop loop, String name) throws Exception { - Field f = CursorWebSocketSendLoop.class.getDeclaredField(name); - f.setAccessible(true); - return f.getInt(loop); - } - - private static boolean readBoolean(CursorWebSocketSendLoop loop, String name) throws Exception { - Field f = CursorWebSocketSendLoop.class.getDeclaredField(name); - f.setAccessible(true); - return f.getBoolean(loop); - } - - private static long readLong(CursorWebSocketSendLoop loop, String name) throws Exception { - Field f = CursorWebSocketSendLoop.class.getDeclaredField(name); - f.setAccessible(true); - return f.getLong(loop); - } - - // Toggles the loop's @TestOnly mirror-seed fault flag. Reflection because the - // flag is package-private in the production package (this test is in a sibling - // test package), the same non-reflective-path-unavailable reason readInt uses. - private static void setMirrorSeedFault(boolean value) throws Exception { - Field f = CursorWebSocketSendLoop.class.getDeclaredField("forceMirrorSeedFailureForTest"); - f.setAccessible(true); - f.setBoolean(null, value); - } - private static class SilentHandler implements TestWebSocketServer.WebSocketServerHandler { @Override public void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] data) { diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopPoisonFrameTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopPoisonFrameTest.java index d63887db..28846666 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopPoisonFrameTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopPoisonFrameTest.java @@ -41,8 +41,6 @@ import org.junit.Before; import org.junit.Test; -import java.lang.reflect.Field; -import java.lang.reflect.Method; import java.nio.charset.StandardCharsets; import java.nio.file.Paths; import java.util.ArrayList; @@ -326,10 +324,6 @@ public void testNackRecycleIsPacedAgainstHealthyServer() throws Exception { // backoff. Red behavior: hundreds of full recycles per second. final long initialBackoffMillis = 200L; final long runMillis = 1_200L; - // Generous ceiling: with >=200ms pacing, a 1.2s window fits at most - // 6 recycles; allow 10 for scheduling noise. The unpaced bug - // overshoots this by orders of magnitude, so no flakiness risk. - final int maxReconnects = 10; TestUtils.assertMemoryLeak(() -> { try (CursorSendEngine engine = newEngine()) { @@ -357,18 +351,26 @@ public void testNackRecycleIsPacedAgainstHealthyServer() throws Exception { } finally { loop.close(); } - int count; + Long[] stamps; synchronized (reconnectNanos) { - count = reconnectNanos.size(); + stamps = reconnectNanos.toArray(new Long[0]); // insertion order == reconnect order } - assertTrue("NACK-triggered recycles against a healthy server must be paced " - + "by the initial reconnect backoff (" + initialBackoffMillis - + "ms): observed " + count + " reconnects in " + runMillis - + "ms -- the recycle loop is running at server NACK rate " - + "with zero pacing", - count <= maxReconnects); assertTrue("sanity: the NACK must actually recycle the connection at least once", - count >= 1); + stamps.length >= 1); + // Pacing is per-recycle: each recycle parks >= the reconnect backoff before + // the next connect. Assert consecutive-reconnect gaps, not a count in a fixed + // window, so a starved/oversleeping test thread only yields MORE paced + // samples, never a false failure. Floor = 3/4 of the configured backoff: + // below the >=200ms real park, far above the unpaced bug's sub-ms gaps. + final long minGapNanos = initialBackoffMillis * 1_000_000L * 3 / 4; + for (int i = 1; i < stamps.length; i++) { + long gapNanos = stamps[i] - stamps[i - 1]; + assertTrue("NACK-triggered recycles against a healthy server must be paced " + + "by the initial reconnect backoff (" + initialBackoffMillis + + "ms): recycle " + i + " came " + (gapNanos / 1_000_000L) + + "ms after the previous one -- the recycle loop is unpaced", + gapNanos >= minGapNanos); + } } }); } @@ -390,11 +392,6 @@ public void testNonOrderlyCloseRecycleIsPacedAgainstAcceptingMiddlebox() throws // reconnect backoff. Red behavior: hundreds of full recycles per second. final long initialBackoffMillis = 200L; final long runMillis = 1_200L; - // With >=200ms pacing (escalating), a 1.2s window fits at most a - // handful of recycles; allow 10 for scheduling noise. The unpaced bug - // overshoots this by orders of magnitude, so no flakiness risk. - final int maxReconnects = 10; - TestUtils.assertMemoryLeak(() -> { try (CursorSendEngine engine = newEngine()) { appendFrames(engine, 1); @@ -422,17 +419,26 @@ public void testNonOrderlyCloseRecycleIsPacedAgainstAcceptingMiddlebox() throws } finally { loop.close(); } - int count; + Long[] stamps; synchronized (reconnectNanos) { - count = reconnectNanos.size(); + stamps = reconnectNanos.toArray(new Long[0]); // insertion order == reconnect order } - assertTrue("non-orderly-close recycles against an accepting middlebox must be " - + "paced by the reconnect backoff (" + initialBackoffMillis - + "ms): observed " + count + " reconnects in " + runMillis - + "ms -- the close path is recycling unpaced at connect+close RTT rate", - count <= maxReconnects); assertTrue("sanity: the non-orderly close must actually recycle the connection at least once", - count >= 1); + stamps.length >= 1); + // Pacing is per-recycle: each recycle parks >= the reconnect backoff before + // the next connect. Assert consecutive-reconnect gaps, not a count in a fixed + // window, so a starved/oversleeping test thread only yields MORE paced + // samples, never a false failure. Floor = 3/4 of the configured backoff: + // below the >=200ms real park, far above the unpaced bug's sub-ms gaps. + final long minGapNanos = initialBackoffMillis * 1_000_000L * 3 / 4; + for (int i = 1; i < stamps.length; i++) { + long gapNanos = stamps[i] - stamps[i - 1]; + assertTrue("non-orderly-close recycles against an accepting middlebox must be " + + "paced by the reconnect backoff (" + initialBackoffMillis + + "ms): recycle " + i + " came " + (gapNanos / 1_000_000L) + + "ms after the previous one -- the close path is recycling unpaced", + gapNanos >= minGapNanos); + } } }); } @@ -453,11 +459,6 @@ public void testOrderlyCloseChurnIsPacedAfterFirstRecycle() throws Exception { // Red behavior: hundreds of full recycles per second. final long initialBackoffMillis = 200L; final long runMillis = 1_200L; - // First recycle is immediate, then doses escalate from 200ms; a 1.2s - // window fits a handful of recycles. Allow 10 for scheduling noise -- - // the unpaced bug overshoots by orders of magnitude. - final int maxReconnects = 10; - TestUtils.assertMemoryLeak(() -> { try (CursorSendEngine engine = newEngine()) { appendFrames(engine, 1); @@ -487,18 +488,28 @@ public void testOrderlyCloseChurnIsPacedAfterFirstRecycle() throws Exception { } finally { loop.close(); } - int count; + Long[] stamps; synchronized (reconnectNanos) { - count = reconnectNanos.size(); + stamps = reconnectNanos.toArray(new Long[0]); // insertion order == reconnect order } - assertTrue("orderly-close (GOING_AWAY) recycles against a draining peer must " - + "be paced by the reconnect backoff after the first recycle (" - + initialBackoffMillis + "ms): observed " + count - + " reconnects in " + runMillis + "ms -- the orderly-close path " - + "is recycling unpaced at connect+close RTT rate", - count <= maxReconnects); assertTrue("sanity: the orderly close must actually recycle the connection at least once", - count >= 1); + stamps.length >= 1); + // Pacing is per-recycle: the FIRST recycle is immediate (failover latency), + // then each consecutive zero-progress recycle parks >= the reconnect backoff. + // Assert consecutive-reconnect gaps, not a count in a fixed window, so a + // starved/oversleeping test thread only yields MORE paced samples, never a + // false failure. The one immediate first recycle is reconnect[0], never a + // measured gap. Floor = 3/4 of the configured backoff. + final long minGapNanos = initialBackoffMillis * 1_000_000L * 3 / 4; + for (int i = 1; i < stamps.length; i++) { + long gapNanos = stamps[i] - stamps[i - 1]; + assertTrue("orderly-close (GOING_AWAY) recycles against a draining peer must " + + "be paced by the reconnect backoff after the first recycle (" + + initialBackoffMillis + "ms): recycle " + i + " came " + + (gapNanos / 1_000_000L) + "ms after the previous one -- the " + + "orderly-close path is recycling unpaced", + gapNanos >= minGapNanos); + } } }); } @@ -516,7 +527,6 @@ public void testPreSendCloseChurnIsPaced() throws Exception { // must survive (pacing, not a terminal -- Invariant B). final long initialBackoffMillis = 200L; final long runMillis = 1_200L; - final int maxReconnects = 10; TestUtils.assertMemoryLeak(() -> { try (CursorSendEngine engine = newEngine()) { @@ -546,18 +556,28 @@ public void testPreSendCloseChurnIsPaced() throws Exception { } finally { loop.close(); } - int count; + Long[] stamps; synchronized (reconnectNanos) { - count = reconnectNanos.size(); + stamps = reconnectNanos.toArray(new Long[0]); // insertion order == reconnect order } - assertTrue("pre-send close recycles (close before any send, nextWireSeq == 0) " - + "must be paced by the reconnect backoff (" + initialBackoffMillis - + "ms): observed " + count + " reconnects in " + runMillis - + "ms -- the strike-exempt close path is recycling unpaced at " - + "connect+close RTT rate", - count <= maxReconnects); assertTrue("sanity: the pre-send close must actually recycle the connection at least once", - count >= 1); + stamps.length >= 1); + // Pacing is per-recycle: the FIRST recycle is immediate (failover latency), + // then each consecutive zero-progress recycle parks >= the reconnect backoff. + // Assert consecutive-reconnect gaps, not a count in a fixed window, so a + // starved/oversleeping test thread only yields MORE paced samples, never a + // false failure. The one immediate first recycle is reconnect[0], never a + // measured gap. Floor = 3/4 of the configured backoff. + final long minGapNanos = initialBackoffMillis * 1_000_000L * 3 / 4; + for (int i = 1; i < stamps.length; i++) { + long gapNanos = stamps[i] - stamps[i - 1]; + assertTrue("pre-send close recycles (close before any send, nextWireSeq == 0) " + + "must be paced by the reconnect backoff (" + initialBackoffMillis + + "ms): recycle " + i + " came " + (gapNanos / 1_000_000L) + + "ms after the previous one -- the strike-exempt close path " + + "is recycling unpaced", + gapNanos >= minGapNanos); + } } }); } @@ -581,17 +601,22 @@ public void testExemptRecyclePacerFirstImmediateThenEscalatesAndResetsOnProgress long t0 = System.nanoTime(); deliverOrderlyClose(loop); long firstNanos = System.nanoTime() - t0; - assertTrue("the FIRST zero-progress exempt recycle must stay immediate " - + "(failover latency): took " + (firstNanos / 1_000_000L) + "ms", - firstNanos < 300_000_000L); + assertEquals("the FIRST exempt recycle must take the immediate (level-0) branch", + 1, loop.zeroProgressRecycles()); t0 = System.nanoTime(); deliverOrderlyClose(loop); long secondNanos = System.nanoTime() - t0; - assertTrue("a CONSECUTIVE zero-progress exempt recycle must park for at least " - + "the initial reconnect backoff (" + initialBackoffMillis - + "ms): took " + (secondNanos / 1_000_000L) + "ms", + assertEquals("a CONSECUTIVE exempt recycle must take the paced (level-1) branch", + 2, loop.zeroProgressRecycles()); + assertTrue("a CONSECUTIVE zero-progress exempt recycle must park for at least the " + + "initial reconnect backoff (" + initialBackoffMillis + "ms): took " + + (secondNanos / 1_000_000L) + "ms", secondNanos >= initialBackoffMillis * 1_000_000L); + assertTrue("the FIRST exempt recycle must stay immediate (failover latency), far " + + "shorter than the paced second: first=" + (firstNanos / 1_000_000L) + + "ms second=" + (secondNanos / 1_000_000L) + "ms", + firstNanos * 2 <= secondNanos); // OK-level acceptance progress (fsn 0 accepted) resets the // pacer: the next exempt recycle is a fresh failover. @@ -600,10 +625,12 @@ public void testExemptRecyclePacerFirstImmediateThenEscalatesAndResetsOnProgress t0 = System.nanoTime(); deliverOrderlyClose(loop); long thirdNanos = System.nanoTime() - t0; - assertTrue("OK-level progress must reset the zero-progress pacer -- the next " - + "exempt recycle is a fresh failover, not churn: took " - + (thirdNanos / 1_000_000L) + "ms", - thirdNanos < 300_000_000L); + assertEquals("OK-level progress must reset the zero-progress pacer -- the next exempt " + + "recycle is a fresh failover (level 0), not churn", + 1, loop.zeroProgressRecycles()); + assertTrue("the post-progress exempt recycle must stay immediate: third=" + + (thirdNanos / 1_000_000L) + "ms second=" + (secondNanos / 1_000_000L) + "ms", + thirdNanos * 2 <= secondNanos); } finally { closeAll(clients); } @@ -620,38 +647,51 @@ public void testPoisonDwellHoldsEscalationUntilWallClockWindowElapses() throws E // only after it elapses may a further same-frame strike escalate. The // window=0 (legacy immediate) path is covered by // testPoisonTerminalNamesTheRejectedFsn. - final long windowMillis = 600L; TestUtils.assertMemoryLeak(() -> { - List clients = new ArrayList<>(); - try (CursorSendEngine engine = newEngine()) { - appendFrames(engine, 2); - CursorWebSocketSendLoop loop = newDurableLoopWithWindow(engine, clients, 2, windowMillis); - setSentCount(loop, 2); - deliverOk(loop, 0, names("trades"), txns(7L)); - - // Burst PAST the strike threshold (2), all within the window. + // Claim A: strike threshold reached, dwell window still open -> NO escalation. + // Huge window so a synchronous burst can't cross it; no sleep -> no added time. + final long heldWindowMillis = 60_000L; + List clientsHeld = new ArrayList<>(); + try (CursorSendEngine engineHeld = newEngine()) { + appendFrames(engineHeld, 2); + CursorWebSocketSendLoop loopHeld = newDurableLoopWithWindow(engineHeld, clientsHeld, 2, heldWindowMillis); + setSentCount(loopHeld, 2); + deliverOk(loopHeld, 0, names("trades"), txns(7L)); for (int i = 0; i < 3; i++) { - setSentCount(loop, 2); - deliverRetriableNack(loop, 1, "disk full"); + setSentCount(loopHeld, 2); + deliverRetriableNack(loopHeld, 1, "disk full"); } - // Threshold reached, window still open -> must not have escalated. - loop.checkError(); + assertEquals("burst must accrue strikes past the threshold (non-vacuous)", + 3, loopHeld.poisonStrikes()); + loopHeld.checkError(); // window open -> must not have escalated + } finally { + closeAll(clientsHeld); + } - // Cross the window, then one more same-frame strike escalates. - Thread.sleep(windowMillis + 150); - setSentCount(loop, 2); - deliverRetriableNack(loop, 1, "disk full"); + // Claim B: once the window elapses, the next same-frame strike escalates. + final long escWindowMillis = 200L; + List clientsEsc = new ArrayList<>(); + try (CursorSendEngine engineEsc = newEngine()) { + appendFrames(engineEsc, 2); + CursorWebSocketSendLoop loopEsc = newDurableLoopWithWindow(engineEsc, clientsEsc, 2, escWindowMillis); + setSentCount(loopEsc, 2); + deliverOk(loopEsc, 0, names("trades"), txns(7L)); + setSentCount(loopEsc, 2); + deliverRetriableNack(loopEsc, 1, "disk full"); // strike 1 anchors the window + loopEsc.checkError(); // below threshold -> no escalation + Thread.sleep(escWindowMillis + 150); // cross the window (sleep only lengthens under starvation) + setSentCount(loopEsc, 2); + deliverRetriableNack(loopEsc, 1, "disk full"); // threshold reached AND window elapsed try { - loop.checkError(); + loopEsc.checkError(); fail("after the dwell window elapses, the next strike must escalate"); } catch (LineSenderServerException e) { - assertEquals(SenderError.Category.PROTOCOL_VIOLATION, - e.getServerError().getCategory()); + assertEquals(SenderError.Category.PROTOCOL_VIOLATION, e.getServerError().getCategory()); assertEquals("escalated terminal must still name the poisoned frame (fsn 1)", 1L, e.getServerError().getFromFsn()); } } finally { - closeAll(clients); + closeAll(clientsEsc); } }); } @@ -680,7 +720,7 @@ public void testPostSendCatchUpNackDoesNotStrikeOrLaunderPoisonState() throws Ex // strictly below the -1 sentinel so the assertion discriminates. setPostCatchUpDataFrameBaseline(loop, -2L, 3L); assertEquals("precondition: no poison suspect yet", - -1L, getLongField(loop, "poisonFsn")); + -1L, loop.poisonFsn()); deliverRetriableNack(loop, 0, "disk full"); @@ -688,7 +728,7 @@ public void testPostSendCatchUpNackDoesNotStrikeOrLaunderPoisonState() throws Ex // the sentinel is intact. Pre-fix it was set to -2, laundering the // poison detector's state. assertEquals("catch-up NACK must not set/launder the poison sentinel", - -1L, getLongField(loop, "poisonFsn")); + -1L, loop.poisonFsn()); loop.checkError(); } finally { closeAll(clients); @@ -716,7 +756,7 @@ public void testPostSendCatchUpNackAtAckedFsnBoundaryDoesNotStrike() throws Exce // = -1. A NACK of catch-up wire seq 0 maps to fsn -1 == ackedFsn exactly. setPostCatchUpDataFrameBaseline(loop, -1L, 2L); assertEquals("precondition: no strikes yet", - 0L, getLongField(loop, "poisonStrikes")); + 0L, loop.poisonStrikes()); deliverRetriableNack(loop, 0, "disk full"); @@ -728,9 +768,9 @@ public void testPostSendCatchUpNackAtAckedFsnBoundaryDoesNotStrike() throws Exce // strike) shows as poisonStrikes staying 0. A "<=" -> "<" mutation of the // fsn-vs-ackedFsn guard charges the strike and fails this. assertEquals("catch-up NACK at fsn == ackedFsn must not charge a poison strike", - 0L, getLongField(loop, "poisonStrikes")); + 0L, loop.poisonStrikes()); assertEquals("... and must leave the poison sentinel intact", - -1L, getLongField(loop, "poisonFsn")); + -1L, loop.poisonFsn()); loop.checkError(); } finally { closeAll(clients); @@ -797,18 +837,22 @@ public void testNotWritableRecycleFirstImmediateThenPaced() throws Exception { long t0 = System.nanoTime(); deliverNotWritableNack(loop, 1, "read-only replica"); long firstNanos = System.nanoTime() - t0; - assertTrue("the FIRST NOT_WRITABLE recycle must stay immediate (failover " - + "latency): took " + (firstNanos / 1_000_000L) + "ms", - firstNanos < 300_000_000L); + assertEquals("the FIRST NOT_WRITABLE recycle must take the immediate (level-0) branch", + 1, loop.zeroProgressRecycles()); setSentCount(loop, 2); t0 = System.nanoTime(); deliverNotWritableNack(loop, 1, "read-only replica"); long secondNanos = System.nanoTime() - t0; - assertTrue("a CONSECUTIVE zero-progress NOT_WRITABLE recycle must park for at " - + "least the initial reconnect backoff (" + initialBackoffMillis - + "ms): took " + (secondNanos / 1_000_000L) + "ms", + assertEquals("a CONSECUTIVE NOT_WRITABLE recycle must take the paced (level-1) branch", + 2, loop.zeroProgressRecycles()); + assertTrue("a CONSECUTIVE zero-progress NOT_WRITABLE recycle must park for at least the " + + "initial reconnect backoff (" + initialBackoffMillis + "ms): took " + + (secondNanos / 1_000_000L) + "ms", secondNanos >= initialBackoffMillis * 1_000_000L); + assertTrue("the FIRST NOT_WRITABLE recycle must stay immediate: first=" + + (firstNanos / 1_000_000L) + "ms second=" + (secondNanos / 1_000_000L) + "ms", + firstNanos * 2 <= secondNanos); // Pacing only, never a terminal: NOT_WRITABLE says nothing // about the bytes. @@ -1052,21 +1096,11 @@ private static void deliverOk(CursorWebSocketSendLoop loop, long wireSeq, } private static void deliverOrderlyClose(CursorWebSocketSendLoop loop) throws Exception { - Field f = CursorWebSocketSendLoop.class.getDeclaredField("responseHandler"); - f.setAccessible(true); - Object handler = f.get(loop); - Method m = handler.getClass().getDeclaredMethod("onClose", int.class, String.class); - m.setAccessible(true); - m.invoke(handler, 1001, "server draining"); // GOING_AWAY: orderly, strike-exempt + loop.deliverCloseForTest(1001, "server draining"); // GOING_AWAY: orderly, strike-exempt } private static void deliverNonOrderlyClose(CursorWebSocketSendLoop loop) throws Exception { - Field f = CursorWebSocketSendLoop.class.getDeclaredField("responseHandler"); - f.setAccessible(true); - Object handler = f.get(loop); - Method m = handler.getClass().getDeclaredMethod("onClose", int.class, String.class); - m.setAccessible(true); - m.invoke(handler, 1006, "connection reset"); // ABNORMAL_CLOSURE + loop.deliverCloseForTest(1006, "connection reset"); // ABNORMAL_CLOSURE } private static void deliverNotWritableNack(CursorWebSocketSendLoop loop, long wireSeq, @@ -1106,12 +1140,7 @@ private static void closeAll(List clients) { } private static void invokeOnBinaryMessage(CursorWebSocketSendLoop loop, long ptr, int size) throws Exception { - Field f = CursorWebSocketSendLoop.class.getDeclaredField("responseHandler"); - f.setAccessible(true); - Object handler = f.get(loop); - Method m = handler.getClass().getDeclaredMethod("onBinaryMessage", long.class, int.class); - m.setAccessible(true); - m.invoke(handler, ptr, size); + loop.deliverResponseForTest(ptr, size); } private static String[] names(String... v) { @@ -1119,25 +1148,19 @@ private static String[] names(String... v) { } private static void setSentCount(CursorWebSocketSendLoop loop, long count) throws Exception { - Field f = CursorWebSocketSendLoop.class.getDeclaredField("nextWireSeq"); - f.setAccessible(true); - f.setLong(loop, count); + loop.setNextWireSeqForTest(count); // A non-zero sent count models DATA frames sent on this connection. The // poison-strike / pre-send gates key off dataFrameSentThisConnection, not // nextWireSeq (the dictionary catch-up advances nextWireSeq without sending // a data frame), so keep the two in sync for these white-box scenarios. - Field d = CursorWebSocketSendLoop.class.getDeclaredField("dataFrameSentThisConnection"); - d.setAccessible(true); - d.setBoolean(loop, count > 0); + loop.setDataFrameSentThisConnectionForTest(count > 0); } // Sets ONLY nextWireSeq -- deliberately NOT dataFrameSentThisConnection -- to // model the dictionary catch-up having advanced the wire sequence with no data // frame sent yet. Contrast setSentCount, which sets both. private static void setCatchUpWireSeqOnly(CursorWebSocketSendLoop loop, long count) throws Exception { - Field f = CursorWebSocketSendLoop.class.getDeclaredField("nextWireSeq"); - f.setAccessible(true); - f.setLong(loop, count); + loop.setNextWireSeqForTest(count); } // Models a fresh connection that has shipped the dictionary catch-up (advancing @@ -1145,21 +1168,9 @@ private static void setCatchUpWireSeqOnly(CursorWebSocketSendLoop loop, long cou // nextWireSeq, and dataFrameSentThisConnection=true together. private static void setPostCatchUpDataFrameBaseline(CursorWebSocketSendLoop loop, long fsnAtZero, long nextWireSeq) throws Exception { - Field z = CursorWebSocketSendLoop.class.getDeclaredField("fsnAtZero"); - z.setAccessible(true); - z.setLong(loop, fsnAtZero); - Field w = CursorWebSocketSendLoop.class.getDeclaredField("nextWireSeq"); - w.setAccessible(true); - w.setLong(loop, nextWireSeq); - Field d = CursorWebSocketSendLoop.class.getDeclaredField("dataFrameSentThisConnection"); - d.setAccessible(true); - d.setBoolean(loop, true); - } - - private static long getLongField(CursorWebSocketSendLoop loop, String name) throws Exception { - Field f = CursorWebSocketSendLoop.class.getDeclaredField(name); - f.setAccessible(true); - return f.getLong(loop); + loop.setFsnAtZeroForTest(fsnAtZero); + loop.setNextWireSeqForTest(nextWireSeq); + loop.setDataFrameSentThisConnectionForTest(true); } private static long[] txns(long... v) { @@ -1191,12 +1202,10 @@ private CursorWebSocketSendLoop newDurableLoop(CursorSendEngine engine, 5_000L, 5L, 10L, true, CursorWebSocketSendLoop.DEFAULT_DURABLE_ACK_KEEPALIVE_INTERVAL_MILLIS, MAX_REJECTIONS); - // The loop is driven by reflection, not by its own I/O thread, but - // the recycle machinery must see a live loop or connectLoop's guard + // The loop is driven directly by the test, not by its own I/O thread, + // but the recycle machinery must see a live loop or connectLoop's guard // treats the first retriable NACK as fatal. - Field f = CursorWebSocketSendLoop.class.getDeclaredField("running"); - f.setAccessible(true); - f.setBoolean(loop, true); + loop.setRunningForTest(true); return loop; } @@ -1220,9 +1229,7 @@ private CursorWebSocketSendLoop newExemptPacerLoop(CursorSendEngine engine, 5_000L, initialBackoffMillis, 5 * initialBackoffMillis, true, CursorWebSocketSendLoop.DEFAULT_DURABLE_ACK_KEEPALIVE_INTERVAL_MILLIS, 1_000_000); - Field f = CursorWebSocketSendLoop.class.getDeclaredField("running"); - f.setAccessible(true); - f.setBoolean(loop, true); + loop.setRunningForTest(true); return loop; } @@ -1246,9 +1253,7 @@ private CursorWebSocketSendLoop newDurableLoopWithWindow(CursorSendEngine engine 5_000L, 5L, 10L, true, CursorWebSocketSendLoop.DEFAULT_DURABLE_ACK_KEEPALIVE_INTERVAL_MILLIS, maxRejections, windowMillis); - Field f = CursorWebSocketSendLoop.class.getDeclaredField("running"); - f.setAccessible(true); - f.setBoolean(loop, true); + loop.setRunningForTest(true); return loop; } } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopTornDictGuardTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopTornDictGuardTest.java index 5a666dc7..26156d06 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopTornDictGuardTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopTornDictGuardTest.java @@ -40,8 +40,6 @@ import java.io.File; import java.io.IOException; -import java.lang.reflect.Field; -import java.lang.reflect.Method; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; @@ -89,12 +87,12 @@ public void testGuardFiresOnGenuineGapAndShipsNoFrame() throws Exception { // while the first surviving frame's delta starts above it. CursorWebSocketSendLoop loop = newLoop(engine, client); try { - setRunning(loop); + loop.setRunningForTest(true); // Position at the first unsent frame exactly as the I/O loop does // before its first send (no catch-up: the mirror is empty). - invoke(loop, "positionCursorForStart"); + loop.positionCursorForStartForTest(); - boolean sent = (Boolean) invoke(loop, "trySendOne"); + boolean sent = loop.trySendOneForTest(); Assert.assertFalse("the torn-dict guard must refuse to send the gapped frame", sent); Assert.assertEquals("no frame may reach the server through a gap", @@ -128,18 +126,6 @@ private static int countSegmentFiles(Path dir) { return n; } - private static Object invoke(CursorWebSocketSendLoop loop, String method) throws Exception { - Method m = CursorWebSocketSendLoop.class.getDeclaredMethod(method); - m.setAccessible(true); - return m.invoke(loop); - } - - private static void setRunning(CursorWebSocketSendLoop loop) throws Exception { - Field f = CursorWebSocketSendLoop.class.getDeclaredField("running"); - f.setAccessible(true); - f.setBoolean(loop, true); - } - // Constructs a recovery send loop that is never started -- the test drives // positionCursorForStart + trySendOne directly. The reconnect factory throws // because no reconnect is expected before the guard latches its terminal. From 75d89b89dbfb5bccbf3577bc4591ee9ae2c86859 Mon Sep 17 00:00:00 2001 From: glasstiger Date: Sat, 18 Jul 2026 01:19:40 +0100 Subject: [PATCH 95/96] Preserve cap-gap episode; minor test/style nits A cluster of small review follow-ups; only the first changes behavior. - The orphan-tail-retire re-anchor caught a CatchUpSendException and passed fail(e.getCause()), unwrapping the cap-gap marker so connectLoop's isCatchUpCapGap reset the orphan settle episode instead of preserving it. Route unwrap-aware: fail(isCatchUpCapGap(e) ? e : e.getCause()) keeps the attempt count and dwell anchor across the re-anchor recycle for a genuine cap gap, and unwraps to the raw cause otherwise, exactly as a normal send failure does. This only ever errs safe today -- more retries before an orphan drainer quarantines -- but the wrapper exists precisely to carry that signal. - Drop @TestOnly from BackgroundDrainer.connectWithDurableAckRetry: run() calls it in production, so the annotation was inaccurate. - Rename CatchUpSendException.capGap to isCapGap (boolean naming). - Restore the CursorWebSocketSendLoop final-field group and a QwpWebSocketSender import to alphabetical order. - DeltaDictCatchUpTest: fix a comment naming a test that does not exist. - DeltaDictRecoveryTest: delete the never-instantiated CountingHandler and the orphaned javadoc stacked above FullDiskDictFacade. - CursorSendEngineTest: rename the legacy-guard test to describe what it asserts (the guards force an FSN gap; current recovery adopts the v2 slot) -- it never drives a legacy reader to throw. - PoisonFrameTest: the two catch-up-only "does not strike" tests now assert poisonStrikes() == 0 explicitly instead of relying only on checkError() not throwing. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../qwp/client/QwpWebSocketSender.java | 2 +- .../client/sf/cursor/BackgroundDrainer.java | 1 - .../sf/cursor/CursorWebSocketSendLoop.java | 17 ++++++++++------ .../qwp/client/DeltaDictCatchUpTest.java | 2 +- .../qwp/client/DeltaDictRecoveryTest.java | 20 ------------------- .../sf/cursor/CursorSendEngineTest.java | 2 +- ...ursorWebSocketSendLoopPoisonFrameTest.java | 4 ++++ 7 files changed, 18 insertions(+), 30 deletions(-) diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index 8baf5004..28bb70f2 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -47,10 +47,10 @@ import io.questdb.client.cutlass.qwp.client.sf.cursor.DefaultSenderErrorHandler; import io.questdb.client.cutlass.qwp.client.sf.cursor.DefaultSenderProgressHandler; import io.questdb.client.cutlass.qwp.client.sf.cursor.PersistedSymbolDict; -import io.questdb.client.cutlass.qwp.client.sf.cursor.UnreplayableSlotException; import io.questdb.client.cutlass.qwp.client.sf.cursor.SenderConnectionDispatcher; import io.questdb.client.cutlass.qwp.client.sf.cursor.SenderErrorDispatcher; import io.questdb.client.cutlass.qwp.client.sf.cursor.SenderProgressDispatcher; +import io.questdb.client.cutlass.qwp.client.sf.cursor.UnreplayableSlotException; import io.questdb.client.cutlass.qwp.protocol.QwpConstants; import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; import io.questdb.client.std.CharSequenceObjHashMap; diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/BackgroundDrainer.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/BackgroundDrainer.java index 97130cb7..d77d20d9 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/BackgroundDrainer.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/BackgroundDrainer.java @@ -265,7 +265,6 @@ public BackgroundDrainer() { * @return a fresh durable-ack-capable client, or {@code null} if * {@link #outcome} has been set to FAILED or STOPPED */ - @TestOnly public WebSocketClient connectWithDurableAckRetry() { // run() already set runnerThread; setting it again here is a no-op // on that path but wires up direct @TestOnly calls so requestStop() diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java index 8d42af75..753003f2 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java @@ -229,9 +229,9 @@ public final class CursorWebSocketSendLoop implements QuietCloseable { // user-facing 5-minute default is applied at the config layer. private final long catchUpCapGapMinEscalationWindowNanos; private final CatchUpCapGapPolicy catchUpCapGapPolicy; - private final ReconnectPolicy reconnectPolicy; private final CursorSendEngine engine; private final long parkNanos; + private final ReconnectPolicy reconnectPolicy; // FIFO of OK-acked batches awaiting durable-upload confirmation. Used only // when durableAckMode is true. Each entry binds a wireSeq to the per-table // (name, seqTxn) pairs the server reported on the OK frame. The queue is @@ -1548,7 +1548,7 @@ private void clearDurableAckTracking() { } private static boolean isCatchUpCapGap(Throwable t) { - return t instanceof CatchUpSendException && ((CatchUpSendException) t).capGap; + return t instanceof CatchUpSendException && ((CatchUpSendException) t).isCapGap; } private void resetCatchUpCapGapEpisode() { @@ -3103,7 +3103,12 @@ private boolean trySendOne() { // Re-anchor's catch-up send failed. fail() here is a fresh, // non-re-entrant connectLoop entry from the I/O loop body -- // the same recovery a normal trySendOne send failure takes. - fail(e.getCause()); + // Preserve the wrapper on a cap gap so connectLoop's + // isCatchUpCapGap keeps the orphan settle episode (attempt count + // + dwell anchor) alive across the re-anchor recycle; an ordinary + // failure unwraps to the raw cause and restarts the episode, like + // any normal send failure. + fail(isCatchUpCapGap(e) ? e : e.getCause()); return false; } return true; @@ -3325,15 +3330,15 @@ public interface ReconnectFactory { * send failures restart the orphan settle episode. */ private static final class CatchUpSendException extends RuntimeException { - private final boolean capGap; + private final boolean isCapGap; CatchUpSendException(Throwable cause) { this(cause, false); } - CatchUpSendException(Throwable cause, boolean capGap) { + CatchUpSendException(Throwable cause, boolean isCapGap) { super(cause); - this.capGap = capGap; + this.isCapGap = isCapGap; } } diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictCatchUpTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictCatchUpTest.java index 400ea1d3..34d9899d 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictCatchUpTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictCatchUpTest.java @@ -164,7 +164,7 @@ public void testFixedCapNearBoundarySymbolCatchesUpWithoutTerminal() throws Exce // table("t").symbol("s", <173 chars>).atNow() encodes to 198 bytes (<=200, // accepted), its dict entry is 2+173=175 bytes (> old budget 172 -> old // terminal), while the real solo catch-up frame is 12+1+1+175=189 (<=200 -> - // fits). Unlike testCatchUpEntryTooLargeForCapFailsTerminally (a genuinely + // fits). Unlike testCatchUpCapGapRetriesUntilBudgetThenLatches (a genuinely // oversized entry on a shrunk cap, which MUST still terminate), this entry // is legally shippable and must NOT terminate. final int cap = 200; diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java index 4e36e983..dbb9faab 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/DeltaDictRecoveryTest.java @@ -1636,26 +1636,6 @@ public void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] dat } } - /** Counts every binary frame it receives and acks it. */ - private static class CountingHandler implements TestWebSocketServer.WebSocketServerHandler { - final AtomicInteger frames = new AtomicInteger(); - private final AtomicLong nextSeq = new AtomicLong(0); - - @Override - public void onBinaryMessage(TestWebSocketServer.ClientHandler client, byte[] data) { - frames.incrementAndGet(); - try { - client.sendBinary(QwpWireTestUtils.buildAck(nextSeq.getAndIncrement())); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - } - /** - * Short-writes the FIRST dictionary append after it is armed, modelling a disk that - * fills mid-flush. Offset 0 is left alone so the file header still writes -- only the - * entry append fails, which is the path under test. - */ /** * Refuses to grow the symbol dictionary's mmap append window -- the production * shape of a full disk, since PersistedSymbolDict reserves that window through diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineTest.java index 0b7bf989..d45ff6ba 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorSendEngineTest.java @@ -356,7 +356,7 @@ public void testLegacyReaderGuardsSurviveReopenAndAreRepairedWhenDamaged() throw } @Test - public void testV2DiskSlotForcesLegacyReaderToFailClosed() throws Exception { + public void testLegacyGuardsForceFsnGapAndCurrentRecoveryAdoptsV2Slot() throws Exception { TestUtils.assertMemoryLeak(() -> { long buf = Unsafe.malloc(16, MemoryTag.NATIVE_DEFAULT); try { diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopPoisonFrameTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopPoisonFrameTest.java index 28846666..be824068 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopPoisonFrameTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoopPoisonFrameTest.java @@ -270,6 +270,8 @@ public void testNonOrderlyCloseAfterOnlyCatchUpDoesNotStrike() throws Exception } // No strike was ever charged, so nothing escalated: the loop stays // retriable and the producer-facing error latch is clear. + assertEquals("a catch-up-only non-orderly close must charge no poison strike", + 0, loop.poisonStrikes()); loop.checkError(); } finally { closeAll(clients); @@ -306,6 +308,8 @@ public void testNonOrderlyRejectionAfterOnlyCatchUpDoesNotStrike() throws Except } // No strike was ever charged, so nothing escalated: the loop stays // retriable and the producer-facing error latch is clear. + assertEquals("a catch-up-only NACK must charge no poison strike", + 0, loop.poisonStrikes()); loop.checkError(); } finally { closeAll(clients); From 0a3010f024413006ad5c8a3c130036659c8cd23a Mon Sep 17 00:00:00 2001 From: glasstiger Date: Sat, 18 Jul 2026 02:18:17 +0100 Subject: [PATCH 96/96] Fix Windows separator mismatch in SlotLockTest testLogicalLockReportsLockDirectoryCreationFailure built the lock directory path with Paths.get(parentDir, ".slot-locks"), which yields a backslash separator on Windows. SlotLock.acquireLogical builds it as parentDir + "/" + ".slot-locks" (forward slash), so the injected LockDirectoryFailureFacade -- which fails mkdir only when lockDir.equals(path) -- never matched the production path on Windows. The mkdir then succeeded, acquireLogical did not throw, and the test hit its fail() guard. It passed on Linux and macOS only because Paths.get uses a forward slash there. Build the test's lock-dir (and slot) path with the same "/" concatenation SlotLock uses, so the facade matches on every platform. Test-only change; no production behavior is affected. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../cutlass/qwp/client/sf/cursor/SlotLockTest.java | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SlotLockTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SlotLockTest.java index 9489ee82..469dbb84 100644 --- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SlotLockTest.java +++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/sf/cursor/SlotLockTest.java @@ -144,8 +144,14 @@ public void testLogicalLockRejectsInvalidPaths() throws Exception { @Test public void testLogicalLockReportsLockDirectoryCreationFailure() throws Exception { TestUtils.assertMemoryLeak(() -> { - String slot = Paths.get(parentDir, "mkdir-failure").toString(); - String lockDir = Paths.get(parentDir, ".slot-locks").toString(); + String slot = parentDir + "/mkdir-failure"; + // Build the lock-dir path exactly as SlotLock.acquireLogical does + // (parent + "/" + ".slot-locks"), NOT via Paths.get: on Windows + // Paths.get yields a '\' separator, so the facade's lockDir.equals(path) + // check never matched the production forward-slash path and the mkdir + // failure was never injected -- the sole cause of the Windows-only + // failure of this test. + String lockDir = parentDir + "/.slot-locks"; LockDirectoryFailureFacade ff = new LockDirectoryFailureFacade(lockDir); try { SlotLock.acquireLogical(ff, slot);