diff --git a/core/src/main/java/io/questdb/client/Sender.java b/core/src/main/java/io/questdb/client/Sender.java index 4ec70dbd..bb04411b 100644 --- a/core/src/main/java/io/questdb/client/Sender.java +++ b/core/src/main/java/io/questdb/client/Sender.java @@ -38,6 +38,10 @@ import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender; import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorSendEngine; import io.questdb.client.cutlass.qwp.client.sf.cursor.CursorWebSocketSendLoop; +import io.questdb.client.cutlass.qwp.client.sf.cursor.OrphanScanner; +import io.questdb.client.cutlass.qwp.client.sf.cursor.PersistedSymbolDict; +import io.questdb.client.cutlass.qwp.client.sf.cursor.SlotLock; +import io.questdb.client.cutlass.qwp.client.sf.cursor.UnreplayableSlotException; import io.questdb.client.impl.ConfStringParser; import io.questdb.client.impl.ConfigString; import io.questdb.client.impl.ConfigView; @@ -48,6 +52,7 @@ import io.questdb.client.std.Decimal256; import io.questdb.client.std.Decimal64; import io.questdb.client.std.Files; +import io.questdb.client.std.FilesFacade; import io.questdb.client.std.IntList; import io.questdb.client.std.Numbers; import io.questdb.client.std.NumericException; @@ -790,12 +795,9 @@ default Sender uuidColumn(CharSequence name, long lo, long hi) { * unconnected sender; the I/O thread runs the same retry loop in * the background. The user thread can call {@code at()} / * {@code flush()} immediately; rows accumulate in the cursor SF - * engine until the wire is up. Connect failures are retried - * indefinitely in the background; a terminal upgrade failure - * (auth reject, capability mismatch) is delivered to the async - * error inbox as a {@link io.questdb.client.SenderError} (no - * synchronous throw on the user call site). Wire - * {@code error_handler=...} to observe these. + * engine until the wire is up. Transport, auth, upgrade and + * capability failures are retried indefinitely in the background; + * none is surfaced to producer calls or the async error inbox. * *

* Default resolution when the caller does not pick a value: @@ -988,6 +990,12 @@ final class LineSenderBuilder { // build() time. 0 or negative is a documented "disable" value, so // a Long.MIN_VALUE sentinel keeps it distinguishable from "unset". private static final long DURABLE_ACK_KEEPALIVE_NOT_SET = Long.MIN_VALUE; + private static final org.slf4j.Logger LOG = org.slf4j.LoggerFactory.getLogger(LineSenderBuilder.class); + // How many quarantined copies of one slot may pile up under sf_dir before build() + // refuses to set aside another. Each is an unreplayable slot a human still has to + // look at; accumulating them without bound would turn a disk-space problem into a + // second incident. + private static final int MAX_QUARANTINE_SLOT_ATTEMPTS = 64; private static final int MIN_BUFFER_SIZE = AuthUtils.CHALLENGE_LEN + 1; // challenge size + 1; // sf-client.md section 4.4: the inbox capacity must accommodate the // distinct error categories in a bursty error stream so that drop-oldest @@ -1003,6 +1011,15 @@ final class LineSenderBuilder { private static final int PROTOCOL_TCP = 0; private static final int PROTOCOL_UDP = 3; private static final int PROTOCOL_WEBSOCKET = 2; + @TestOnly + private static volatile Runnable quarantineAfterCloseHook; + @TestOnly + private static volatile FilesFacade quarantineFilesFacade = FilesFacade.INSTANCE; + // Suffix for a slot set aside by quarantineTornSlot. Deliberately NOT the + // sender's own slot name, so a restarted sender does not re-adopt it as its own; + // quarantineTornSlot then marks it .failed, so the orphan drainer skips it too and + // the bytes stay put for a human to inspect and resend. + private static final String QUARANTINE_SLOT_SUFFIX = ".unreplayable-"; private final ObjList hosts = new ObjList<>(); private final IntList ports = new IntList(); private long authTimeoutMillis = QwpWebSocketSender.DEFAULT_AUTH_TIMEOUT_MS; @@ -1051,6 +1068,7 @@ final class LineSenderBuilder { private int errorInboxCapacity = PARAMETER_NOT_SET_EXPLICITLY; private int maxFrameRejections = PARAMETER_NOT_SET_EXPLICITLY; private long poisonMinEscalationWindowMillis = PARAMETER_NOT_SET_EXPLICITLY; + private long catchUpCapGapMinEscalationWindowMillis = PARAMETER_NOT_SET_EXPLICITLY; private String httpPath; private String httpSettingsPath; private int httpTimeout = PARAMETER_NOT_SET_EXPLICITLY; @@ -1059,8 +1077,8 @@ final class LineSenderBuilder { // explicitly", which build() resolves to SYNC when any reconnect_* // knob was tuned by the user, otherwise OFF. SYNC retries on the // user thread up to the reconnect cap. ASYNC returns immediately - // and lets the I/O thread retry in the background, surfacing - // terminal failures via the error inbox. + // and lets the I/O thread retry every endpoint/transport failure in + // the background without surfacing it to the producer. private InitialConnectMode initialConnectMode = null; private String keyId; private int maxBackgroundDrainers = DEFAULT_MAX_BACKGROUND_DRAINERS; @@ -1505,6 +1523,10 @@ public Sender build() { long actualPoisonMinEscalationWindowMillis = poisonMinEscalationWindowMillis != PARAMETER_NOT_SET_EXPLICITLY ? poisonMinEscalationWindowMillis : CursorWebSocketSendLoop.DEFAULT_POISON_MIN_ESCALATION_WINDOW_MILLIS; + long actualCatchUpCapGapMinEscalationWindowMillis = + catchUpCapGapMinEscalationWindowMillis != PARAMETER_NOT_SET_EXPLICITLY + ? catchUpCapGapMinEscalationWindowMillis + : CursorWebSocketSendLoop.DEFAULT_CATCHUP_CAP_GAP_MIN_ESCALATION_WINDOW_MILLIS; // sfDir is the parent (group root); the actual slot lives // under sfDir/senderId. This is what the engine sees — the @@ -1540,55 +1562,101 @@ public Sender build() { sfAppendDeadlineMillis == PARAMETER_NOT_SET_EXPLICITLY ? CursorSendEngine.DEFAULT_APPEND_DEADLINE_NANOS : sfAppendDeadlineMillis * 1_000_000L; - CursorSendEngine cursorEngine = new CursorSendEngine( - slotPath, actualSfMaxBytes, - actualSfMaxTotalBytes, actualSfAppendDeadlineNanos); - int actualErrorInboxCapacity = errorInboxCapacity != PARAMETER_NOT_SET_EXPLICITLY - ? errorInboxCapacity - : io.questdb.client.cutlass.qwp.client.sf.cursor.SenderErrorDispatcher.DEFAULT_CAPACITY; - int actualConnectionListenerInboxCapacity = connectionListenerInboxCapacity != PARAMETER_NOT_SET_EXPLICITLY - ? connectionListenerInboxCapacity - : io.questdb.client.cutlass.qwp.client.sf.cursor.SenderConnectionDispatcher.DEFAULT_CAPACITY; - List wsEndpoints = - new ArrayList<>(hosts.size()); - for (int i = 0, n = hosts.size(); i < n; i++) { - wsEndpoints.add(new QwpWebSocketSender.Endpoint(hosts.getQuick(i), ports.getQuick(i))); - } - QwpWebSocketSender connected; - try { - connected = QwpWebSocketSender.connect( - wsEndpoints, - wsTlsConfig, - actualAutoFlushRows, - actualAutoFlushBytes, - actualAutoFlushIntervalNanos, - wsAuthHeader, - requestDurableAck, - cursorEngine, - actualCloseFlushTimeoutMillis, - actualReconnectMaxDurationMillis, - actualReconnectInitialBackoffMillis, - actualReconnectMaxBackoffMillis, - actualInitialConnectMode, - errorHandler, - actualErrorInboxCapacity, - actualDurableAckKeepaliveIntervalMillis, - authTimeoutMillis, - connectTimeoutMillis == PARAMETER_NOT_SET_EXPLICITLY ? 0 : connectTimeoutMillis, - connectionListener, - actualConnectionListenerInboxCapacity, - actualMaxFrameRejections, - actualPoisonMinEscalationWindowMillis - ); - } catch (Throwable t) { - // connect() failed before ownership of cursorEngine - // transferred — close it ourselves. - try { - cursorEngine.close(); - } catch (Throwable ignored) { - // best-effort + QwpWebSocketSender connected = null; + // The parent-anchored logical lock is stable across a slot rename. Keep it + // from before the directory-local lock is acquired until connect() has either + // adopted that engine or quarantine has closed, renamed and recreated it. + // This closes the inode-swap window in which an already-queued orphan drainer + // could otherwise acquire the renamed directory's old .lock and later operate + // on the fresh slot through the original pathname. + try (SlotLock logicalSlotLock = slotPath == null + ? null + : SlotLock.acquireLogical(slotPath)) { + CursorSendEngine cursorEngine = new CursorSendEngine( + slotPath, actualSfMaxBytes, + actualSfMaxTotalBytes, actualSfAppendDeadlineNanos); + int actualErrorInboxCapacity = errorInboxCapacity != PARAMETER_NOT_SET_EXPLICITLY + ? errorInboxCapacity + : io.questdb.client.cutlass.qwp.client.sf.cursor.SenderErrorDispatcher.DEFAULT_CAPACITY; + int actualConnectionListenerInboxCapacity = connectionListenerInboxCapacity != PARAMETER_NOT_SET_EXPLICITLY + ? connectionListenerInboxCapacity + : io.questdb.client.cutlass.qwp.client.sf.cursor.SenderConnectionDispatcher.DEFAULT_CAPACITY; + List wsEndpoints = + new ArrayList<>(hosts.size()); + for (int i = 0, n = hosts.size(); i < n; i++) { + wsEndpoints.add(new QwpWebSocketSender.Endpoint(hosts.getQuick(i), ports.getQuick(i))); + } + // The recovery seed inside connect() is the authority on whether a recovered + // slot can be replayed: it rebuilds the dictionary from its intact prefix and + // then from the surviving frames' own delta sections, and throws + // UnreplayableSlotException only once neither source holds the missing ids. + // Quarantining on anything weaker would set aside slots that recovery can + // still rescue, so build() waits for that verdict rather than pre-judging it. + boolean quarantined = false; + while (connected == null) { + try { + connected = QwpWebSocketSender.connect( + wsEndpoints, + wsTlsConfig, + actualAutoFlushRows, + actualAutoFlushBytes, + actualAutoFlushIntervalNanos, + wsAuthHeader, + requestDurableAck, + cursorEngine, + actualCloseFlushTimeoutMillis, + actualReconnectMaxDurationMillis, + actualReconnectInitialBackoffMillis, + actualReconnectMaxBackoffMillis, + actualInitialConnectMode, + errorHandler, + actualErrorInboxCapacity, + actualDurableAckKeepaliveIntervalMillis, + authTimeoutMillis, + connectTimeoutMillis == PARAMETER_NOT_SET_EXPLICITLY ? 0 : connectTimeoutMillis, + connectionListener, + actualConnectionListenerInboxCapacity, + actualMaxFrameRejections, + actualPoisonMinEscalationWindowMillis, + actualCatchUpCapGapMinEscalationWindowMillis + ); + } catch (UnreplayableSlotException e) { + // The one failure build() recovers from. The slot's frames reference ids + // that nothing still holds, so they can never go on the wire -- but that is + // no reason to take the producer down with them. Before this, the throw + // escaped build() and, because senderId is stable and a not-fully-drained + // slot is retained on close, every retry re-recovered the same slot and + // threw again: the application could not construct a Sender at all, so it + // could not even BUFFER new rows. An already-lost batch became an unbounded + // outage of everything after it. + // + // Set the slot aside instead, keep its bytes for forensics and resend, and + // start the producer on a clean one. Once only: a second such failure would + // mean the FRESH slot is unreplayable, which cannot happen, so let it out + // rather than loop. + if (quarantined || slotPath == null) { + try { + cursorEngine.close(); + } catch (Throwable ignored) { + // best-effort + } + throw e; + } + quarantined = true; + cursorEngine = quarantineTornSlot( + cursorEngine, e, sfDir, senderId, slotPath, actualSfMaxBytes, + actualSfMaxTotalBytes, actualSfAppendDeadlineNanos); + } catch (Throwable t) { + // connect() failed before ownership of cursorEngine + // transferred — close it ourselves. + try { + cursorEngine.close(); + } catch (Throwable ignored) { + // best-effort + } + throw t; + } } - throw t; } // connect() succeeded — `connected` now owns cursorEngine // via setCursorEngine(engine, true). From here on, ANY @@ -1703,6 +1771,35 @@ public Sender build() { return sender; } + /** + * Minimum wall-clock time (millis) a symbol-dictionary catch-up CAP GAP must + * persist before an orphan store-and-forward drainer quarantines its slot. + *

+ * A cap gap means a symbol already accepted by one node is too large to + * re-register on the node the sender just failed over to, because that node + * advertises a smaller maximum batch size. On a homogeneous cluster this cannot + * happen; it takes a heterogeneous or mid-roll cluster, or an operator lowering + * the cap below existing data. + *

+ * A foreground sender retries such a gap indefinitely because the larger-cap node + * may simply be away. An orphan drainer may quarantine only once the gap has BOTH + * recurred many times AND persisted for this long. Raise it for a cluster whose + * rolling restarts take longer than the 5-minute default; set it to {@code 0} to + * quarantine an orphan slot as soon as the retry count is exhausted. + *

+ * WebSocket transport only. + */ + public LineSenderBuilder catchUpCapGapMinEscalationWindowMillis(long millis) { + if (protocol != PARAMETER_NOT_SET_EXPLICITLY && protocol != PROTOCOL_WEBSOCKET) { + throw new LineSenderException("catchup_cap_gap_min_escalation_window_millis is only supported for WebSocket transport"); + } + if (millis < 0) { + throw new LineSenderException("catchup_cap_gap_min_escalation_window_millis must be >= 0: ").put(millis); + } + this.catchUpCapGapMinEscalationWindowMillis = millis; + return this; + } + /** * close() drain timeout in milliseconds. The sender's {@code close()} * method blocks up to this many millis waiting for the server to ACK @@ -2149,11 +2246,10 @@ public LineSenderBuilder initialConnectMode(InitialConnectMode mode) { } /** - * Opt in to retrying the initial connect with the same backoff / - * cap / auth-terminal policy as in-flight reconnect. Set true if - * your deployment expects the server to come up shortly after the - * sender. Auth failures (HTTP 401/403/non-101) stay terminal in - * either mode. + * Opt in to retrying the initial connect with backoff on the calling + * thread, bounded by the configured reconnect cap. Set true if your + * deployment expects the server to come up shortly after the sender. + * Auth failures (HTTP 401/403/non-101) stay terminal in this SYNC mode. *

* When this method is not called, the resolution rule documented * on {@link InitialConnectMode} applies: SYNC implicitly when any @@ -2494,8 +2590,8 @@ public LineSenderBuilder reconnectMaxBackoffMillis(long millis) { * with exponential backoff until connect succeeds or this many * millis elapse, then throws. The background reconnect loop * (mid-stream outages and async initial connect) does NOT consult - * this value: it retries indefinitely and halts only on a terminal - * auth/upgrade error or {@code close()}. + * this value: endpoint and transport failures are retried indefinitely + * until {@code close()}. *

* Default {@code 300_000} (5 minutes). Lower for fail-fast startup; * higher for tolerating a slow server boot. Must be positive: a zero @@ -2916,6 +3012,95 @@ private static long parseSizeValue(@NotNull StringSink value, @NotNull String na } } + /** + * Sets a recovered slot whose symbol dictionary cannot cover its surviving frames + * aside, and returns a fresh engine on an empty slot so the producer can keep + * producing. + *

+ * Such a slot is unreplayable BY THIS PRODUCER: its frames reference symbol ids the + * recovered dictionary lost (a host/power crash tore the unsynced side-file), so a + * producer seeded from the short dictionary would hand those ids to different + * symbols and silently misattribute values. Detecting that is correct and + * load-bearing -- but simply THROWING is not a safe response. {@code senderId} + * defaults to a stable name, so a restarted process re-adopts the same slot; the + * engine's close retains a slot that is not fully drained; and so every subsequent + * {@code build()} would re-recover the same slot and throw again -- forever, until + * an operator deleted the directory by hand. The application could not construct a + * Sender at all, and so could not even BUFFER new rows. That trades a bounded, + * already-lost batch for an unbounded outage of everything after it, which inverts + * the one guarantee store-and-forward exists to give. + *

+ * So: rename the slot aside instead, and mark it {@code .failed}. The verdict is + * authoritative -- the recovery seed already tried every source of truth (the + * persisted prefix AND the surviving frames' own deltas), and the orphan drainer's + * own replay guard uses that same walk, so there is nothing a drainer could rebuild + * that the seed did not. {@code markFailed} (below) therefore quarantines the copy + * for a human rather than leaving a drainer to retry an unreplayable slot forever; + * a full-dictionary-fallback slot never reaches here, because its dictionary is + * discarded at recovery and it never throws. The bytes are preserved on disk for + * forensics and a manual resend, and the new name -- NOT the sender's own slot name + * -- keeps a restarted sender from re-adopting it. The producer, meanwhile, starts + * on a clean empty slot and never notices. + *

+ * If the rename fails (a Windows share lock, a read-only mount) there is no way to + * free the slot name without destroying data, so fall back to the old behaviour and + * throw -- loudly, and never silently dropping bytes. + */ + private static CursorSendEngine quarantineTornSlot( + CursorSendEngine torn, UnreplayableSlotException cause, String sfDir, + String senderId, String slotPath, + long sfMaxBytes, long sfMaxTotalBytes, long sfAppendDeadlineNanos + ) { + // The verdict, and the reason, come from the recovery seed -- the only code that + // has tried every source of truth. Recomputing them here would mean a second, + // independently-drifting notion of "unreplayable". + final String detail = cause.getMessage(); + // Release the slot lock and the dictionary fd before renaming. connect()'s failure + // path already closed the engine; close() is idempotent, so make it explicit rather + // than depend on that. + torn.close(); + Runnable hook = quarantineAfterCloseHook; + if (hook != null) { + hook.run(); + } + + FilesFacade ff = quarantineFilesFacade; + String quarantinePath = null; + for (int i = 0; i < MAX_QUARANTINE_SLOT_ATTEMPTS; i++) { + String candidate = sfDir + "/" + senderId + QUARANTINE_SLOT_SUFFIX + i; + if (!ff.exists(candidate)) { + quarantinePath = candidate; + break; + } + } + if (quarantinePath == null || ff.rename(slotPath, quarantinePath) != 0) { + throw new LineSenderException( + detail + "; the affected data must be resent. The slot could not be set aside " + + "automatically (" + (quarantinePath == null + ? "too many quarantined slots already under " + sfDir + : "rename to " + quarantinePath + " failed") + + "), so this sender cannot start until " + + slotPath + " is moved or removed by hand"); + } + // Mark the quarantined copy so the orphan drainer treats it as a + // human-in-the-loop slot rather than silently retrying it forever. + OrphanScanner.markFailed(quarantinePath, detail); + LOG.error("{} -- the slot has been set aside at {} and the affected data must be resent; " + + "this sender continues on a fresh, empty slot at {}", + detail, quarantinePath, slotPath); + return new CursorSendEngine(slotPath, sfMaxBytes, sfMaxTotalBytes, sfAppendDeadlineNanos); + } + + @TestOnly + public static void setQuarantineAfterCloseHookForTest(Runnable hook) { + quarantineAfterCloseHook = hook; + } + + @TestOnly + public static void setQuarantineFilesFacadeForTest(FilesFacade ff) { + quarantineFilesFacade = ff == null ? FilesFacade.INSTANCE : ff; + } + private static int resolveIPv4(String host) { try { byte[] addr = InetAddress.getByName(host).getAddress(); @@ -3413,6 +3598,12 @@ private LineSenderBuilder fromConfig(CharSequence configurationString) { } pos = getValue(configurationString, pos, sink, "poison_min_escalation_window_millis"); poisonMinEscalationWindowMillis(parseLongValue(sink, "poison_min_escalation_window_millis")); + } else if (Chars.equals("catchup_cap_gap_min_escalation_window_millis", sink)) { + if (protocol != PROTOCOL_WEBSOCKET) { + throw new LineSenderException("catchup_cap_gap_min_escalation_window_millis is only supported for WebSocket transport"); + } + pos = getValue(configurationString, pos, sink, "catchup_cap_gap_min_escalation_window_millis"); + catchUpCapGapMinEscalationWindowMillis(parseLongValue(sink, "catchup_cap_gap_min_escalation_window_millis")); } else if (Chars.equals("initial_connect_retry", sink)) { if (protocol != PROTOCOL_WEBSOCKET) { throw new LineSenderException("initial_connect_retry is only supported for WebSocket transport"); @@ -3686,6 +3877,9 @@ private LineSenderBuilder fromConfigWebSocket(CharSequence configurationString) if (view.has("poison_min_escalation_window_millis")) { poisonMinEscalationWindowMillis(wsLong(view, v, "poison_min_escalation_window_millis")); } + if (view.has("catchup_cap_gap_min_escalation_window_millis")) { + catchUpCapGapMinEscalationWindowMillis(wsLong(view, v, "catchup_cap_gap_min_escalation_window_millis")); + } if (view.has("sf_append_deadline_millis")) { sfAppendDeadlineMillis(wsLong(view, v, "sf_append_deadline_millis")); } @@ -3862,6 +4056,10 @@ public java.util.Map wsConfigSnapshotForTest() { m.put("max_background_drainers", maxBackgroundDrainers); m.put("max_frame_rejections", maxFrameRejections); m.put("poison_min_escalation_window_millis", poisonMinEscalationWindowMillis); + m.put("catchup_cap_gap_min_escalation_window_millis", + catchUpCapGapMinEscalationWindowMillis == PARAMETER_NOT_SET_EXPLICITLY + ? CursorWebSocketSendLoop.DEFAULT_CATCHUP_CAP_GAP_MIN_ESCALATION_WINDOW_MILLIS + : catchUpCapGapMinEscalationWindowMillis); m.put("error_inbox_capacity", errorInboxCapacity); m.put("connection_listener_inbox_capacity", connectionListenerInboxCapacity); m.put("token", httpToken); diff --git a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java index fd16dca7..ef5a6bf4 100644 --- a/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java +++ b/core/src/main/java/io/questdb/client/cutlass/http/client/WebSocketClient.java @@ -460,6 +460,35 @@ public void sendBinary(long dataPtr, int length, int timeout) { sendBuffer.reset(); } + /** + * Sends two native-memory slices as one WebSocket binary frame. The slices + * are copied directly into the WebSocket send buffer and masked there, so a + * caller assembling a small protocol prefix around a large immutable body + * does not need a second contiguous staging buffer. + * + * @param firstPtr pointer to the first payload slice + * @param firstLength first payload-slice length + * @param secondPtr pointer to the second payload slice + * @param secondLength second payload-slice length + * @param timeout timeout in milliseconds + */ + public void sendBinary( + long firstPtr, + int firstLength, + long secondPtr, + int secondLength, + int timeout + ) { + checkConnected(); + sendBuffer.reset(); + sendBuffer.beginFrame(); + sendBuffer.putBlockOfBytes(firstPtr, firstLength); + sendBuffer.putBlockOfBytes(secondPtr, secondLength); + WebSocketSendBuffer.FrameInfo frame = sendBuffer.endBinaryFrame(); + doSend(sendBuffer.getBufferPtr() + frame.offset, frame.length, timeout); + sendBuffer.reset(); + } + /** * Sends binary data with default timeout. */ @@ -467,6 +496,13 @@ public void sendBinary(long dataPtr, int length) { sendBinary(dataPtr, length, defaultTimeout); } + /** + * Sends two native-memory slices as one binary frame with the default timeout. + */ + public void sendBinary(long firstPtr, int firstLength, long secondPtr, int secondLength) { + sendBinary(firstPtr, firstLength, secondPtr, secondLength, defaultTimeout); + } + /** * Sends a close frame. */ diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/GlobalSymbolDictionary.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/GlobalSymbolDictionary.java index 3b4c9a90..d1b0f13e 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/GlobalSymbolDictionary.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/GlobalSymbolDictionary.java @@ -50,6 +50,37 @@ public GlobalSymbolDictionary(int initialCapacity) { this.idToSymbol = new ObjList<>(initialCapacity); } + /** + * Appends {@code symbol} at the next sequential id, matching a recovered / + * persisted dictionary's dense id order, WITHOUT de-duplicating. + *

+ * Recovery ({@code QwpWebSocketSender.seedGlobalDictionaryFromPersisted}) + * replays the persisted entries in id order to rebuild this dictionary. It must + * NOT collapse two source strings that decode to the same characters, because + * the persisted {@code .symbol-dict}, the on-wire delta and the I/O-thread + * catch-up mirror all key on the entry POSITION (id), not on the string. The + * only strings that collide this way are malformed lone UTF-16 surrogates, + * which the UTF-8 encoder maps to {@code '?'}: {@link #getOrAddSymbol} would + * de-dup them and leave this dictionary SHORTER than the persisted entry count, + * desyncing the producer's delta baseline from the catch-up mirror (which uses + * {@code pd.size()}) and silently misattributing later symbols. Appending + * unconditionally keeps {@link #size()} equal to that count. The reverse lookup + * keeps the highest id for a colliding string, which is harmless: both ids + * encode to the same bytes, so resolving either is equivalent. + * + * @param symbol the recovered symbol string (must not be null) + * @return the id assigned (the previous {@link #size()}) + */ + public int addRecoveredSymbol(String symbol) { + if (symbol == null) { + throw new IllegalArgumentException("symbol cannot be null"); + } + int newId = idToSymbol.size(); + symbolToId.put(symbol, newId); + idToSymbol.add(symbol); + return newId; + } + /** * Clears all symbols from the dictionary. *

diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/NativeBufferWriter.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/NativeBufferWriter.java index aad95f3c..86343772 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/NativeBufferWriter.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/NativeBufferWriter.java @@ -76,6 +76,27 @@ public static int varintSize(long value) { return (64 - Long.numberOfLeadingZeros(value) + 6) / 7; } + /** + * Writes {@code value} as an unsigned LEB128 varint directly at native address + * {@code addr} and returns the address just past the last byte. The canonical + * raw-address varint writer shared by the SF cursor's persisted dictionary and + * catch-up frame builder. + *

+ * {@code value} must be non-negative: the signed {@code value > 0x7F} loop emits + * a SINGLE truncated byte for a negative long, whereas {@link #varintSize} + * returns 10 for it -- a size/write mismatch that would corrupt the stream. All + * callers pass ids/lengths/counts (non-negative); the assert pins that contract. + */ + public static long writeVarint(long addr, long value) { + assert value >= 0 : "unsigned LEB128 varint requires a non-negative value: " + value; + while (value > 0x7F) { + Unsafe.getUnsafe().putByte(addr++, (byte) ((value & 0x7F) | 0x80)); + value >>>= 7; + } + Unsafe.getUnsafe().putByte(addr++, (byte) value); + return addr; + } + @Override public void close() { if (bufferPtr != 0) { @@ -305,6 +326,7 @@ public void putUtf8(CharSequence value) { */ @Override public void putVarint(long value) { + assert value >= 0 : "unsigned LEB128 varint requires a non-negative value: " + value; ensureCapacity(10); // max varint bytes long addr = bufferPtr + position; while (value > 0x7F) { @@ -336,11 +358,7 @@ public void skip(int bytes) { } private static void writeVarintDirect(long addr, long value) { - while (value > 0x7F) { - Unsafe.getUnsafe().putByte(addr++, (byte) ((value & 0x7F) | 0x80)); - value >>>= 7; - } - Unsafe.getUnsafe().putByte(addr, (byte) value); + writeVarint(addr, value); } private void encodeUtf8(CharSequence value, int utf8Len) { diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java index ced1a1b5..f38d9e93 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketEncoder.java @@ -26,6 +26,8 @@ import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; import io.questdb.client.std.QuietCloseable; +import io.questdb.client.std.Unsafe; +import io.questdb.client.std.Vect; import static io.questdb.client.cutlass.qwp.protocol.QwpConstants.*; @@ -39,6 +41,16 @@ public class QwpWebSocketEncoder implements QuietCloseable { private final QwpColumnWriter columnWriter = new QwpColumnWriter(); private NativeBufferWriter buffer; + // Byte offsets, within the buffer, of the symbol-dict delta ENTRY region + // ([len][utf8]... only, without the two section varints) that beginMessage + // last wrote. Let the producer persist those bytes straight to the slot's + // .symbol-dict instead of re-encoding the same symbols (see + // QwpWebSocketSender.persistNewSymbolsBeforePublish). Valid until the next + // beginMessage; stored as offsets so they survive a buffer realloc. + private int deltaCount; + private int deltaEntriesEnd; + private int deltaEntriesStart; + private int deltaStart; // QWP ingress always advertises Gorilla timestamp encoding. The column // writer still emits a per-column encoding byte and falls back to raw // values when delta-of-delta overflows int32. @@ -65,8 +77,8 @@ public void beginMessage( int batchMaxId ) { buffer.reset(); - int deltaStart = confirmedMaxId + 1; - int deltaCount = Math.max(0, batchMaxId - confirmedMaxId); + deltaStart = confirmedMaxId + 1; + deltaCount = Math.max(0, batchMaxId - confirmedMaxId); byte headerFlags = (byte) (flags | FLAG_DELTA_SYMBOL_DICT); byte origFlags = flags; flags = headerFlags; @@ -75,10 +87,12 @@ public void beginMessage( payloadStart = buffer.getPosition(); buffer.putVarint(deltaStart); buffer.putVarint(deltaCount); + deltaEntriesStart = buffer.getPosition(); for (int id = deltaStart; id < deltaStart + deltaCount; id++) { String symbol = globalDict.getSymbol(id); buffer.putString(symbol); } + deltaEntriesEnd = buffer.getPosition(); columnWriter.setBuffer(buffer); } @@ -90,6 +104,64 @@ public void close() { } } + /** + * Copies one single-table split message from the combined message currently + * staged in this encoder. The table body is copied byte-for-byte from its + * recorded offset; columns and rows are not encoded again. + */ + public int copySplitMessage( + MicrobatchBuffer target, + int tableBodyOffset, + int tableBodyLength, + boolean deferCommit, + int confirmedMaxId, + int batchMaxId + ) { + if (target.getBufferPos() != 0) { + throw new IllegalStateException("split message target is not empty"); + } + if (tableBodyOffset < deltaEntriesEnd + || tableBodyLength < 0 + || (long) tableBodyOffset + tableBodyLength > buffer.getPosition()) { + throw new IllegalArgumentException("table body slice is outside the staged message"); + } + + int splitDeltaStart = confirmedMaxId + 1; + int splitDeltaCount = Math.max(0, batchMaxId - confirmedMaxId); + int deltaEntriesLength = splitDeltaEntriesLength(splitDeltaStart, splitDeltaCount); + int messageSize = splitMessageSize( + tableBodyLength, splitDeltaStart, splitDeltaCount, deltaEntriesLength); + target.ensureCapacity(messageSize); + + long source = buffer.getBufferPtr(); + long destination = target.getBufferPtr(); + Vect.memcpy(destination, source, HEADER_SIZE); + + byte splitFlags = Unsafe.getUnsafe().getByte(source + HEADER_OFFSET_FLAGS); + if (deferCommit) { + splitFlags |= FLAG_DEFER_COMMIT; + } else { + splitFlags &= ~FLAG_DEFER_COMMIT; + } + Unsafe.getUnsafe().putByte(destination + HEADER_OFFSET_FLAGS, splitFlags); + Unsafe.getUnsafe().putShort(destination + 6, (short) 1); + Unsafe.getUnsafe().putInt(destination + 8, messageSize - HEADER_SIZE); + + long writeAddress = destination + HEADER_SIZE; + writeAddress = NativeBufferWriter.writeVarint(writeAddress, splitDeltaStart); + writeAddress = NativeBufferWriter.writeVarint(writeAddress, splitDeltaCount); + if (deltaEntriesLength > 0) { + Vect.memcpy(writeAddress, source + deltaEntriesStart, deltaEntriesLength); + writeAddress += deltaEntriesLength; + } + Vect.memcpy(writeAddress, source + tableBodyOffset, tableBodyLength); + writeAddress += tableBodyLength; + assert writeAddress == destination + messageSize; + + target.setBufferPos(messageSize); + return messageSize; + } + public int encode(QwpTableBuffer tableBuffer) { buffer.reset(); writeHeader(1, 0); @@ -122,6 +194,33 @@ public QwpBufferWriter getBuffer() { return buffer; } + /** + * Byte length of the symbol-dict delta ENTRY region ({@code [len][utf8]...}, + * excluding the two section varints) that {@link #beginMessage} last wrote. + */ + public int getDeltaEntriesLen() { + return deltaEntriesEnd - deltaEntriesStart; + } + + /** + * Byte offset, within {@link #getBuffer()}, of the symbol-dict delta ENTRY + * region {@link #beginMessage} last wrote. + */ + public int getDeltaEntriesStart() { + return deltaEntriesStart; + } + + public int getSplitMessageSize(int tableBodyLength, int confirmedMaxId, int batchMaxId) { + if (tableBodyLength < 0) { + throw new IllegalArgumentException("tableBodyLength must be non-negative"); + } + int splitDeltaStart = confirmedMaxId + 1; + int splitDeltaCount = Math.max(0, batchMaxId - confirmedMaxId); + int deltaEntriesLength = splitDeltaEntriesLength(splitDeltaStart, splitDeltaCount); + return splitMessageSize( + tableBodyLength, splitDeltaStart, splitDeltaCount, deltaEntriesLength); + } + public void setDeferCommit(boolean defer) { if (defer) { flags |= FLAG_DEFER_COMMIT; @@ -144,4 +243,35 @@ public void writeHeader(int tableCount, int payloadLength) { buffer.putShort((short) tableCount); buffer.putInt(payloadLength); } + + private int splitDeltaEntriesLength(int splitDeltaStart, int splitDeltaCount) { + if (splitDeltaCount == 0) { + return 0; + } + if (splitDeltaStart != deltaStart || splitDeltaCount != deltaCount) { + throw new IllegalStateException("split delta does not match the staged message" + + " [stagedStart=" + deltaStart + + ", stagedCount=" + deltaCount + + ", splitStart=" + splitDeltaStart + + ", splitCount=" + splitDeltaCount + ']'); + } + return deltaEntriesEnd - deltaEntriesStart; + } + + private int splitMessageSize( + int tableBodyLength, + int splitDeltaStart, + int splitDeltaCount, + int deltaEntriesLength + ) { + long messageSize = (long) HEADER_SIZE + + NativeBufferWriter.varintSize(splitDeltaStart) + + NativeBufferWriter.varintSize(splitDeltaCount) + + deltaEntriesLength + + tableBodyLength; + if (messageSize > Integer.MAX_VALUE) { + throw new OutOfMemoryError("split QWP message size overflow: " + messageSize); + } + return (int) messageSize; + } } diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java index d1744065..28bb70f2 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/QwpWebSocketSender.java @@ -46,9 +46,11 @@ import io.questdb.client.cutlass.qwp.client.sf.cursor.DefaultSenderConnectionListener; import io.questdb.client.cutlass.qwp.client.sf.cursor.DefaultSenderErrorHandler; import io.questdb.client.cutlass.qwp.client.sf.cursor.DefaultSenderProgressHandler; +import io.questdb.client.cutlass.qwp.client.sf.cursor.PersistedSymbolDict; import io.questdb.client.cutlass.qwp.client.sf.cursor.SenderConnectionDispatcher; import io.questdb.client.cutlass.qwp.client.sf.cursor.SenderErrorDispatcher; import io.questdb.client.cutlass.qwp.client.sf.cursor.SenderProgressDispatcher; +import io.questdb.client.cutlass.qwp.client.sf.cursor.UnreplayableSlotException; import io.questdb.client.cutlass.qwp.protocol.QwpConstants; import io.questdb.client.cutlass.qwp.protocol.QwpTableBuffer; import io.questdb.client.std.CharSequenceObjHashMap; @@ -56,6 +58,7 @@ import io.questdb.client.std.Decimal128; import io.questdb.client.std.Decimal256; import io.questdb.client.std.Decimal64; +import io.questdb.client.std.IntList; import io.questdb.client.std.Misc; import io.questdb.client.std.Numbers; import io.questdb.client.std.NumericException; @@ -168,6 +171,11 @@ public class QwpWebSocketSender implements Sender { // behind a drainer's endpoint walk. private final ReentrantLock connectWalkLock = new ReentrantLock(); private final QwpHostHealthTracker hostTracker; + // Per-table encoded body byte counts captured during flushPendingRows' combined + // encode. flushPendingRowsSplit uses them both for preflight sizing and to walk + // the staged body slices without encoding the batch a second time. Cleared and + // repopulated on every flush; reused to stay zero-GC. + private final IntList splitFrameBodyBytes = new IntList(); private final CharSequenceObjHashMap tableBuffers; // null means plain text (no TLS) private final ClientTlsConfiguration tlsConfig; @@ -221,6 +229,17 @@ public class QwpWebSocketSender implements Sender { private CursorSendEngine cursorEngine; private CursorWebSocketSendLoop cursorSendLoop; private boolean deferCommit; + // True when the sender emits incremental (delta) symbol dictionaries: each + // message carries only symbol ids not yet sent on the wire, rather than the + // full dictionary from id 0. Enabled in memory-mode (a reconnect replays from + // the in-process ring) and in file-mode store-and-forward when the per-slot + // persisted dictionary opened. In both, the I/O thread re-registers the whole + // dictionary via a catch-up frame before replaying, so a non-self-sufficient + // delta frame never dangles an id on a fresh server. Falls back to full + // self-sufficient frames only when the persisted dictionary is unavailable in + // file-mode (recovery/orphan-drain would then have nothing to rebuild the + // deltas from). Set in setCursorEngine. + private boolean deltaDictEnabled; // User-supplied observer for background orphan-slot drainer events. // Volatile: written by setDrainerListener (any thread, before or after // startOrphanDrainers) and read at pool-creation time. Null -> drainers @@ -271,9 +290,8 @@ public class QwpWebSocketSender implements Sender { // still terminal. // ASYNC → user thread does not connect at all. The I/O thread runs // the reconnect loop in the background, indefinitely - // (Invariant B); terminal failures (auth/upgrade reject) - // are delivered to the SenderError dispatcher rather than - // thrown from the constructor. + // (Invariant B); endpoint-policy and transport failures stay + // contained in that loop and never reach the producer. private Sender.InitialConnectMode initialConnectMode = Sender.InitialConnectMode.OFF; private boolean ownsCursorEngine; private long pendingBytes; @@ -295,6 +313,12 @@ public class QwpWebSocketSender implements Sender { // maxFrameRejections (connect-string key poison_min_escalation_window_millis). private long poisonMinEscalationWindowMillis = CursorWebSocketSendLoop.DEFAULT_POISON_MIN_ESCALATION_WINDOW_MILLIS; + // Minimum wall-clock dwell a symbol-dict catch-up cap gap must persist before an + // orphan drainer may quarantine its slot (connect-string key + // catchup_cap_gap_min_escalation_window_millis). Foreground senders retry forever. See + // CursorWebSocketSendLoop.DEFAULT_CATCHUP_CAP_GAP_MIN_ESCALATION_WINDOW_MILLIS. + private long catchUpCapGapMinEscalationWindowMillis = + CursorWebSocketSendLoop.DEFAULT_CATCHUP_CAP_GAP_MIN_ESCALATION_WINDOW_MILLIS; private long reconnectInitialBackoffMillis = CursorWebSocketSendLoop.DEFAULT_RECONNECT_INITIAL_BACKOFF_MILLIS; private long reconnectMaxBackoffMillis = @@ -313,6 +337,12 @@ public class QwpWebSocketSender implements Sender { // beginRound(true) call. roundSeq=1 is the first round; CONNECTED in the // first round indicates the initial connect. private long roundSeq; + // Highest global symbol id the producer has baked into a frame so far, or -1. + // Lifetime-monotonic in delta mode -- it is NOT reset on reconnect, because + // the I/O thread re-registers the full dictionary via a catch-up frame before + // replaying, so the producer's delta baseline stays valid across the wire + // boundary. Used only when deltaDictEnabled; ignored in full-dict mode. + private int sentMaxSymbolId = -1; // When true, auto-flush sends messages with FLAG_DEFER_COMMIT and only // explicit flush() triggers the server-side commit. Enables accumulating // arbitrarily large datasets that exceed the server's recv buffer. @@ -667,7 +697,47 @@ public static QwpWebSocketSender connect( durableAckKeepaliveIntervalMillis, authTimeoutMs, connectTimeoutMs, connectionListener, connectionListenerInboxCapacity, CursorWebSocketSendLoop.DEFAULT_MAX_HEAD_FRAME_REJECTIONS, - CursorWebSocketSendLoop.DEFAULT_POISON_MIN_ESCALATION_WINDOW_MILLIS); + CursorWebSocketSendLoop.DEFAULT_POISON_MIN_ESCALATION_WINDOW_MILLIS, + CursorWebSocketSendLoop.DEFAULT_CATCHUP_CAP_GAP_MIN_ESCALATION_WINDOW_MILLIS); + } + + /** + * Compatibility overload retained for callers compiled before the + * catch-up cap-gap dwell was added to the master signature. + */ + public static QwpWebSocketSender connect( + List endpoints, + ClientTlsConfiguration tlsConfig, + int autoFlushRows, + int autoFlushBytes, + long autoFlushIntervalNanos, + String authorizationHeader, + boolean requestDurableAck, + CursorSendEngine cursorEngine, + long closeFlushTimeoutMillis, + long reconnectMaxDurationMillis, + long reconnectInitialBackoffMillis, + long reconnectMaxBackoffMillis, + Sender.InitialConnectMode initialConnectMode, + SenderErrorHandler errorHandler, + int errorInboxCapacity, + long durableAckKeepaliveIntervalMillis, + long authTimeoutMs, + int connectTimeoutMs, + SenderConnectionListener connectionListener, + int connectionListenerInboxCapacity, + int maxFrameRejections, + long poisonMinEscalationWindowMillis + ) { + return connect(endpoints, tlsConfig, autoFlushRows, autoFlushBytes, + autoFlushIntervalNanos, authorizationHeader, requestDurableAck, + cursorEngine, closeFlushTimeoutMillis, reconnectMaxDurationMillis, + reconnectInitialBackoffMillis, reconnectMaxBackoffMillis, + initialConnectMode, errorHandler, errorInboxCapacity, + durableAckKeepaliveIntervalMillis, authTimeoutMs, connectTimeoutMs, + connectionListener, connectionListenerInboxCapacity, + maxFrameRejections, poisonMinEscalationWindowMillis, + CursorWebSocketSendLoop.DEFAULT_CATCHUP_CAP_GAP_MIN_ESCALATION_WINDOW_MILLIS); } /** @@ -698,7 +768,8 @@ public static QwpWebSocketSender connect( SenderConnectionListener connectionListener, int connectionListenerInboxCapacity, int maxFrameRejections, - long poisonMinEscalationWindowMillis + long poisonMinEscalationWindowMillis, + long catchUpCapGapMinEscalationWindowMillis ) { QwpWebSocketSender sender = new QwpWebSocketSender( endpoints, tlsConfig, @@ -716,6 +787,7 @@ public static QwpWebSocketSender connect( sender.durableAckKeepaliveIntervalMillis = durableAckKeepaliveIntervalMillis; sender.maxFrameRejections = maxFrameRejections; sender.poisonMinEscalationWindowMillis = poisonMinEscalationWindowMillis; + sender.catchUpCapGapMinEscalationWindowMillis = catchUpCapGapMinEscalationWindowMillis; sender.initialConnectMode = initialConnectMode == null ? Sender.InitialConnectMode.OFF : initialConnectMode; @@ -1762,6 +1834,38 @@ public long getPendingBytes() { return pendingBytes; } + /** + * Snapshot of the producer's symbol prefix whose persisted-dictionary chunks + * have committed. The persisted size advances only after the chunk CRC and + * payload have been written, so this observes the write-ahead boundary without + * reopening the live mmap-backed dictionary (which would attempt recovery-tail + * truncation and is not supported while the producer owns the file on Windows). + * Returns {@code null} in memory mode or when the persisted dictionary is + * unavailable. + */ + @TestOnly + public ObjList getPersistedSymbolsForTest() { + CursorSendEngine engine = cursorEngine; + if (engine == null) { + return null; + } + PersistedSymbolDict persisted = engine.getPersistedSymbolDict(); + if (persisted == null) { + return null; + } + int persistedSize = persisted.size(); + int globalSize = globalSymbolDictionary.size(); + if (persistedSize > globalSize) { + throw new IllegalStateException("persisted symbol dictionary exceeds producer dictionary" + + " [persisted=" + persistedSize + ", producer=" + globalSize + ']'); + } + ObjList snapshot = new ObjList<>(persistedSize); + for (int i = 0; i < persistedSize; i++) { + snapshot.add(globalSymbolDictionary.getSymbol(i)); + } + return snapshot; + } + /** * Server-advertised cap on the per-batch raw byte size. Zero before the * first connect; updated by every successful reconnect via @@ -2215,6 +2319,22 @@ public void setCursorEngine(CursorSendEngine engine, boolean takeOwnership) { } this.cursorEngine = engine; this.ownsCursorEngine = takeOwnership && engine != null; + // Delta encoding is available in memory-mode (in-process catch-up) and in + // file-mode when the persisted dictionary opened (recovery / orphan-drain + // rebuild the dictionary from it). Otherwise fall back to full self- + // sufficient frames. See CursorSendEngine.isDeltaDictEnabled. + this.deltaDictEnabled = engine != null && engine.isDeltaDictEnabled(); + // Recovery: repopulate the producer's global dictionary from the slot's + // persisted dictionary so newly ingested symbols continue from the + // recovered ids (rather than colliding with them at 0), and the delta + // baseline resumes where the crashed session left off. + // NOT gated on deltaDictEnabled. That flag is false exactly when the slot's dictionary + // failed to open -- which is precisely when the frames on disk are the only surviving + // copy of the symbols and the rebuild matters most. Gating the seed on it made the + // rebuild dead code for the very case it was written for. + if (engine != null && engine.wasRecoveredFromDisk()) { + seedGlobalDictionaryFromPersisted(engine.getPersistedSymbolDict()); + } } /** @@ -2378,7 +2498,8 @@ public synchronized void startOrphanDrainers( requestDurableAck, durableAckKeepaliveIntervalMillis, maxFrameRejections, - poisonMinEscalationWindowMillis); + poisonMinEscalationWindowMillis, + catchUpCapGapMinEscalationWindowMillis); ref[0] = drainer; drainerPool.submit(drainer); } @@ -3277,10 +3398,10 @@ private void ensureConnected() { // Encoder stays at its default (V1 -- the only supported wire // version today). Frames written before the first successful // connect commit to V1 because cursor segments are immutable; - // a future version bump must account for that. Auth/upgrade - // rejects are surfaced via the error inbox by the I/O - // thread, not thrown here; plain connect failures retry - // indefinitely (Invariant B). + // a future version bump must account for that. Auth, upgrade, + // capability and transport failures all retry indefinitely on + // the I/O thread (Invariant B); none is surfaced here or later + // to the producer. client = null; break; case OFF: @@ -3306,7 +3427,9 @@ private void ensureConnected() { requestDurableAck, durableAckKeepaliveIntervalMillis, maxFrameRejections, - poisonMinEscalationWindowMillis); + poisonMinEscalationWindowMillis, + catchUpCapGapMinEscalationWindowMillis, + CursorWebSocketSendLoop.ReconnectPolicy.FOREGROUND); // Plug the async-delivery sink before start() so the I/O thread // never observes a null dispatcher between recordFatal and // notification — the test for null in dispatchError handles @@ -3332,6 +3455,17 @@ private void ensureConnected() { cursorSendLoop.setConnectionDispatcher(connectionDispatcher); cursorSendLoop.start(); } catch (Throwable t) { + // start() (or dispatcher construction) failed after cursorSendLoop was + // assigned. Close it so a caller that retries -- re-entering + // ensureConnected and reassigning cursorSendLoop above -- cannot orphan + // a recovered slot's ctor-seeded native mirror (freed only by close() + // or the I/O loop, neither of which has run). close() is idempotent and + // frees the mirror via its loopNeverRan path; it also closes the shared + // client, so the client.close() below is a safe idempotent no-op. + if (cursorSendLoop != null) { + cursorSendLoop.close(); + cursorSendLoop = null; + } if (client != null) { client.close(); client = null; @@ -3359,18 +3493,18 @@ private void ensureConnected() { host, port, client.getServerQwpVersion(), serverMaxBatchSize, effectiveAutoFlushBytes); } else { // Async mode: I/O thread will drive the connect. Encoder uses - // its default version (V1). The symbol-dict watermark still gets - // reset for consistency with the sync path; the post-connect replay - // path does not need a producer-side reset signal because every - // cursor frame is self-sufficient. + // its default version (V1). The per-batch symbol-dict watermark still + // gets reset for consistency with the sync path; the post-connect + // replay path needs no producer-side reset signal (see below). Endpoint ep = endpoints.get(0); LOG.info("Async initial connect deferred to I/O thread [firstHost={}, firstPort={}, endpointCount={}]", ep.host, ep.port, endpoints.size()); } - // Server starts fresh on each connection, so reset the symbol-dict - // watermark. Cursor frames are self-sufficient (every frame carries its - // full inline schema + a symbol-dict delta from id 0), so post-reconnect - // replay needs no producer-side reset signal. + // Server starts fresh on each connection, so reset the per-batch + // symbol-dict watermark. Every frame still carries its full inline schema, + // and the fresh server's dictionary is re-established either by a full-dict + // frame (full-dict mode) or by an I/O-thread catch-up frame before replay + // (delta mode), so post-reconnect replay needs no producer-side reset signal. resetSymbolDictStateForNewConnection(); connectionError.set(null); @@ -3423,14 +3557,23 @@ private void flushPendingRows(boolean deferCommit) { } ensureActiveBufferReady(); - // Cursor SF requires every on-disk frame to be self-sufficient: - // recorded frames replay to fresh server connections (orphan-slot - // drainers and post-reconnect replay), so always emit the full - // symbol-dict delta from id=0 and the full column schema inline, - // never a back-reference the target server may not have seen. + // In full-dict mode every frame is self-sufficient: it carries the whole + // symbol dictionary from id 0 so orphan-drain / recovery replay to a fresh + // server never dangles a symbol id. In delta mode (memory-mode, and + // file-mode store-and-forward once the persisted dictionary opened) each + // frame carries only ids above sentMaxSymbolId; a reconnect re-registers + // the dictionary via an I/O-thread catch-up frame before replay, so the + // producer's monotonic baseline stays valid across the wire boundary. encoder.setDeferCommit(deferCommit); encoder.beginMessage(tableCount, globalSymbolDictionary, - /*confirmedMaxId=*/ -1, currentBatchMaxSymbolId); + symbolDeltaBaseline(), currentBatchMaxSymbolId); + // Record each table's encoded body size (position delta across addTable). + // When the batch needs splitting, these lengths delimit immutable body + // slices in the combined encoder buffer for direct frame assembly. The + // capture is a couple of int ops per table on the common path. + splitFrameBodyBytes.clear(); + int combinedBodyStart = encoder.getBuffer().getPosition(); + int bodyStart = combinedBodyStart; for (int i = 0, n = keys.size(); i < n; i++) { CharSequence tableName = keys.getQuick(i); if (tableName == null) { @@ -3447,19 +3590,40 @@ private void flushPendingRows(boolean deferCommit) { } encoder.addTable(tableBuffer); + int bodyEnd = encoder.getBuffer().getPosition(); + splitFrameBodyBytes.add(bodyEnd - bodyStart); + bodyStart = bodyEnd; } int messageSize = encoder.finishMessage(); QwpBufferWriter buffer = encoder.getBuffer(); - if (serverMaxBatchSize > 0 && messageSize > serverMaxBatchSize) { - flushPendingRowsSplit(keys, deferCommit); + // Snapshot the volatile cap ONCE for this whole flush. The I/O thread lowers + // serverMaxBatchSize on a mid-stream failover to a smaller-cap node + // (applyServerBatchSizeLimit); if the split pre-flight and the publish loop + // re-read the field independently, a failover between them would size frames + // against two different caps -- breaking the all-or-nothing guarantee and + // firing the publish-loop assert on a legitimate race. Both use this snapshot; + // the next flush picks up the new cap. + int cap = serverMaxBatchSize; + if (cap > 0 && messageSize > cap) { + // Keep the completed combined frame staged in the encoder while the + // split path copies its delta entries and table-body slices. + flushPendingRowsSplit(keys, deferCommit, combinedBodyStart, cap); return; } + // Write-ahead: durably persist this frame's new symbols BEFORE it is + // published, so a recovered/orphan-drained slot can always rebuild the + // dictionary the (non-self-sufficient) delta frame references. No-op in + // memory mode and when the frame introduces no new symbols. + persistNewSymbolsBeforePublish(); activeBuffer.ensureCapacity(messageSize); activeBuffer.write(buffer.getBufferPtr(), messageSize); activeBuffer.incrementRowCount(); sealAndSwapBuffer(); + // The frame carrying ids up to currentBatchMaxSymbolId is now on the ring; + // advance the delta baseline so the next frame ships only newer ids. + advanceSentMaxSymbolId(); hasDeferredMessages = deferCommit; if (!deferCommit) { @@ -3475,30 +3639,97 @@ private void flushPendingRows(boolean deferCommit) { * own message. All messages except the last carry FLAG_DEFER_COMMIT * so the server appends rows without committing until the final * message arrives. + *

+ * Not atomic across frames. The frames publish one at a time, so a + * publish failure partway through -- {@link #sealAndSwapBuffer()} throwing on + * frame k>1, e.g. a backpressure deadline or the buffer-recycle timeout -- + * leaves frames 1..k-1 already on the ring as deferred (appended, not yet + * committed). The throw propagates past the {@code resetTableBuffersAfterFlush} + * at the end of the loop, so the source rows survive in their table buffers + * and the NEXT flush re-emits the whole batch; the eventual commit then + * commits the already-published prefix alongside the re-sent copies, + * delivering those rows at-least-once (duplicated), not exactly-once. This is + * within store-and-forward's at-least-once contract -- a DEDUP table or a + * durable-ack await absorbs the duplicate, and the symbol-dict state stays + * consistent on the retry (the re-sent frames carry empty deltas and the + * write-ahead persist is a {@code pd.size()} no-op). Making the split atomic + * (rolling back the published prefix, or skipping it on retry) would be a + * larger change. * * @param deferCommit when true, ALL messages (including the last) * carry FLAG_DEFER_COMMIT. When false, only the * last message omits the flag. */ - private void flushPendingRowsSplit(ObjList keys, boolean deferCommit) { + private void flushPendingRowsSplit( + ObjList keys, + boolean deferCommit, + int combinedBodyStart, + int cap + ) { if (LOG.isDebugEnabled()) { - LOG.debug("Splitting flush across multiple messages [serverMaxBatchSize={}, defer={}]", serverMaxBatchSize, deferCommit); - } - - // Collect non-empty table indices so we know which is last. + LOG.debug("Splitting flush across multiple messages [serverMaxBatchSize={}, defer={}]", cap, deferCommit); + } + + // Collect non-empty table indices so we know which is last, AND pre-flight + // every split frame's size BEFORE publishing any of them. The split hands + // frames to the ring one at a time (all but the last deferred -- appended but + // uncommitted); if a later table's frame were only found oversized + // mid-publish, the already-published prefix would strand on the ring, a + // subsequent commit would deliver it as a partial batch, and + // resetTableBuffersAfterFlush would discard every source row -- a partial + // commit the caller was told (by the throw) had failed. Checking all sizes up + // front makes the split all-or-nothing: either every frame fits and all + // publish, or none publish and we throw with nothing stranded. + // + // Each split frame's size is derived from the combined encode flushPendingRows + // already performed. simBaseline mirrors the publish loop's baseline advance + // (advanceSentMaxSymbolId), so each size equals the frame the staged-slice + // assembler will build; this pass mutates no delta/persist state. int nonEmptyCount = 0; + int simBaseline = symbolDeltaBaseline(); + int bodyIdx = 0; for (int i = 0, n = keys.size(); i < n; i++) { CharSequence tableName = keys.getQuick(i); if (tableName == null) { continue; } QwpTableBuffer tableBuffer = tableBuffers.get(tableName); - if (tableBuffer != null && tableBuffer.getRowCount() > 0) { - nonEmptyCount++; + if (tableBuffer == null || tableBuffer.getRowCount() == 0) { + continue; + } + nonEmptyCount++; + int messageSize = encoder.getSplitMessageSize( + splitFrameBodyBytes.getQuick(bodyIdx), simBaseline, currentBatchMaxSymbolId); + bodyIdx++; + if (messageSize > cap) { + // The batch stays BUFFERED: this throw precedes every publish, and a + // rejected flush must not silently discard the caller's rows (see + // SelfSufficientFramesTest#testOversizedTableSplitStrandsNothing). So the + // next flush() re-encodes and re-rejects the same batch until either the + // reachable cap grows -- a failover to a larger-cap node, which is the + // case retaining the rows exists to survive -- or the sender is closed, + // which discards them. It cannot drain against a cap this table will + // never fit, so say that here rather than let a caller read the repeat + // rejections as a transient and keep appending to a batch that only + // grows. + throw new LineSenderException("single table batch too large for server batch cap") + .put(" [table=").put(tableName) + .put(", messageSize=").put(messageSize) + .put(", serverMaxBatchSize=").put(cap).put(']') + .put("; the batch is retained for retry and every flush() will " + + "reject it again until a larger-cap node is reached -- " + + "close the sender to discard it, or produce smaller batches"); + } + // Mirror advanceSentMaxSymbolId: once the first frame ships the batch's + // new ids, the remaining frames carry an empty delta above the baseline. + if (deltaDictEnabled && currentBatchMaxSymbolId > simBaseline) { + simBaseline = currentBatchMaxSymbolId; } } int sent = 0; + bodyIdx = 0; + int tableBodyOffset = combinedBodyStart; for (int i = 0, n = keys.size(); i < n; i++) { CharSequence tableName = keys.getQuick(i); if (tableName == null) { @@ -3513,26 +3744,43 @@ private void flushPendingRowsSplit(ObjList keys, boolean deferComm boolean isLast = (sent == nonEmptyCount); boolean deferThis = deferCommit || !isLast; - encoder.setDeferCommit(deferThis); - encoder.beginMessage(1, globalSymbolDictionary, - /*confirmedMaxId=*/ -1, currentBatchMaxSymbolId); - encoder.addTable(tableBuffer); - int messageSize = encoder.finishMessage(); - QwpBufferWriter buffer = encoder.getBuffer(); - - if (messageSize > serverMaxBatchSize) { - resetTableBuffersAfterFlush(keys); - throw new LineSenderException("single table batch too large for server batch cap") - .put(" [table=").put(tableName) - .put(", messageSize=").put(messageSize) - .put(", serverMaxBatchSize=").put(serverMaxBatchSize).put(']'); - } - + int tableBodyLength = splitFrameBodyBytes.getQuick(bodyIdx++); + // Persist before touching activeBuffer. If the write-ahead fails, the + // caller can retry with both the source rows and active microbatch + // unchanged. The first frame carries the batch's new symbols; later + // frames are no-ops once the baseline has advanced. + persistNewSymbolsBeforePublish(); ensureActiveBufferReady(); - activeBuffer.ensureCapacity(messageSize); - activeBuffer.write(buffer.getBufferPtr(), messageSize); + // The combined encoder buffer remains immutable for the whole split. + // Assemble this frame directly into the active microbatch: patched + // header + staged delta bytes + the staged table-body slice. No row or + // column is encoded a second time. + int messageSize = encoder.copySplitMessage( + activeBuffer, + tableBodyOffset, + tableBodyLength, + deferThis, + symbolDeltaBaseline(), + currentBatchMaxSymbolId + ); + tableBodyOffset += tableBodyLength; + // The pre-flight pass above already verified every split frame fits the + // cap, so none can be found oversized here -- which is what keeps this + // loop from publishing (and stranding) a deferred prefix before an + // oversized table. Both passes size against the SAME snapshot cap, so a + // mid-flush failover cannot make them disagree; the assert therefore only + // catches a genuine divergence between the pre-flight arithmetic and the + // staged assembler (a future bug), not a cap race. It deliberately does NOT + // reset+throw here, because by this point a prefix may already be on the ring. + assert messageSize <= cap + : "split frame exceeded serverMaxBatchSize after pre-flight [table=" + tableName + + ", messageSize=" + messageSize + ", serverMaxBatchSize=" + cap + ']'; + activeBuffer.incrementRowCount(); sealAndSwapBuffer(); + // Frame queued: advance so the next split frame's delta starts above + // the ids this one just registered. + advanceSentMaxSymbolId(); } encoder.setDeferCommit(false); @@ -3573,8 +3821,25 @@ private void sendCommitMessage() { LOG.debug("Sending commit message for deferred batch"); } encoder.setDeferCommit(false); + // A commit carries no rows, and it must also carry NO new symbols. Unlike + // the flush paths, sendCommitMessage does NOT write-ahead-persist the + // dictionary, so shipping a symbol here would put an id on the wire that a + // recovered slot cannot rebuild from the persisted .symbol-dict, diverging + // the producer dictionary from the surviving frames and silently + // misattributing reused ids after a crash. currentBatchMaxSymbolId can sit + // ABOVE sentMaxSymbolId (e.g. a cancelled row: cancelRow does not roll back + // currentBatchMaxSymbolId or unregister the symbol), so bound the delta at + // what has already been sent -- and therefore already persisted. In delta + // mode pass sentMaxSymbolId, yielding an empty delta + // [sentMaxSymbolId+1 .. sentMaxSymbolId]; in full-dict mode keep + // currentBatchMaxSymbolId (reset to -1 by the prior flush, so the commit frame + // carries an empty delta too -- harmless, since the preceding full-dict data + // frames already registered the dictionary on this connection). Any symbol a + // cancelled row leaked is picked up (and persisted) by the next real flush, + // whose persistNewSymbolsBeforePublish resumes from pd.size(). + int commitBatchMaxId = deltaDictEnabled ? sentMaxSymbolId : currentBatchMaxSymbolId; encoder.beginMessage(0, globalSymbolDictionary, - /*confirmedMaxId=*/ -1, currentBatchMaxSymbolId); + symbolDeltaBaseline(), commitBatchMaxId); int messageSize = encoder.finishMessage(); QwpBufferWriter buffer = encoder.getBuffer(); ensureActiveBufferReady(); @@ -3586,15 +3851,220 @@ private void sendCommitMessage() { lastCommitBoundaryFsn = cursorEngine.publishedFsn(); } + /** + * Advances the delta baseline once a frame carrying the current batch's + * symbols has been queued onto the ring. No-op in full-dict mode. Only ever + * moves the baseline forward, so a batch that used no new symbols leaves it + * unchanged. + */ + private void advanceSentMaxSymbolId() { + if (deltaDictEnabled && currentBatchMaxSymbolId > sentMaxSymbolId) { + sentMaxSymbolId = currentBatchMaxSymbolId; + } + } + + /** + * Appends the symbols this frame introduces ({@code [sentMaxSymbolId+1 .. + * currentBatchMaxSymbolId]}) to the slot's persisted dictionary BEFORE the + * frame is published to the ring. This write-ahead ordering keeps the + * persisted dictionary a superset of every process-crash-recoverable frame's + * references, so recovery and orphan-drain can re-register it on a fresh + * server. Not fsync'd (see PersistedSymbolDict) -- a host crash that tears it + * is caught by the send loop's replay guard. No-op in memory mode (no + * persisted dictionary) and when the frame introduces no new symbols. + */ + private void persistNewSymbolsBeforePublish() { + if (!deltaDictEnabled || cursorEngine == null) { + return; + } + PersistedSymbolDict pd = cursorEngine.getPersistedSymbolDict(); + if (pd == null) { + return; + } + // Persist [pd.size() .. currentBatchMaxSymbolId] as ONE write, BEFORE the + // frame is published. + // + // Resume from the dictionary's own durable size, NOT sentMaxSymbolId+1: + // the persist advances pd.size() only after a full write, whereas + // sentMaxSymbolId only advances after the WHOLE frame is published (via + // advanceSentMaxSymbolId, after activeBuffer.write). If a prior persist + // threw (short write -- disk full/quota) or the publish threw, the frame + // was not published and sentMaxSymbolId stayed put, while the symbols + // before the failure are already on disk. Keying the resume point off + // sentMaxSymbolId+1 would re-append that persisted prefix on the retry, + // duplicating entries and corrupting the dense id->symbol mapping recovery + // relies on (position i must be symbol id i). pd.size() resumes exactly + // past what is already durable, so the write-ahead is idempotent. + int from = pd.size(); + if (currentBatchMaxSymbolId < from) { + return; // nothing new to persist (warm batch, or an idempotent retry) + } + // Fast path: the frame the encoder just built already holds these symbols + // in its delta section as [len][utf8]... -- byte-identical to what + // PersistedSymbolDict stores. In the common case pd.size() equals the + // frame's delta start id (sentMaxSymbolId+1), so persist those bytes + // straight from the frame instead of re-encoding the symbols. After a + // failed publish the durable size has run ahead of the wire baseline, so + // the frame's delta covers MORE than remains to persist; then re-encode + // just the [from .. currentBatchMaxSymbolId] suffix. + try { + if (from == sentMaxSymbolId + 1) { + QwpBufferWriter buffer = encoder.getBuffer(); + pd.appendRawEntries( + buffer.getBufferPtr() + encoder.getDeltaEntriesStart(), + encoder.getDeltaEntriesLen(), + currentBatchMaxSymbolId - from + 1); + } else { + pd.appendSymbols(globalSymbolDictionary, from, currentBatchMaxSymbolId); + } + } catch (Throwable t) { + // A failed write to the persisted dictionary throws a low-level + // IllegalStateException: in production that is ff.allocate refusing to grow + // the mmap append window (how a full disk / exhausted quota surfaces there), + // and behind an injected facade a short positioned write. Surface it as a + // LineSenderException -- like every other flush-path failure, e.g. the cursor + // append in sealAndSwapBuffer -- so a caller catching LineSenderException + // around flush() also catches a disk-full during the write-ahead persist. The + // persist ran before publish and pd.size() did not advance on the failure, so + // the still-buffered rows re-persist the same range idempotently on retry. + // A JVM Error is never a persist failure; let it propagate. + if (t instanceof Error) { + throw (Error) t; + } + throw new LineSenderException("failed to persist symbol dictionary before publish", t); + } + } + private void resetSymbolDictStateForNewConnection() { - // The new server has an empty symbol dictionary, so the next batch - // must ship a delta starting at id 0. beginMessage() always passes - // confirmedMaxId = -1; resetting the batch watermark here keeps a - // stale value from suppressing re-emission of symbol ids the new - // server has never seen. + // Runs on the foreground (initial) connect only -- NOT on the I/O thread's + // reconnect/failover path. The per-batch watermark is drained state, so + // clearing it here is harmless. sentMaxSymbolId is deliberately left + // untouched: in delta mode the I/O thread re-registers the whole + // dictionary with a catch-up frame on reconnect, so the producer's + // monotonic baseline must survive the wire boundary; resetting it would + // desync the producer from the I/O thread's sent-dictionary count. currentBatchMaxSymbolId = -1; } + /** + * On recovery, repopulates the producer's {@link GlobalSymbolDictionary} so that newly + * ingested symbols continue ABOVE every id the surviving frames already define, and + * resumes the delta baseline at that tip. + *

+ * Seeds from TWO sources, in this order: + *

    + *
  1. the slot's persisted {@code .symbol-dict} -- its intact prefix; then
    2. + *
  2. the surviving frames' OWN delta sections, for every id above that prefix + * ({@link CursorSendEngine#addRecoveredSymbolsTo}).
    3. + *
+ * Those are exactly the two sources, in exactly the order, that the send loop's mirror + * is built from: its constructor seeds {@code sentDictCount} from the same dictionary, + * and {@code accumulateSentDict} then extends it from the same frames as they replay. So + * the producer's {@code sentMaxSymbolId + 1} and the loop's {@code sentDictCount} land on + * the same number BY CONSTRUCTION -- the invariant the torn-dictionary guard rests on -- + * rather than by the two happening to agree. + *

+ * Uses {@link GlobalSymbolDictionary#addRecoveredSymbol} (append, NOT de-dup): the + * persisted dictionary, the on-wire delta and the mirror all key on the entry POSITION + * (id), so the producer's id space must match the recovered entry count exactly. + * {@code getOrAddSymbol} would collapse two source strings that decode to the same + * characters -- only malformed lone UTF-16 surrogates, which UTF-8-encode to {@code '?'} + * -- leaving this dictionary SHORTER than the count and silently misattributing later + * symbols. + *

+ * Why seeding from the frames matters. The dictionary is not fsync'd (see + * {@code PersistedSymbolDict}), so a host/power crash can tear off its newest entries + * while the segment frames that introduced those ids survive -- and those newest frames, + * being the least likely to be acked, are exactly the ones that replay. Seeded from the + * short dictionary alone, this producer would hand its next new symbol an id those frames + * already define, putting two symbols on one id and silently misattributing values. The + * old code detected that and threw, which was safe but far too blunt: it bricked + * {@code build()} for slots the background drainer replays PERFECTLY, because the frames + * carry the torn-off symbols in their own deltas and {@code accumulateSentDict} rebuilds + * the dictionary from them. This method now rebuilds the producer from the same bytes, + * so a torn -- or entirely lost -- dictionary is recoverable whenever the surviving + * frames define the ids themselves. The next flush's write-ahead persist then re-writes + * those ids (it resumes from {@code pd.size()}), healing the side-file on disk. + *

+ * What still fails clean. A genuine GAP: the ids below a surviving frame's delta + * start were introduced by frames that were acked and TRIMMED away, so they lived only in + * the lost dictionary and nothing can rebuild them. + * {@code addRecoveredSymbolsTo} returns -1 for that and we throw. It is the same + * condition the send loop's replay guard ({@code deltaStart > sentDictCount}) trips on, so + * producer and drainer now agree on exactly which slots are recoverable, instead of the + * producer rejecting slots the drainer drains. + */ + private void seedGlobalDictionaryFromPersisted(PersistedSymbolDict pd) { + if (cursorEngine == null) { + return; + } + // 1. The dictionary's intact prefix. addRecoveredSymbol appends without de-dup, so + // the producer's size tracks pd.size() exactly -- which is what the send loop's + // mirror also seeds sentDictCount from. + int baseline = 0; + if (pd != null && pd.size() > 0) { + pd.addLoadedSymbolsTo(globalSymbolDictionary); + baseline = globalSymbolDictionary.size(); + } + // 2. Everything the surviving frames define above that prefix, straight out of their + // own delta sections -- the same bytes, in the same order, accumulateSentDict will + // feed the mirror as those frames go back on the wire. + long coverage = cursorEngine.addRecoveredSymbolsTo(baseline, globalSymbolDictionary); + if (coverage < 0) { + // A gap: the surviving frames reference ids below their own delta start, + // introduced by frames since acked and trimmed away, and the persisted + // dictionary no longer holds them (a host crash tore its unsynced tail, or it + // could not be opened). That gap only matters for frames that will REPLAY. + // When every recovered committed frame is already acked + // (ackedFsn >= recoveredCommitBoundaryFsn), NOTHING replays: the gap is in + // data the server already has, and the retired orphan-deferred tail above the + // commit boundary is never transmitted. Throwing here would raise a false + // "resend required" for delivered data AND -- because such a slot is fully + // drained -- let build()'s connect-path close unlink the (already-delivered) + // bytes the quarantine claims to preserve. So DON'T throw: seed the intact + // prefix only; addRecoveredSymbolsTo adds nothing on a -1 exactly as the + // send loop's mirror does, so the producer baseline and the mirror's + // sentDictCount still agree by construction. The producer resumes above the + // prefix and the fully-drained slot is cleaned up on close. + long ackedFsn = cursorEngine.ackedFsn(); + long commitBoundaryFsn = cursorEngine.recoveredCommitBoundaryFsn(); + if (ackedFsn >= commitBoundaryFsn) { + sentMaxSymbolId = globalSymbolDictionary.size() - 1; + LOG.info("recovered store-and-forward slot has a torn/incomplete symbol dictionary, " + + "but every committed frame was already acked so nothing needs replaying; " + + "resuming on the intact prefix without quarantine and without data loss " + + "[recoveredPrefixSize={}, ackedFsn={}, commitBoundaryFsn={}]", + baseline, ackedFsn, commitBoundaryFsn); + return; + } + // Genuine loss: unacked committed frames reference ids nothing still holds. + // Typed, because Sender.build() sets such a slot aside instead of failing: this + // is the point at which every source of truth has been tried and none of them + // holds the missing ids. See UnreplayableSlotException. + throw new UnreplayableSlotException( + "recovered store-and-forward symbol dictionary is incomplete and cannot be " + + "rebuilt from the surviving frames (likely a host crash tore its unsynced " + + "tail): the frames reference symbol ids below their own delta start, which " + + "were introduced by frames since acked and trimmed away, so nothing still " + + "holds them; the recovered dictionary holds only " + + (pd == null ? 0 : pd.size()) + " id(s) -- resend the affected data"); + } + // Producer baseline == the coverage the replay will establish == the mirror's + // sentDictCount once those frames have gone out. The first new frame therefore + // starts its delta exactly at the tip, and the replay guard passes. + sentMaxSymbolId = globalSymbolDictionary.size() - 1; + } + + /** + * The symbol id below which the server already holds every dictionary entry, + * used as {@code confirmedMaxId} when encoding a frame. In delta mode this is + * the producer's monotonic sent watermark; in full-dict mode it is -1 so every + * frame re-ships the dictionary from id 0. + */ + private int symbolDeltaBaseline() { + return deltaDictEnabled ? sentMaxSymbolId : -1; + } + private void rollbackRow() { if (currentTableBuffer != null) { currentTableBuffer.cancelCurrentRow(); @@ -3656,7 +4126,7 @@ private void sealAndSwapBuffer() { // back to it; flushPendingRows aborts its post-enqueue state // updates after this throw, so the source rows stay intact and the // next batch re-emits the same rows along with the full inline - // schema and symbol-dict delta from id 0. + // schema and the symbol-dict delta the batch requires. if (toSend.isSending()) { toSend.markRecycled(); } else if (toSend.isSealed()) { diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/BackgroundDrainer.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/BackgroundDrainer.java index d79080d4..d77d20d9 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/BackgroundDrainer.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/BackgroundDrainer.java @@ -36,6 +36,7 @@ import org.slf4j.LoggerFactory; import java.util.concurrent.ThreadLocalRandom; +import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.LockSupport; /** @@ -44,7 +45,9 @@ *

* Lifecycle: *

    - *
  1. Acquire the slot's {@code .lock}; skip silently on contention.
    2. + *
  2. Acquire the parent-anchored logical-slot lock and revalidate the + * scanner snapshot; skip silently on contention or a stale snapshot.
    3. + *
  3. Acquire the slot's {@code .lock}, then release the logical lock.
    4. *
  4. Open a {@link CursorSendEngine} on the slot — recovery picks up * every {@code .sfa} file already on disk.
    5. *
  5. Open a fresh {@link WebSocketClient} via the supplied factory @@ -75,12 +78,10 @@ public final class BackgroundDrainer implements Runnable { * wall-clock budget {@code reconnectMaxDurationMillis} also caps this * capability-gap loop; whichever is hit first triggers escalation. Both * halves of the budget measure a capability-gap episode: the - * wall clock accumulates only across uninterrupted gap-to-gap intervals - * (never before the first gap is observed, and never across an - * intervening transport window -- an unreachable cluster is not - * "failing to settle"), and an intervening role reject restarts the - * episode -- it proves the topology changed, so the next capability-gap - * error is a fresh episode against a newly promoted node. 16 + * wall clock accumulates only across uninterrupted gap-to-gap intervals. + * Any intervening transport or role state restarts the episode: capability + * gaps separated by an unrelated transient are not consecutive evidence + * of a persistent cluster capability mismatch. 16 * attempts gives the cluster room to settle through a rolling upgrade * (each attempt walks every endpoint internally) without letting a genuine * cluster-wide misconfig hang the drainer forever. @@ -132,6 +133,11 @@ public final class BackgroundDrainer implements Runnable { // Minimum wall-clock dwell before poison escalation, forwarded to every // drain loop; mirrors the owner sender's poison_min_escalation_window_millis. private final long poisonMinEscalationWindowMillis; + // Minimum wall-clock dwell a symbol-dict catch-up cap gap must persist before this + // orphan drainer's send loop latches a terminal. Foreground senders retry forever; + // this bounded policy is permitted only so a persistent orphan slot can be + // quarantined for operator intervention. + private final long catchUpCapGapMinEscalationWindowMillis; public BackgroundDrainer( String slotPath, @@ -149,7 +155,33 @@ public BackgroundDrainer( reconnectMaxBackoffMillis, requestDurableAck, durableAckKeepaliveIntervalMillis, CursorWebSocketSendLoop.DEFAULT_MAX_HEAD_FRAME_REJECTIONS, - CursorWebSocketSendLoop.DEFAULT_POISON_MIN_ESCALATION_WINDOW_MILLIS); + CursorWebSocketSendLoop.DEFAULT_POISON_MIN_ESCALATION_WINDOW_MILLIS, + CursorWebSocketSendLoop.DEFAULT_CATCHUP_CAP_GAP_MIN_ESCALATION_WINDOW_MILLIS); + } + + /** + * Compatibility constructor retained for callers compiled before the + * catch-up cap-gap dwell was added to the master signature. + */ + public BackgroundDrainer( + String slotPath, + long segmentSizeBytes, + long sfMaxTotalBytes, + CursorWebSocketSendLoop.ReconnectFactory clientFactory, + long reconnectMaxDurationMillis, + long reconnectInitialBackoffMillis, + long reconnectMaxBackoffMillis, + boolean requestDurableAck, + long durableAckKeepaliveIntervalMillis, + int maxHeadFrameRejections, + long poisonMinEscalationWindowMillis + ) { + this(slotPath, segmentSizeBytes, sfMaxTotalBytes, clientFactory, + reconnectMaxDurationMillis, reconnectInitialBackoffMillis, + reconnectMaxBackoffMillis, requestDurableAck, + durableAckKeepaliveIntervalMillis, maxHeadFrameRejections, + poisonMinEscalationWindowMillis, + CursorWebSocketSendLoop.DEFAULT_CATCHUP_CAP_GAP_MIN_ESCALATION_WINDOW_MILLIS); } /** @@ -169,7 +201,8 @@ public BackgroundDrainer( boolean requestDurableAck, long durableAckKeepaliveIntervalMillis, int maxHeadFrameRejections, - long poisonMinEscalationWindowMillis + long poisonMinEscalationWindowMillis, + long catchUpCapGapMinEscalationWindowMillis ) { this.slotPath = slotPath; this.segmentSizeBytes = segmentSizeBytes; @@ -182,6 +215,7 @@ public BackgroundDrainer( this.durableAckKeepaliveIntervalMillis = durableAckKeepaliveIntervalMillis; this.maxHeadFrameRejections = maxHeadFrameRejections; this.poisonMinEscalationWindowMillis = poisonMinEscalationWindowMillis; + this.catchUpCapGapMinEscalationWindowMillis = catchUpCapGapMinEscalationWindowMillis; } /** @@ -194,7 +228,7 @@ public BackgroundDrainer( @TestOnly public BackgroundDrainer() { this(null, 0L, 0L, null, 0L, 0L, 0L, false, 0L, - CursorWebSocketSendLoop.DEFAULT_MAX_HEAD_FRAME_REJECTIONS, 0L); + CursorWebSocketSendLoop.DEFAULT_MAX_HEAD_FRAME_REJECTIONS, 0L, 0L); } /** @@ -208,8 +242,9 @@ public BackgroundDrainer() { * durable ack -- i.e. the symptom of a misconfigured cluster or a * rolling-upgrade transient. *

    - * For the foreground sender that condition is loud-fail: the producer - * is actively pushing data. The drainer is asymmetric: source data is + * Blocking foreground startup keeps its fail-fast policy, while asynchronous + * foreground startup and reconnect keep buffering and retrying through a + * rolling capability change. The orphan drainer is asymmetric: source data is * pinned (durable-ack-mode trims only on STATUS_DURABLE_ACK frames, * which the offending endpoints by definition do not send), so we * give the cluster a budget to settle before quarantining the slot. @@ -222,18 +257,14 @@ public BackgroundDrainer() { * {@link QwpDurableAckMismatchException} sweeps only. Transient * conditions -- an all-replica failover window (role reject) or a * transport error -- are retried indefinitely (Invariant B) and never - * consume the budget: the wall-clock half accumulates only across - * uninterrupted gap-to-gap intervals, so a mid-episode transport window - * pauses the clock (without touching the attempt count), and a role - * reject additionally restarts the episode, because it proves the - * topology changed under the rolling upgrade. + * consume the budget. Either transient restarts the attempt count and wall + * clock so only uninterrupted capability-gap sweeps can escalate. * Genuine terminals (auth failure, non-421 upgrade reject) preserve * the original behavior: mark failed, exit. * * @return a fresh durable-ack-capable client, or {@code null} if * {@link #outcome} has been set to FAILED or STOPPED */ - @TestOnly public WebSocketClient connectWithDurableAckRetry() { // run() already set runnerThread; setting it again here is a no-op // on that path but wires up direct @TestOnly calls so requestStop() @@ -244,25 +275,29 @@ public WebSocketClient connectWithDurableAckRetry() { // QwpDurableAckMismatchException sweeps; the wall-clock half // accumulates ONLY across uninterrupted gap-to-gap intervals, so // transient churn (role reject, transport) can never burn the budget - // -- neither before the first gap is observed nor mid-episode (a - // cluster unreachable for longer than the whole budget that comes - // back still gapped has consumed none of it). An intervening role - // reject resets the episode (topology churn: the offending node is - // gone); a transport error neither increments nor resets the attempt - // count -- a dropped socket does not prove promotion churn, and - // resetting on it would let a flaky-but-misconfigured cluster evade - // the cap forever -- it only pauses the wall clock: the gap-to-gap - // interval spanning the transport window is not charged. + // -- neither before the first gap is observed nor mid-episode. Any + // intervening role or transport state resets the episode: after the + // cluster leaves the capability-gap state, later gaps must establish a + // fresh consecutive run before quarantine is permitted. int capabilityGapAttempts = 0; // Wall-clock time accumulated across uninterrupted gap-to-gap // intervals of the current episode; escalates once it reaches // capabilityGapBudgetNanos (or the attempt cap fires first). long capabilityGapElapsedNanos = 0L; // Timestamp of the previous capability-gap sweep; 0 = the next gap - // charges nothing (episode start, post-role-reject restart, or the - // interval was interrupted by a transport window). + // charges nothing because a fresh episode is starting. long lastCapabilityGapNanos = 0L; - final long capabilityGapBudgetNanos = reconnectMaxDurationMillis * 1_000_000L; + // Saturate rather than multiply: reconnect_max_duration_millis is validated only + // as > 0, so a large value (Long.MAX_VALUE is the natural way to ask for "never + // give up") wraps a raw multiply NEGATIVE. capabilityGapElapsedNanos then clears + // the budget on the FIRST capability-gap sweep -- 0 >= a negative -- and, because + // this gate is an OR with the attempt cap, nothing else holds it back: the slot + // quarantines immediately, skipping the whole 16-sweep settle budget. Asking for + // more tolerance would buy exactly none. TimeUnit clamps at Long.MAX_VALUE, which + // is the intended "effectively unbounded". CursorWebSocketSendLoop's dwell + // conversion guards the same way for the same reason. + final long capabilityGapBudgetNanos = + TimeUnit.MILLISECONDS.toNanos(reconnectMaxDurationMillis); // Observability-only counter for the transient all-replica window; // never consulted for escalation (Invariant B). int roleRejectAttempts = 0; @@ -283,8 +318,7 @@ public WebSocketClient connectWithDurableAckRetry() { } catch (QwpAuthFailedException | WebSocketUpgradeException e) { // Genuinely non-retriable across the cluster (auth 401/403, or a // non-421 upgrade reject): waiting will not fix it, so quarantine - // immediately -- exactly as the live sender's background loop - // (CursorWebSocketSendLoop.connectLoop) halts on these errors. + // immediately under the orphan reconnect policy. String msg = e.getMessage(); LOG.error("drainer terminal upgrade/auth error for slot {}: {}", slotPath, msg); lastErrorMessage = msg; @@ -397,10 +431,11 @@ public WebSocketClient connectWithDurableAckRetry() { // Invariant B -- but it is NOT a transport outage, so log it // truthfully below rather than mislabelling it "cluster unreachable". lastErrorMessage = t.getMessage(); - // Pause the episode wall clock: the gap-to-gap interval this - // window interrupts is never charged. Attempts and elapsed - // already accumulated are preserved (anti-evasion: see the - // budget comment above). + // This unrelated state breaks the consecutive capability-gap + // run. Restart both halves of the settle budget so a later gap + // must establish a fresh episode before quarantine. + capabilityGapAttempts = 0; + capabilityGapElapsedNanos = 0L; lastCapabilityGapNanos = 0L; long nowWarn = System.nanoTime(); if (nowWarn - lastTransportWarnNanos >= 5_000_000_000L) { @@ -521,27 +556,59 @@ private boolean stopRequestedOrInterrupted() { @Override public void run() { runnerThread = Thread.currentThread(); + SlotLock logicalSlotLock = null; CursorSendEngine engine = null; WebSocketClient client = null; CursorWebSocketSendLoop loop = null; try { - // The engine acquires the slot's .lock itself — we don't need - // (and must not) double-lock it. If another sender or drainer - // holds it, the engine constructor throws and we exit silently - // (no .failed sentinel — contention is expected, not an error). + // Scanner results are only snapshots. Serialize adoption against + // a producer's close -> quarantine rename -> fresh-slot recreate + // transition, then revalidate while that stable parent-anchored + // lock is held. The slot's own .lock inode moves with a rename and + // cannot provide this guarantee by itself. + if (slotPath != null) { + try { + logicalSlotLock = SlotLock.acquireLogical(slotPath); + } catch (IllegalStateException t) { + if (isLockContention(t)) { + LOG.info("orphan logical slot already locked, skipping: {} ({})", + slotPath, t.getMessage()); + outcome = DrainOutcome.LOCKED_BY_OTHER; + return; + } + throw t; + } + if (!OrphanScanner.isCandidateOrphan(slotPath)) { + LOG.info("orphan candidate changed before adoption, skipping: {}", slotPath); + outcome = DrainOutcome.SUCCESS; + return; + } + } + + // The engine acquires the directory-local .lock itself. Keep the + // lock order logical -> local, and release the short-lived logical + // lock only after the engine has secured stable ownership. try { engine = new CursorSendEngine(slotPath, segmentSizeBytes, sfMaxTotalBytes, CursorSendEngine.DEFAULT_APPEND_DEADLINE_NANOS); } catch (IllegalStateException t) { - String msg = t.getMessage(); - if (msg != null && msg.contains("already in use")) { + if (isLockContention(t)) { LOG.info("orphan slot already locked, skipping: {} ({})", - slotPath, msg); + slotPath, t.getMessage()); outcome = DrainOutcome.LOCKED_BY_OTHER; return; } throw t; } + if (logicalSlotLock != null) { + logicalSlotLock.close(); + logicalSlotLock = null; + } + // A recovered deferred-only tail is an aborted transaction and can + // be retired locally once everything below it is already ACKed. + // Do this before opening a socket: auth/upgrade failures must not + // quarantine a slot that has no wire-visible work left. + engine.retireRecoveredOrphanTailIfReady(); long target = engine.publishedFsn(); if (engine.ackedFsn() >= target) { LOG.info("orphan slot already drained: {} (acked={} target={})", @@ -575,7 +642,9 @@ public void run() { requestDurableAck, durableAckKeepaliveIntervalMillis, maxHeadFrameRejections, - poisonMinEscalationWindowMillis); + poisonMinEscalationWindowMillis, + catchUpCapGapMinEscalationWindowMillis, + CursorWebSocketSendLoop.ReconnectPolicy.ORPHAN); loop.start(); while (!stopRequestedOrInterrupted()) { @@ -714,12 +783,20 @@ public void run() { + "slot lock releases when it exits", slotPath); } } + if (logicalSlotLock != null) { + logicalSlotLock.close(); + } // Don't let a later requestStop() unpark an unrelated task that // the pool's executor may have scheduled onto this same thread. runnerThread = null; } } + private static boolean isLockContention(IllegalStateException error) { + String message = error.getMessage(); + return message != null && message.contains("already in use"); + } + /** * Plug an observer for durable-ack-related events. {@code null} clears * any previously installed listener. See {@link BackgroundDrainerListener} diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java index 64ca75d0..fcfb2a45 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorSendEngine.java @@ -24,10 +24,13 @@ package io.questdb.client.cutlass.qwp.client.sf.cursor; +import io.questdb.client.cutlass.qwp.client.GlobalSymbolDictionary; import io.questdb.client.std.Compat; import io.questdb.client.std.Files; +import io.questdb.client.std.FilesFacade; import io.questdb.client.std.ObjList; import io.questdb.client.std.QuietCloseable; +import org.jetbrains.annotations.TestOnly; import java.util.concurrent.locks.LockSupport; @@ -84,9 +87,9 @@ public final class CursorSendEngine implements QuietCloseable { private final SlotLock slotLock; // True when the constructor recovered an existing on-disk slot rather // than starting fresh. Diagnostic accessor for tests and observability; - // cursor frames are self-sufficient (every frame carries full schema + - // full symbol-dict delta), so producer-side schema reset on recovery - // is not required. + // every frame carries its full inline schema, so producer-side schema reset + // on recovery is not required (the symbol dictionary, which delta frames do + // NOT carry in full, is re-registered by an I/O-thread catch-up instead). private final boolean wasRecoveredFromDisk; // FSN of the last commit-bearing (non-FLAG_DEFER_COMMIT) frame found in a // ring recovered from disk, or -1 for fresh/memory rings and recovered @@ -96,6 +99,23 @@ public final class CursorSendEngine implements QuietCloseable { // covers them. Read by the sender's close-time drain to avoid waiting on // acks that cannot arrive. private long recoveredCommitBoundaryFsn = -1L; + // Highest symbol id any recovered delta frame references, or -1 for + // fresh/memory rings (and recovered rings with no symbol-bearing frame). A + // resuming producer seeds its dictionary baseline from the persisted + // .symbol-dict; if that dictionary was torn below this id by a host crash + // (the side-file is not fsync'd), the producer would re-use ids the surviving + // frames already define. seedGlobalDictionaryFromPersisted compares this + // against the recovered dictionary size to fail clean instead. Computed once + // in the constructor's recovery branch; -1 elsewhere. + private long recoveredMaxSymbolId = -1L; + // Highest deltaStart across the recovered COMMITTED frames; 0 when none carries a symbol + // dictionary. ZERO means every surviving frame is SELF-SUFFICIENT -- it re-registers its + // dictionary from id 0 -- so the slot replays with no dictionary at all and the send loop + // needs no catch-up. ABOVE zero means at least one frame is a true delta whose ids depend + // on registrations it does not itself carry, so the loop must seed its mirror (and ship a + // catch-up) before replaying. Both the full-dict-fallback discard below and the send + // loop's mirror seeding key off this. + private long recoveredMaxSymbolDeltaStart; // FSN of the last frame of a recovered orphaned deferred tail, or -1 when // the recovered ring has no such tail. When >= 0, frames // [recoveredCommitBoundaryFsn + 1 .. recoveredOrphanTipFsn] all carry @@ -110,6 +130,17 @@ public final class CursorSendEngine implements QuietCloseable { // in the constructor, closed by {@link #close()}. The segment manager // writes through this on every tick where ackedFsn has advanced. private final AckWatermark watermark; + // Engine-owned per-slot symbol dictionary file (disk mode only; {@code null} + // in memory mode and if open() failed). Enables delta-encoded SF frames: + // recovery / orphan-drain load it to re-register the dictionary on the fresh + // server before replaying non-self-sufficient frames. Opened in the + // constructor, closed by {@link #close()}. When null in disk mode the engine + // reports delta encoding as unavailable and the sender keeps full-dict frames. + private final PersistedSymbolDict persistedSymbolDict; + // Engine-owned output of the single ordered recovery walk. It is retained + // because both producer seeding and every recycled send loop need the same + // frame-rebuilt symbol suffix. Null for fresh and memory-only engines. + private final RecoveredFrameAnalysis recoveredFrameAnalysis; // close() is publicly callable from any thread (Sender.close from a user // thread, JVM shutdown hooks, test cleanup). volatile + synchronized // close() makes the check-and-set atomic and gives readers a fence. @@ -137,9 +168,29 @@ public CursorSendEngine(String sfDir, long segmentSizeBytes) { */ public CursorSendEngine(String sfDir, long segmentSizeBytes, long maxTotalBytes, long appendDeadlineNanos) { + this(sfDir, segmentSizeBytes, maxTotalBytes, appendDeadlineNanos, FilesFacade.INSTANCE); + } + + /** + * As {@link #CursorSendEngine(String, long, long, long)}, but with an explicit + * {@link FilesFacade} for the persisted symbol dictionary. + *

    + * The seam exists so a test can drive a dictionary I/O fault -- a short write from + * a full disk or an exhausted quota -- through the REAL producer path + * ({@code flush()} -> the write-ahead persist), and assert it surfaces as a + * {@code LineSenderException} like every other flush-path failure rather than as a + * raw {@code IllegalStateException} that would sail past every caller's + * {@code catch (LineSenderException)}. Nothing else could reach that translation: + * {@code PersistedSymbolDict} has facade-aware overloads, but the engine used to + * call only the {@code FilesFacade.INSTANCE} ones, so no fault could be injected + * from outside. + */ + @TestOnly + public CursorSendEngine(String sfDir, long segmentSizeBytes, + long maxTotalBytes, long appendDeadlineNanos, FilesFacade dictFf) { this(sfDir, segmentSizeBytes, new SegmentManager(segmentSizeBytes, SegmentManager.DEFAULT_POLL_NANOS, maxTotalBytes), - true, appendDeadlineNanos); + true, appendDeadlineNanos, dictFf); } /** @@ -148,11 +199,12 @@ public CursorSendEngine(String sfDir, long segmentSizeBytes, * ownership of the manager. Uses the default append deadline. */ public CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager manager) { - this(sfDir, segmentSizeBytes, manager, false, DEFAULT_APPEND_DEADLINE_NANOS); + this(sfDir, segmentSizeBytes, manager, false, DEFAULT_APPEND_DEADLINE_NANOS, + FilesFacade.INSTANCE); } private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager manager, - boolean ownsManager, long appendDeadlineNanos) { + boolean ownsManager, long appendDeadlineNanos, FilesFacade dictFf) { // sfDir == null → memory-only mode (non-SF async ingest). Same // cursor architecture, no disk involvement; segments // live in malloc'd native memory. @@ -199,7 +251,17 @@ private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager man // reference instead of orphaning the mmap'd segments + fds. SegmentRing ringInProgress = null; AckWatermark watermarkInProgress = null; + PersistedSymbolDict persistedDictInProgress = null; + RecoveredFrameAnalysis recoveredFrameAnalysisInProgress = null; try { + // v2 segment payloads may depend on a persisted symbol-dictionary + // prefix. Install the rollback barrier before recovery or any new + // append so a v1-only client can never skip the v2 files, treat the + // slot as empty, and silently restart at FSN 0. Current recovery + // recognizes and skips the reserved guard filenames. + if (!memoryMode) { + SegmentRing.installLegacyReaderBarrier(sfDir); + } // Disk mode: try to recover any *.sfa files left behind by a prior // session before deciding to start fresh. Without this the engine // would create a new sf-initial.sfa at baseSeq=0, overlapping FSNs @@ -257,6 +319,10 @@ private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager man // mmap doesn't take down the engine -- we just fall // back to the bare lowestBase - 1 seed. watermarkInProgress = AckWatermark.open(sfDir); + // Load the persisted symbol dictionary so delta-encoded frames + // in this recovered slot can be re-registered on the fresh + // server before replay. Null on open failure -> delta disabled. + persistedDictInProgress = PersistedSymbolDict.open(dictFf, sfDir); long baseSeed = lowestBase - 1; long watermarkFsn = watermarkInProgress != null ? watermarkInProgress.read() @@ -276,6 +342,12 @@ private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager man if (seed >= 0) { recovered.acknowledge(seed); } + // Fold the whole recovered ring once. The result checkpoints all + // running metadata and raw symbol bytes at each commit-bearing + // frame, so its final snapshot excludes an orphan deferred tail + // without requiring a second bounded scan. + recoveredFrameAnalysisInProgress = recovered.analyzeRecovery( + persistedDictInProgress == null ? 0 : persistedDictInProgress.size()); // Locate the last commit-bearing frame below a potentially // orphaned FLAG_DEFER_COMMIT tail. A producer that crashed (or // closed) mid-transaction leaves deferred frames with no @@ -285,12 +357,7 @@ private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager man // drainOnClose), and (b) replaying them into a NEW session's // commit would resurrect half a transaction -- see the WARN // below. Computed before the I/O loop or producer append. - this.recoveredCommitBoundaryFsn = recovered.findLastFsnWithoutPayloadFlag( - io.questdb.client.cutlass.qwp.protocol.QwpConstants.HEADER_OFFSET_FLAGS, - io.questdb.client.cutlass.qwp.protocol.QwpConstants.FLAG_DEFER_COMMIT, - io.questdb.client.cutlass.qwp.protocol.QwpConstants.MAGIC_MESSAGE, - io.questdb.client.cutlass.qwp.protocol.QwpConstants.HEADER_SIZE - ); + this.recoveredCommitBoundaryFsn = recoveredFrameAnalysisInProgress.commitBoundaryFsn(); if (publishedFsn >= 0 && recoveredCommitBoundaryFsn < publishedFsn) { this.recoveredOrphanTipFsn = publishedFsn; LOG.warn("recovered SF log ends with {} deferred frame(s) whose transaction was never " @@ -300,6 +367,73 @@ private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager man publishedFsn - Math.max(recoveredCommitBoundaryFsn, -1L), recoveredCommitBoundaryFsn, publishedFsn); } + // Highest symbol id the surviving COMMITTED frames reference. A + // resuming producer compares this against its recovered dictionary + // size (seedGlobalDictionaryFromPersisted) to detect a host-crash + // tear: if a committed frame references an id the (unsynced, torn) + // .symbol-dict no longer holds, resuming would re-use it. The walk is + // bounded to recoveredCommitBoundaryFsn so the aborted orphan-deferred + // tail -- retired without ever being transmitted -- does not inflate + // this and over-reject an otherwise-recoverable slot. maxDeltaEnd() + // returns 0 when no such frame carries a symbol, yielding -1 here. + // Computed before the I/O loop or producer append; single-threaded. + this.recoveredMaxSymbolId = recoveredFrameAnalysisInProgress.maxDeltaEnd() - 1L; + // Full-dict-fallback recovery. When the persisted .symbol-dict is a + // SUBSET of the ids the surviving frames reference + // (recoveredMaxSymbolId >= its size) YET every such frame is + // self-sufficient (maxDeltaStart() == 0 -- a full-dict frame that + // re-registers its dictionary from id 0), the slot was written in + // full-dict fallback: the dictionary never opened when writing, so no + // side-file exists and this recovery opened a FRESH EMPTY one. Those + // frames replay with no dictionary, so discard the empty side-file and + // recover in full-dict mode -- isDeltaDictEnabled() then reports false + // and the producer + send loop both run full-dict, exactly as the slot + // was written. Without this the sender's seed-time guard would treat the + // empty dictionary as a host-crash tear and brick build(), even though + // the orphan drainer drains the same frames fine. A genuine torn DELTA + // dictionary keeps a frame with deltaStart > 0 (maxDeltaStart() > 0) + // and is NOT discarded here: it still fails clean at seed time, since + // the ids its delta frames reference cannot be rebuilt without the lost + // dictionary. The recoveredMaxSymbolId >= size guard means this never + // fires for a slot whose dictionary is intact, nor for an empty slot + // (recoveredMaxSymbolId == -1). Single-threaded; before the I/O loop. + this.recoveredMaxSymbolDeltaStart = recoveredFrameAnalysisInProgress.maxDeltaStart(); + if (persistedDictInProgress != null + && recoveredMaxSymbolId >= persistedDictInProgress.size() + && recoveredMaxSymbolDeltaStart == 0L) { + persistedDictInProgress.close(); + persistedDictInProgress = null; + // Re-fold at baseline 0. The analysis above was keyed to the + // dictionary's size, and every consumer of it presents the baseline it + // derived the same way -- seedGlobalDictionaryFromPersisted computes + // baseline 0 once pd is gone, and checkedRecoveryAnalysis rejects a + // baseline that disagrees with the fold. Discarding a dictionary that + // held entries (size > 0) therefore desynchronised the two and threw + // IllegalStateException("recovery symbol baseline mismatch") out of + // build(). That is NOT an UnreplayableSlotException, so build()'s + // quarantine handler could not catch it and set the slot aside: with a + // stable senderId every restart re-recovered the same slot and threw + // again, so the application could never construct a Sender -- it could + // not even BUFFER new rows. Exactly the outage quarantineTornSlot + // exists to prevent, on a slot that is fully recoverable (its frames + // carry their whole dictionary inline). + // + // Reachable on one transient plus one crash: a session whose + // .symbol-dict fails to open (EIO, fd exhaustion, a Windows share + // lock) falls back to full-dict frames and, per the never-recreate + // contract, leaves the previous session's populated side-file intact; + // if that session then crashes, this recovery opens a dictionary with + // size > 0 next to self-sufficient frames that out-reach it. + // + // Re-folding rather than keeping the dictionary preserves the discard's + // whole point -- the slot recovers in full-dict mode, exactly as it was + // written, with producer, mirror and replay guard all anchored at 0. + recoveredFrameAnalysisInProgress.close(); + recoveredFrameAnalysisInProgress = recovered.analyzeRecovery(0); + this.recoveredCommitBoundaryFsn = recoveredFrameAnalysisInProgress.commitBoundaryFsn(); + this.recoveredMaxSymbolId = recoveredFrameAnalysisInProgress.maxDeltaEnd() - 1L; + this.recoveredMaxSymbolDeltaStart = recoveredFrameAnalysisInProgress.maxDeltaStart(); + } } else { // Fresh start with no recovered segments. Any stale // watermark from a prior fully-drained session refers @@ -309,6 +443,19 @@ private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager man if (!memoryMode) { AckWatermark.removeOrphan(sfDir); watermarkInProgress = AckWatermark.open(sfDir); + // A fresh slot MUST start with an EMPTY symbol dictionary. + // Unlike the ack watermark above -- a discardable optimization a + // max() clamp protects -- the dictionary is load-bearing: a + // delta frame referencing an id missing from it is unrecoverable, + // and a STALE dictionary inherited here (the segments are gone, so + // the producer is NOT seeded from it) shifts the dense id->symbol + // mapping and silently misattributes symbols on the next + // reconnect. openClean() truncates any survivor to empty rather + // than trusting a best-effort delete that may have failed (e.g. a + // Windows share lock); if the clean open itself fails, + // persistedSymbolDict stays null and the sender falls back to full + // self-sufficient frames, which is also safe. + persistedDictInProgress = PersistedSymbolDict.openClean(dictFf, sfDir); } MmapSegment initial; String initialPath = null; @@ -333,10 +480,12 @@ private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager man manager.start(); } manager.register(ringInProgress, sfDir, watermarkInProgress); - // All construction succeeded — commit the ring and - // watermark references. + // All construction succeeded — commit the ring, watermark and + // symbol-dictionary references. this.ring = ringInProgress; this.watermark = watermarkInProgress; + this.persistedSymbolDict = persistedDictInProgress; + this.recoveredFrameAnalysis = recoveredFrameAnalysisInProgress; } catch (Throwable t) { // Stop an owned manager before freeing the ring and watermark it may // touch, then release the slot lock. Each cleanup is in its own @@ -362,6 +511,18 @@ private CursorSendEngine(String sfDir, long segmentSizeBytes, SegmentManager man } catch (Throwable ignored) { } } + if (persistedDictInProgress != null) { + try { + persistedDictInProgress.close(); + } catch (Throwable ignored) { + } + } + if (recoveredFrameAnalysisInProgress != null) { + try { + recoveredFrameAnalysisInProgress.close(); + } catch (Throwable ignored) { + } + } if (acquiredLock != null) { try { acquiredLock.close(); @@ -541,6 +702,18 @@ public synchronized void close() { } catch (Throwable ignored) { } } + if (persistedSymbolDict != null) { + try { + persistedSymbolDict.close(); + } catch (Throwable ignored) { + } + } + if (recoveredFrameAnalysis != null) { + try { + recoveredFrameAnalysis.close(); + } catch (Throwable ignored) { + } + } if (fullyDrained) { try { unlinkAllSegmentFiles(sfDir); @@ -550,6 +723,11 @@ public synchronized void close() { AckWatermark.removeOrphan(sfDir); } catch (Throwable ignored) { } + try { + // Slot fully drained: the dictionary has no frames behind it. + PersistedSymbolDict.removeOrphan(sfDir); + } catch (Throwable ignored) { + } } } finally { if (slotLock != null) { @@ -562,6 +740,74 @@ public synchronized void close() { } } + /** + * Decodes the cached recovery suffix directly into the producer's global + * dictionary. Recovery always builds the analysis with the persisted + * prefix size as its baseline, so no intermediate cardinality-sized list is + * needed on the production path. + */ + public long addRecoveredSymbolsTo(int baseline, GlobalSymbolDictionary target) { + if (recoveredFrameAnalysis == null) { + return baseline; + } + RecoveredFrameAnalysis analysis = checkedRecoveryAnalysis(baseline); + long coverage = analysis.coverage(); + if (coverage >= 0L) { + analysis.addDecodedSymbolsTo(target); + } + return coverage; + } + + long recoveredSymbolCoverage(int baseline) { + return checkedRecoveryAnalysis(baseline).coverage(); + } + + int recoveredSymbolSuffixCount(int baseline) { + return checkedRecoveryAnalysis(baseline).rawCount(); + } + + int recoveredSymbolSuffixLen(int baseline) { + return checkedRecoveryAnalysis(baseline).rawLen(); + } + + void copyRecoveredSymbolSuffix(int baseline, long target) { + RecoveredFrameAnalysis analysis = checkedRecoveryAnalysis(baseline); + int len = analysis.rawLen(); + if (len > 0) { + io.questdb.client.std.Unsafe.getUnsafe().copyMemory(analysis.rawAddr(), target, len); + } + } + + void releaseRecoveredSymbolStorage() { + if (recoveredFrameAnalysis != null) { + recoveredFrameAnalysis.releaseRawStorage(); + } + } + + @TestOnly + public long recoveryFramesVisited() { + return recoveredFrameAnalysis == null ? 0L : recoveredFrameAnalysis.framesVisited(); + } + + @TestOnly + public long recoverySymbolEntriesVisited() { + return recoveredFrameAnalysis == null ? 0L : recoveredFrameAnalysis.symbolEntriesVisited(); + } + + @TestOnly + public int recoverySymbolNativeCapacity() { + return recoveredFrameAnalysis == null ? 0 : recoveredFrameAnalysis.rawCapacity(); + } + + private RecoveredFrameAnalysis checkedRecoveryAnalysis(int baseline) { + if (recoveredFrameAnalysis == null || recoveredFrameAnalysis.baseline() != baseline) { + throw new IllegalStateException("recovery symbol baseline mismatch [expected=" + + (recoveredFrameAnalysis == null ? "none" : recoveredFrameAnalysis.baseline()) + + ", actual=" + baseline + ']'); + } + return recoveredFrameAnalysis; + } + /** * Pass-through to {@link SegmentRing#findSegmentContaining(long)}. */ @@ -576,6 +822,15 @@ public MmapSegment firstSealed() { return ring.firstSealed(); } + /** + * The engine's persisted symbol dictionary, or {@code null} in memory mode + * (and in disk mode if it failed to open). The producer appends new symbols + * to it; recovery / orphan-drain read its loaded entries to seed catch-up. + */ + public PersistedSymbolDict getPersistedSymbolDict() { + return persistedSymbolDict; + } + /** * Number of times {@link #appendBlocking} hit * {@link SegmentRing#BACKPRESSURE_NO_SPARE} on its first attempt and @@ -586,6 +841,18 @@ public long getTotalBackpressureStalls() { return backpressureStallCount.get(); } + /** + * Whether the sender may delta-encode symbol dictionaries on this engine. + * Always true in memory mode (the send loop keeps an in-process catch-up + * mirror). In disk mode it requires the persisted dictionary to have opened, + * since delta frames are not self-sufficient and recovery / orphan-drain must + * be able to rebuild the dictionary from disk. When false in disk mode the + * sender falls back to full self-sufficient frames. + */ + public boolean isDeltaDictEnabled() { + return sfDir == null || persistedSymbolDict != null; + } + /** * Pass-through to {@link SegmentRing#nextSealedAfter(MmapSegment)}. */ @@ -650,6 +917,30 @@ public long recoveredCommitBoundaryFsn() { return recoveredCommitBoundaryFsn; } + /** + * Highest symbol id any recovered delta frame references, or {@code -1} for + * fresh/memory rings (and recovered rings with no symbol-bearing frame). A + * resuming producer compares this against its recovered dictionary size to + * detect a host-crash tear of the persisted {@code .symbol-dict}. + */ + public long recoveredMaxSymbolId() { + return recoveredMaxSymbolId; + } + + /** + * Highest {@code deltaStart} across the recovered committed frames; {@code 0} when every + * surviving frame is self-sufficient (or none carries a dictionary at all). + *

    + * The send loop uses this to decide whether it needs a catch-up: at zero, every frame + * re-registers its dictionary from id 0 as it replays, so seeding the mirror -- and + * shipping a catch-up frame off it -- would be pure redundancy. Above zero, at least one + * frame's delta starts above ids it does not itself carry, so the mirror must hold those + * ids before the replay begins or the server null-pads the hole. + */ + public long recoveredMaxSymbolDeltaStart() { + return recoveredMaxSymbolDeltaStart; + } + /** * FSN of the last frame of a recovered orphaned deferred tail, or * {@code -1} when none. See {@link #recoveredCommitBoundaryFsn()}: the @@ -659,6 +950,31 @@ public long recoveredOrphanTipFsn() { return recoveredOrphanTipFsn; } + /** + * Retires a recovered deferred tail once every frame below it is ACKed. + * The operation is local and idempotent: no wire sequence ever referred + * to these aborted-transaction frames. + * + * @return true if no orphan tail remains, false if lower frames still need + * server ACKs + */ + public boolean retireRecoveredOrphanTailIfReady() { + long orphanTip = recoveredOrphanTipFsn; + if (orphanTip < 0L) { + return true; + } + long orphanStart = recoveredCommitBoundaryFsn + 1L; + if (ackedFsn() < orphanStart - 1L) { + return false; + } + LOG.warn("retiring orphaned deferred tail: {} frame(s) [fsn {}..{}] belong to a transaction " + + "whose commit was never published; aborting them (never transmitted, slots trimmed)", + orphanTip - orphanStart + 1L, orphanStart, orphanTip); + acknowledge(orphanTip); + recoveredOrphanTipFsn = -1L; + return true; + } + /** * Unlinks every {@code .sfa} file under {@code dir}. Called only on * clean shutdown when the ring confirms every published FSN has been diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java index 13a69f77..753003f2 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/CursorWebSocketSendLoop.java @@ -31,14 +31,17 @@ import io.questdb.client.cutlass.http.client.WebSocketFrameHandler; import io.questdb.client.cutlass.http.client.WebSocketUpgradeException; import io.questdb.client.cutlass.line.LineSenderException; +import io.questdb.client.cutlass.qwp.client.NativeBufferWriter; import io.questdb.client.cutlass.qwp.client.QwpAuthFailedException; import io.questdb.client.cutlass.qwp.client.QwpDurableAckMismatchException; import io.questdb.client.cutlass.qwp.client.QwpIngressRoleRejectedException; import io.questdb.client.cutlass.qwp.client.QwpRoleMismatchException; import io.questdb.client.cutlass.qwp.client.QwpVersionMismatchException; import io.questdb.client.cutlass.qwp.client.WebSocketResponse; +import io.questdb.client.cutlass.qwp.protocol.QwpConstants; import io.questdb.client.cutlass.qwp.websocket.WebSocketCloseCode; import io.questdb.client.std.CharSequenceLongHashMap; +import io.questdb.client.std.MemoryTag; import io.questdb.client.std.QuietCloseable; import io.questdb.client.std.Unsafe; import org.jetbrains.annotations.TestOnly; @@ -49,6 +52,7 @@ import java.util.Arrays; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ThreadLocalRandom; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.locks.LockSupport; @@ -65,8 +69,10 @@ *

  6. On wire failure, runs the configured reconnect policy: capped * exponential backoff with jitter, retried indefinitely (Invariant B -- * a store-and-forward drainer never gives up on a wall-clock budget), - * with only auth-style failures (401/403/non-101 upgrade reject) treated - * as terminal. On reconnect success, repositions the cursor at + * with endpoint-policy failures terminal only for an orphan drainer. A + * foreground sender keeps retrying from asynchronous startup onward so + * credential and rolling-capability changes remain contained by + * store-and-forward. On reconnect success, repositions the cursor at * {@code ackedFsn+1} and replays.
    7. *
* No locks on the steady-state path. The producer thread (user) writes @@ -129,11 +135,78 @@ public final class CursorWebSocketSendLoop implements QuietCloseable { * {@code LineSenderBuilder.poisonMinEscalationWindowMillis(long)}. */ public static final long DEFAULT_POISON_MIN_ESCALATION_WINDOW_MILLIS = 5_000L; + /** + * Default minimum wall-clock dwell (millis) a symbol-dict catch-up CAP GAP must + * persist before an orphan drainer may latch a terminal, even once + * {@link #MAX_CATCHUP_CAP_GAP_ATTEMPTS} cap gaps have accrued. Same idea as + * {@link #DEFAULT_POISON_MIN_ESCALATION_WINDOW_MILLIS}, for the same reason: a + * strike count measures "how many times did we look", not "how long has this been + * true", and only the latter distinguishes a permanent cluster capability gap from + * a node that is briefly away. + *

+ * It matters here because the cap gap's trigger IS a failover. With the reconnect + * backoff capped at {@link #DEFAULT_RECONNECT_MAX_BACKOFF_MILLIS} (5 s), 16 cap + * gaps accrue in roughly two minutes -- comfortably less than an ordinary rolling + * restart of the larger-cap node. A count-only budget could therefore quarantine an + * otherwise drainable orphan slot during a routine cluster operation. Requiring the + * dwell as well means the orphan terminal needs BOTH "we have looked many times" AND + * "it has stayed true for a long time". Foreground senders never apply this terminal + * policy: they retry indefinitely until a larger-cap node returns. + *

+ * 5 minutes -- the same figure as {@link #DEFAULT_RECONNECT_MAX_DURATION_MILLIS}, + * this codebase's existing notion of how long a transient outage may plausibly + * last. Configurable per sender via the + * {@code catchup_cap_gap_min_escalation_window_millis} connect-string key or + * {@code LineSenderBuilder.catchUpCapGapMinEscalationWindowMillis(long)}; {@code 0} + * restores count-only escalation for orphan drainers. + */ + public static final long DEFAULT_CATCHUP_CAP_GAP_MIN_ESCALATION_WINDOW_MILLIS = 300_000L; private static final Logger LOG = LoggerFactory.getLogger(CursorWebSocketSendLoop.class); + // Settle budget for the symbol-dict catch-up cap gap: how many cap-gap attempts + // -- catch-ups that reached a fresh server and found a single dictionary entry + // too large for its advertised batch cap -- may occur consecutively before an + // orphan drainer latches a terminal. This is a + // SANCTIONED orphan-only terminal (a genuine cluster batch-size capability gap), + // the connect-time analog of the orphan drainer's durable-ack capability gap + // (DEFAULT_MAX_DURABLE_ACK_MISMATCH_ATTEMPTS). A homogeneous cluster never trips + // it -- an entry that fit its data frame under a cap always fits its bare catch-up + // frame under that same cap -- so it only affects a heterogeneous / rolling-cap + // cluster, where a failover to a smaller-cap node can hit it for an entry an + // earlier node accepted. A foreground sender retries forever. An orphan drainer + // rides out the transient window until a larger-cap node returns; only a persistent + // gap (this many consecutive cap gaps with no successful catch-up or unrelated + // reconnect state in between) latches. + // + // Budget accounting (satisfies "a transient must never burn the terminal + // budget"): catchUpCapGapAttempts increments ONLY inside sendDictCatchUp when a + // node is reached and an entry is oversized. A successful catch-up ends the + // episode, as does any unrelated reconnect state (connect refusal, catch-up send + // failure, upgrade/role rejection): otherwise its downtime would count toward the + // wall-clock dwell even though no cap gap was observed. The cap-gap exception + // itself does NOT reset the episode, so consecutive small-cap nodes still prove a + // persistent cluster capability gap and can exhaust the orphan policy. + private static final int MAX_CATCHUP_CAP_GAP_ATTEMPTS = 16; + // Hard ceiling for the lifetime-monotonic sent-dictionary mirror. The mirror + // fields are int, so it cannot exceed Integer.MAX_VALUE bytes; reaching even + // this needs ~200M+ distinct symbols on a single connection, far past any real + // workload. The guard exists so that pathological growth fails loudly instead + // of overflowing the int capacity math into a heap-corrupting copyMemory. + private static final int MAX_SENT_DICT_BYTES = Integer.MAX_VALUE - 8; + private static final int MAX_SENT_DICT_ENTRIES = MAX_SENT_DICT_BYTES / Integer.BYTES; /** * Throttle "reconnect attempt N failed" WARN logs to one per 5 s. */ private static final long RECONNECT_LOG_THROTTLE_NANOS = 5_000_000_000L; + // Test seam: when true, recovery mirror seeding throws immediately AFTER + // ensureSentDictCapacity has grown (and therefore taken ownership of) the mirror, + // standing in for the copyRecoveredSymbolSuffix-adjacent failure that leaves a + // freshly malloc'd mirror with no owner. It sits after the grow deliberately: the + // constructor's cleanup only frees an OWNED mirror, so a seam before the grow + // would leave nothing to free and the leak guard would pass with the cleanup + // deleted. Production never sets it; volatile only so a test thread's write is + // visible to the loop under test. + @TestOnly + public static volatile boolean forceMirrorSeedFailureForTest; // Pre-converted to nanos for the comparison in sendDurableAckKeepaliveIfDue. // Zero or negative disables the keepalive entirely. private final long durableAckKeepaliveIntervalNanos; @@ -151,8 +224,14 @@ public final class CursorWebSocketSendLoop implements QuietCloseable { // by the server -- holding stale watermarks across the wire boundary // would falsely advance trim before re-confirmation. private final CharSequenceLongHashMap durableTableWatermarks = new CharSequenceLongHashMap(); + // Pre-converted to nanos. Consulted only by the orphan terminal policy. Zero disables + // the dwell entirely (count-only escalation at MAX_CATCHUP_CAP_GAP_ATTEMPTS); the + // user-facing 5-minute default is applied at the config layer. + private final long catchUpCapGapMinEscalationWindowNanos; + private final CatchUpCapGapPolicy catchUpCapGapPolicy; private final CursorSendEngine engine; private final long parkNanos; + private final ReconnectPolicy reconnectPolicy; // FIFO of OK-acked batches awaiting durable-upload confirmation. Used only // when durableAckMode is true. Each entry binds a wireSeq to the per-table // (name, seqTxn) pairs the server reported on the OK frame. The queue is @@ -200,6 +279,87 @@ public final class CursorWebSocketSendLoop implements QuietCloseable { // by category. Includes both retriable and terminal outcomes — i.e. every // server-side rejection observed regardless of how the loop reacted. private final AtomicLong totalServerErrors = new AtomicLong(); + // Delta symbol dictionary catch-up state (see swapClient). + // ALWAYS active -- in memory mode, in disk mode, and (critically) even when the + // per-slot persisted dictionary failed to open. sentDictCount is this loop's model + // of how many ids the CURRENT server has been told about, and the torn-dict guard + // in trySendOne reads it, so it must track the wire in every mode; gating it on + // engine.isDeltaDictEnabled() is what once froze it at 0 and terminal'd a slot + // whose frames replay perfectly from id 0 (see the constructor). On a recovered / + // orphan-drained slot the constructor SEEDS sentDict* from the persisted + // dictionary when there is one; otherwise the mirror starts empty and grows from + // the frames themselves. The loop mirrors, in sentDict*, every symbol it has ever + // sent -- the concatenated [len varint][utf8] bytes in global-id order + // (sentDictBytes*) plus the count (sentDictCount) -- so that on reconnect it can + // re-register the whole dictionary on the fresh server (which discards its + // dictionary on every disconnect) before replaying frames whose deltas start above + // id 0. All of this is touched only by the I/O thread. + // Footprint note: this mirror is a SECOND copy of the dictionary -- the same + // symbols the producer's GlobalSymbolDictionary already holds as Java Strings -- + // kept as native UTF-8 bytes for the reconnect-catch-up capability. So a + // memory-mode connection's steady-state dictionary footprint is ~2x the symbol + // set. It is bounded by distinct-symbol count (not per-row) and never trimmed for + // the connection's lifetime (a reconnect may need the whole dictionary at any + // moment), so it cannot be dropped; it is an intentional cost of the feature. + private long sentDictBytesAddr; + private int sentDictBytesCapacity; + private int sentDictBytesLen; + // False only while an orphan-drainer loop borrows the persisted prefix. + // Any growth first performs a copy-on-write; only owned buffers are freed. + private boolean sentDictBytesOwned; + private int sentDictCount; + // Relative byte end of every [len varint][utf8] entry in sentDictBytesAddr. + // The mirror is parsed once as it is built/recovered; reconnect catch-up then + // reads fixed-width offsets instead of decoding every length varint again. + private long sentDictEntryEndsAddr; + private int sentDictEntryEndsCapacity; + private int sentDictIndexedCount; + // True when replay frames can start above dictionary id zero and therefore + // depend on a catch-up on a fresh connection. Delta-enabled live engines + // always have this dependency. A recovered delta slot whose dictionary + // failed to open is identified by its non-zero recovered delta start. The + // remaining false case is full-dictionary fallback: every frame re-registers + // from id zero, so sending the mirror first would duplicate the dictionary. + private final boolean hasReplayDictionaryDependency; + // Reusable staging buffer for the small QWP catch-up prefix (fixed header plus + // delta range varints). Symbol bytes stay in sentDictBytesAddr and are passed as a + // second WebSocket payload slice, avoiding a full-dictionary staging copy. + private long catchUpFrameAddr; + private int catchUpFrameCapacity; + private int catchUpEntryIndexBuildCount; + private int catchUpFrameGrowthCount; + // Orphan-policy cap-gap attempts -- catch-ups that reached a node and found an entry + // too large for its batch cap -- with no intervening successful catch-up or unrelated + // reconnect state (see MAX_CATCHUP_CAP_GAP_ATTEMPTS for the full budget accounting). + // Foreground retries never increment it. I/O-thread-only. + private int catchUpCapGapAttempts; + // System.nanoTime() of the FIRST cap gap of the current orphan-policy episode. + // Anchors the escalation dwell. Reset together with catchUpCapGapAttempts on a + // successful catch-up or an unrelated reconnect state, so only an uninterrupted run + // of cap-gap observations contributes wall-clock dwell. Anchoring at the FIRST gap + // (not refreshed per attempt) lets consecutive small-cap nodes satisfy the dwell; + // resetting after transport/role states prevents their downtime from doing so. + // I/O-thread-only, like catchUpCapGapAttempts. + // + // -1 marks "no episode open" for a debugger or a heap dump, but it is NOT the test + // for one -- catchUpCapGapAttempts == 0 is (see sendDictCatchUp). A nanoTime instant + // is only meaningful as a difference: its origin is arbitrary and the spec permits + // negative values, so no state may ride on this field's sign. + private long catchUpCapGapFirstNanos = -1L; + // True once a real ring frame (data or commit) has been sent on the CURRENT + // connection, as opposed to only the dictionary catch-up. The catch-up consumes + // wire sequences (nextWireSeq), so nextWireSeq > 0 no longer implies "the head + // frame was sent": onClose's poison-strike gate and handleServerRejection's + // pre-send gate key off THIS instead. Without it, a transient outage AFTER the + // catch-up but BEFORE the first data frame (a flapping LB/middlebox that accepts + // the upgrade + catch-up then closes) would be mistaken for a deterministic + // head-frame rejection and escalate to a PROTOCOL_VIOLATION terminal -- breaking + // the store-and-forward "retry a transient outage forever" contract. Reset per + // connection in setWireBaselineWithCatchUp; set in trySendOne after a successful + // send. + private boolean dataFrameSentThisConnection; + // End position (native address) written by the last readVarintAt() call. + private long varintEnd; private WebSocketClient client; // Optional: when non-null, every server-rejection error (retriable and // terminal alike) is offered to the dispatcher for async delivery to the user's @@ -227,10 +387,8 @@ public final class CursorWebSocketSendLoop implements QuietCloseable { private long fsnAtZero; // Sticky flag: false until the very first time a live client is installed // (either via the constructor in SYNC/OFF mode or via swapClient on a - // successful connect attempt in any mode). Once true, stays true. Used to - // distinguish a "never reached the server" terminal failure (looks like a - // config typo or firewall block) from "lost connection after we were - // up" (looks transient). + // successful connect attempt in any mode). Once true, stays true. This is + // connection-state observability only; reconnect policy is role-based. private volatile boolean hasEverConnected; private volatile Thread ioThread; // Typed marker for a durable-ack CAPABILITY-GAP terminal: set (before the @@ -239,8 +397,8 @@ public final class CursorWebSocketSendLoop implements QuietCloseable { // QwpDurableAckMismatchException. The orphan drainer consults it to route // a mid-drain capability gap into its budgeted settle-retry // (BackgroundDrainer.connectWithDurableAckRetry) instead of quarantining - // the slot on the first sweep; the foreground sender ignores it and keeps - // its spec'd loud-fail (sf-client.md section 8.1). Write-once alongside + // the slot on the first sweep. Foreground reconnects never set this marker; + // they keep retrying after a successful initial connection. Write-once alongside // terminalError: the only writer runs on the I/O thread under the same // first-writer-wins latch. private volatile QwpDurableAckMismatchException capabilityGapTerminal; @@ -370,10 +528,10 @@ public final class CursorWebSocketSendLoop implements QuietCloseable { * {@code client} may be {@code null} only if {@code reconnectFactory} * is non-null — this is the async-initial-connect path: the I/O thread * runs the same retry loop on its first iteration to obtain a live - * client, and a terminal failure (auth/upgrade reject) is delivered - * through the dispatcher rather than thrown to the constructor's - * caller; plain connect failures are retried indefinitely - * (Invariant B: no wall-clock budget give-up). + * client. Every endpoint-policy and transport failure stays inside that + * background retry loop; it is never delivered to the foreground producer + * (Invariant B: no wall-clock budget give-up). Blocking OFF/SYNC startup + * performs its fail-fast policy before constructing this loop. */ public CursorWebSocketSendLoop(WebSocketClient client, CursorSendEngine engine, long fsnAtZero, long parkNanos, @@ -454,13 +612,45 @@ public CursorWebSocketSendLoop(WebSocketClient client, CursorSendEngine engine, } /** - * Master constructor — also accepts the poison-frame detector threshold + * Twelve-arg overload — omits the symbol-dict cap-gap escalation dwell, which + * defaults to {@code 0}. This and the thirteen-arg overload use the foreground-safe + * retry-forever policy; orphan drainers opt into count-and-dwell escalation through + * the policy-aware master constructor. + */ + public CursorWebSocketSendLoop(WebSocketClient client, CursorSendEngine engine, + long fsnAtZero, long parkNanos, + ReconnectFactory reconnectFactory, + long reconnectMaxDurationMillis, + long reconnectInitialBackoffMillis, + long reconnectMaxBackoffMillis, + boolean durableAckMode, + long durableAckKeepaliveIntervalMillis, + int maxHeadFrameRejections, + long poisonMinEscalationWindowMillis) { + this(client, engine, fsnAtZero, parkNanos, reconnectFactory, + reconnectMaxDurationMillis, reconnectInitialBackoffMillis, + reconnectMaxBackoffMillis, durableAckMode, + durableAckKeepaliveIntervalMillis, maxHeadFrameRejections, + poisonMinEscalationWindowMillis, 0L); + } + + /** + * Thirteen-arg overload. Catch-up cap gaps are retried indefinitely, which is the + * required policy for a foreground store-and-forward sender. Orphan drainers use the + * policy-aware overload below to opt into bounded terminal escalation. + *

+ * Also accepts the poison-frame detector threshold * ({@code max_frame_rejections}): consecutive server-active rejections of * the same head-of-line frame, with no ack progress in between, before the - * loop escalates to a typed terminal. Must be {@code >= 1}. The final - * argument is the minimum wall-clock dwell (millis) the suspect must stay - * poisoned before escalation ({@code poison_min_escalation_window_millis}; - * {@code >= 0}, where 0 = legacy immediate escalation at the threshold). + * loop escalates to a typed terminal. Must be {@code >= 1}. Then the minimum + * wall-clock dwell (millis) the suspect must stay poisoned before escalation + * ({@code poison_min_escalation_window_millis}; {@code >= 0}, where 0 = legacy + * immediate escalation at the threshold). + *

+ * The final argument is the analogous dwell for the symbol-dict catch-up cap gap. + * It is retained here for source and binary compatibility but is consulted only when + * the policy-aware overload selects + * {@link CatchUpCapGapPolicy#TERMINAL_AFTER_SETTLE_BUDGET}. */ public CursorWebSocketSendLoop(WebSocketClient client, CursorSendEngine engine, long fsnAtZero, long parkNanos, @@ -471,7 +661,33 @@ public CursorWebSocketSendLoop(WebSocketClient client, CursorSendEngine engine, boolean durableAckMode, long durableAckKeepaliveIntervalMillis, int maxHeadFrameRejections, - long poisonMinEscalationWindowMillis) { + long poisonMinEscalationWindowMillis, + long catchUpCapGapMinEscalationWindowMillis) { + this(client, engine, fsnAtZero, parkNanos, reconnectFactory, + reconnectMaxDurationMillis, reconnectInitialBackoffMillis, + reconnectMaxBackoffMillis, durableAckMode, + durableAckKeepaliveIntervalMillis, maxHeadFrameRejections, + poisonMinEscalationWindowMillis, catchUpCapGapMinEscalationWindowMillis, + CatchUpCapGapPolicy.RETRY_FOREVER); + } + + /** + * Compatibility policy-aware constructor. New production call sites should + * use the {@link ReconnectPolicy} overload below, which names the foreground + * versus orphan ownership distinction directly. + */ + public CursorWebSocketSendLoop(WebSocketClient client, CursorSendEngine engine, + long fsnAtZero, long parkNanos, + ReconnectFactory reconnectFactory, + long reconnectMaxDurationMillis, + long reconnectInitialBackoffMillis, + long reconnectMaxBackoffMillis, + boolean durableAckMode, + long durableAckKeepaliveIntervalMillis, + int maxHeadFrameRejections, + long poisonMinEscalationWindowMillis, + long catchUpCapGapMinEscalationWindowMillis, + CatchUpCapGapPolicy catchUpCapGapPolicy) { if (maxHeadFrameRejections < 1) { throw new IllegalArgumentException( "maxHeadFrameRejections must be >= 1: " + maxHeadFrameRejections); @@ -480,6 +696,24 @@ public CursorWebSocketSendLoop(WebSocketClient client, CursorSendEngine engine, throw new IllegalArgumentException( "poisonMinEscalationWindowMillis must be >= 0: " + poisonMinEscalationWindowMillis); } + if (catchUpCapGapMinEscalationWindowMillis < 0) { + throw new IllegalArgumentException( + "catchUpCapGapMinEscalationWindowMillis must be >= 0: " + + catchUpCapGapMinEscalationWindowMillis); + } + // TimeUnit conversion saturates at Long.MAX_VALUE. A raw multiply + // wraps large valid millisecond values negative, which makes the + // elapsed-time gate appear satisfied as soon as the strike threshold + // is reached and can quarantine a recoverable orphan slot prematurely. + this.catchUpCapGapMinEscalationWindowNanos = + TimeUnit.MILLISECONDS.toNanos(catchUpCapGapMinEscalationWindowMillis); + if (catchUpCapGapPolicy == null) { + throw new IllegalArgumentException("catchUpCapGapPolicy must be non-null"); + } + this.catchUpCapGapPolicy = catchUpCapGapPolicy; + this.reconnectPolicy = catchUpCapGapPolicy == CatchUpCapGapPolicy.RETRY_FOREVER + ? ReconnectPolicy.FOREGROUND + : ReconnectPolicy.ORPHAN; if (engine == null) { throw new IllegalArgumentException("engine must be non-null"); } @@ -489,6 +723,115 @@ public CursorWebSocketSendLoop(WebSocketClient client, CursorSendEngine engine, } this.client = client; this.engine = engine; + this.hasReplayDictionaryDependency = engine.isDeltaDictEnabled() + || engine.recoveredMaxSymbolDeltaStart() > 0L; + // Recovery / orphan-drain: the loop starts with a fresh in-memory mirror, + // so seed it from the slot's persisted dictionary. That way the very first + // connection re-registers the whole dictionary (via a catch-up frame) + // before replaying the recovered delta frames. + // + // Deliberately NOT gated on engine.isDeltaDictEnabled(). The mirror, the + // catch-up and the replay guard together are this loop's model of what the + // SERVER's dictionary holds, and that model must track every mode. Gating + // them on that flag is what once bricked a recoverable slot: an unopenable + // .symbol-dict reports isDeltaDictEnabled()=false, yet the frames already on + // disk are still DELTA frames (deltaStart 0,1,2,...). With the mirror gated + // off, sentDictCount froze at 0 while the ungated guard kept comparing + // against it, so the frame at deltaStart=1 tripped a terminal -- even though + // replaying the whole sequence from id 0 rebuilds the dictionary on the + // server contiguously and drains perfectly. isDeltaDictEnabled() decides + // what the PRODUCER emits and persists; it must never decide what this loop + // tracks. When the dictionary could not be opened, pd is null, the mirror + // simply starts empty and grows from the frames themselves. + PersistedSymbolDict pd = engine.getPersistedSymbolDict(); + int persistedPrefixLen = 0; + if (pd != null && pd.size() > 0) { + int len = pd.loadedEntriesLen(); + if (len > 0) { + // Seed by reference. The foreground loop takes ownership after + // construction succeeds; orphan-drainer loops keep borrowing because + // one engine can create several sessions during capability-gap + // recycling. A borrowed mirror performs copy-on-write if a recovered + // frame suffix must extend it. + persistedPrefixLen = len; + sentDictBytesAddr = pd.loadedEntriesAddr(); + sentDictBytesCapacity = len; + sentDictBytesLen = len; + // Set the count only alongside the bytes so sentDictCount can + // never claim symbols the mirror does not hold. A recovered slot + // always has loadedEntriesLen > 0 when size > 0, so this is the + // same result -- it just makes the coupling explicit. + sentDictCount = pd.size(); + } + } + // ...and then from the surviving frames' own delta sections, exactly as the producer + // seeds its dictionary. The mirror is the loop's model of what the SERVER holds and it + // is what the catch-up frame ships, so when it stops at the persisted prefix while the + // producer's baseline runs past it, the loop condemns (deltaStart > sentDictCount) a + // slot the producer just seeded successfully. Same two sources, same order, so the two + // land on the same number by construction. + // ...but ONLY when a surviving frame actually depends on ids it does not carry + // (recoveredMaxSymbolDeltaStart > 0). At zero every frame is self-sufficient and + // re-registers its dictionary from id 0 as it replays, so seeding the mirror would buy + // nothing and cost a catch-up frame on every connection -- full-dict slots must stay + // catch-up-free. + // The prefix seed above may be borrowed, while the entry-end index is always + // loop-owned native memory. Build/extend both inside one cleanup boundary: + // any allocation failure propagates out of the constructor before a caller + // can own the half-built loop, so nothing else could release them. + try { + if (engine.recoveredMaxSymbolDeltaStart() > 0L) { + int baseline = sentDictCount; + if (engine.recoveredSymbolCoverage(baseline) >= 0L) { + int suffixLen = engine.recoveredSymbolSuffixLen(baseline); + int suffixCount = engine.recoveredSymbolSuffixCount(baseline); + if (suffixLen > 0) { + ensureSentDictCapacity((long) sentDictBytesLen + suffixLen); + if (forceMirrorSeedFailureForTest) { + // Throw AFTER the grow, never before it. ensureSentDictCapacity + // is what copy-on-writes a borrowed prefix into a loop-OWNED + // allocation (sentDictBytesOwned = true), and the catch's + // releaseSentDictBytes() is gated on exactly that flag. A seam + // in front of the grow fires while the mirror is still borrowed + // from PersistedSymbolDict, so the cleanup frees nothing and the + // guard below passes even with the cleanup deleted -- it would + // stop pinning the leak it exists for. + throw new LineSenderException( + "simulated mirror seed allocation failure (test only)"); + } + engine.copyRecoveredSymbolSuffix( + baseline, sentDictBytesAddr + sentDictBytesLen); + sentDictBytesLen += suffixLen; + sentDictCount += suffixCount; + } + } + } + // The entry-ends index is NOT built here. sendDictCatchUp is its only + // reader and builds it on demand, so a recovered slot that drains without + // ever reconnecting -- the normal case -- never pays the O(n) walk nor + // retains the 4-bytes-per-symbol index. + } catch (Throwable t) { + releaseSentDictBytes(); + throw t; + } + // QwpWebSocketSender seeds the producer dictionary before constructing + // its one foreground loop, so after the loop has built its mirror it can + // retire both engine-owned recovery sources. BackgroundDrainer must retain + // them for later recycled loops and therefore keeps borrowing them. + if (reconnectPolicy == ReconnectPolicy.FOREGROUND) { + if (persistedPrefixLen > 0) { + long persistedPrefixAddr = pd.takeLoadedEntries(); + if (sentDictBytesOwned) { + // Copy-on-grow already produced the combined mirror. Retire the + // no-longer-needed persisted prefix after successful construction. + Unsafe.free(persistedPrefixAddr, persistedPrefixLen, MemoryTag.NATIVE_DEFAULT); + } else { + assert persistedPrefixAddr == sentDictBytesAddr; + sentDictBytesOwned = true; + } + } + engine.releaseRecoveredSymbolStorage(); + } this.fsnAtZero = fsnAtZero; this.parkNanos = parkNanos; this.reconnectFactory = reconnectFactory; @@ -496,12 +839,20 @@ public CursorWebSocketSendLoop(WebSocketClient client, CursorSendEngine engine, this.reconnectInitialBackoffMillis = reconnectInitialBackoffMillis; this.reconnectMaxBackoffMillis = reconnectMaxBackoffMillis; this.durableAckMode = durableAckMode; + // Saturate, never multiply raw -- the same hazard the cap-gap dwell above + // guards. A raw multiply wraps a large millisecond value NEGATIVE, and both of + // these read as "elapsed >= window", so a negative makes the gate trivially + // true: the keepalive would ping on every loop pass, and the poison detector + // would escalate on the strike count alone -- exactly the transient-to-terminal + // false positive the dwell exists to stop (Invariant B). Both keys are + // user-supplied and validated only as >= 0, so asking for a very long window is + // legal, and asking for a longer one must never buy a shorter one. this.durableAckKeepaliveIntervalNanos = durableAckKeepaliveIntervalMillis > 0 - ? durableAckKeepaliveIntervalMillis * 1_000_000L + ? TimeUnit.MILLISECONDS.toNanos(durableAckKeepaliveIntervalMillis) : 0L; this.maxHeadFrameRejections = maxHeadFrameRejections; this.poisonMinEscalationWindowNanos = poisonMinEscalationWindowMillis > 0 - ? poisonMinEscalationWindowMillis * 1_000_000L + ? TimeUnit.MILLISECONDS.toNanos(poisonMinEscalationWindowMillis) : 0L; // SYNC/OFF startup hands a live client to the constructor, so we // already know we reached the server at least once. ASYNC startup @@ -519,6 +870,41 @@ public CursorWebSocketSendLoop(WebSocketClient client, CursorSendEngine engine, } } + /** + * Policy-aware master constructor. A foreground sender fails fast while + * establishing its first connection, then retries endpoint-policy failures + * indefinitely after it has been live. An orphan drainer returns such failures + * to its owner so the slot can follow its settle/quarantine policy. + */ + public CursorWebSocketSendLoop(WebSocketClient client, CursorSendEngine engine, + long fsnAtZero, long parkNanos, + ReconnectFactory reconnectFactory, + long reconnectMaxDurationMillis, + long reconnectInitialBackoffMillis, + long reconnectMaxBackoffMillis, + boolean durableAckMode, + long durableAckKeepaliveIntervalMillis, + int maxHeadFrameRejections, + long poisonMinEscalationWindowMillis, + long catchUpCapGapMinEscalationWindowMillis, + ReconnectPolicy reconnectPolicy) { + this(client, engine, fsnAtZero, parkNanos, reconnectFactory, + reconnectMaxDurationMillis, reconnectInitialBackoffMillis, + reconnectMaxBackoffMillis, durableAckMode, + durableAckKeepaliveIntervalMillis, maxHeadFrameRejections, + poisonMinEscalationWindowMillis, catchUpCapGapMinEscalationWindowMillis, + catchUpPolicyFor(reconnectPolicy)); + } + + private static CatchUpCapGapPolicy catchUpPolicyFor(ReconnectPolicy reconnectPolicy) { + if (reconnectPolicy == null) { + throw new IllegalArgumentException("reconnectPolicy must be non-null"); + } + return reconnectPolicy == ReconnectPolicy.FOREGROUND + ? CatchUpCapGapPolicy.RETRY_FOREVER + : CatchUpCapGapPolicy.TERMINAL_AFTER_SETTLE_BUDGET; + } + /** * Maps a server status byte to a {@link SenderError.Category}. Exposed for unit tests. */ @@ -565,12 +951,21 @@ public static WebSocketClient connectWithRetry( String contextLabel ) { long startNanos = System.nanoTime(); - long deadlineNanos = startNanos + maxDurationMillis * 1_000_000L; + // Saturating budget, compared against ELAPSED rather than an absolute + // startNanos+budget deadline. A large reconnect_max_duration_millis (e.g. + // Long.MAX_VALUE, the natural "retry until the server boots") must not collapse + // the budget to zero: a raw maxDurationMillis*1_000_000L wraps NEGATIVE -- the + // same overflow the drainer's capabilityGapBudgetNanos and the loop dwells were + // hardened against -- so the pre-condition loop below would make ZERO attempts. + // Even TimeUnit.toNanos saturated to Long.MAX_VALUE would overflow an absolute + // startNanos+budget sum; comparing the bounded nanoTime difference against the + // budget stays correct at saturation. + long budgetNanos = TimeUnit.MILLISECONDS.toNanos(maxDurationMillis); long backoffMillis = initialBackoffMillis; int attempts = 0; long lastLogNanos = 0L; Throwable lastError = null; - while (System.nanoTime() < deadlineNanos) { + while (System.nanoTime() - startNanos < budgetNanos) { attempts++; try { WebSocketClient c = factory.reconnect(); @@ -625,7 +1020,7 @@ public static WebSocketClient connectWithRetry( // this > 0. Mirrors BackgroundDrainer's sweep-loop jitter guard. long jitter = ThreadLocalRandom.current().nextLong(Math.max(1L, backoffMillis)); long sleepMillis = backoffMillis + jitter; - long remainingMillis = (deadlineNanos - System.nanoTime()) / 1_000_000L; + long remainingMillis = (budgetNanos - (System.nanoTime() - startNanos)) / 1_000_000L; if (remainingMillis <= 0) { break; } @@ -696,6 +1091,16 @@ public static boolean isTerminalCloseCode(int code) { } } + @TestOnly + public static int maxCatchUpCapGapAttempts() { + return MAX_CATCHUP_CAP_GAP_ATTEMPTS; + } + + @TestOnly + public static int maxSentDictBytes() { + return MAX_SENT_DICT_BYTES; + } + /** * Surfaces any error the I/O thread recorded. Called by the producer * thread (typically from inside its append wrapper) so failures don't @@ -754,6 +1159,12 @@ public synchronized void close() { // after — the latch await is only skipped when the loop never ran. running = false; Thread t = ioThread; + // The symbol-dict mirror (sentDictBytesAddr) is I/O-thread-owned and gets + // freed on ioLoop's exit path. When t == null the loop never ran (start() + // was never called, or t.start() failed before committing ioThread), so + // that free never happens and a seeded (recovery / orphan-drain) mirror + // would leak. Capture that here; free it below, after client teardown. + boolean loopNeverRan = t == null; if (t != null) { LockSupport.unpark(t); // Only await the shutdown latch if the I/O thread actually ran. @@ -812,6 +1223,17 @@ public synchronized void close() { } client = null; } + // Free the I/O-thread-owned symbol-dict mirror ONLY when the loop never + // ran (see loopNeverRan). If it ran, ioLoop's exit already freed it -- and + // on the failed-stop path (interrupted latch await, ioThread left set) the + // thread may still be mid-send, so touching the mirror here would race. + // A duplicate close observes sentDictBytesAddr == 0 and skips. + if (loopNeverRan && sentDictBytesAddr != 0) { + releaseSentDictBytes(); + } + if (loopNeverRan) { + freeCatchUpFrameBuffer(); + } } /** @@ -996,7 +1418,28 @@ public synchronized void start() { // walks back to the lowest unacked frame so sealed-segment data // actually reaches the wire — without it, start() would skip // straight to the active and orphan everything in sealed. - positionCursorForStart(); + try { + positionCursorForStart(); + } catch (CatchUpSendException e) { + // A recovered sender re-registers its dictionary with a catch-up on + // the very first connect. Here that runs on the CALLER thread (sync + // start), so we must NOT let it drive connectLoop -- that would block + // Sender construction forever on a transient outage. Drop the dead + // client instead: the I/O thread then reconnects via + // attemptInitialConnect -> swapClient and re-sends the catch-up off + // this thread. If the failure was already terminal (recordFatal set + // running=false, e.g. an entry too large for the batch cap), the I/O + // thread simply winds down and checkError() surfaces it. + WebSocketClient dead = client; + client = null; + if (dead != null) { + try { + dead.close(); + } catch (Throwable ignored) { + // best-effort + } + } + } Thread t = new Thread(this::ioLoop, "qdb-cursor-ws-io"); t.setDaemon(true); try { @@ -1079,10 +1522,10 @@ private void applyDurableAck() { /** * Drives the very first connect attempt on the I/O thread, used in the * async-initial-connect mode (constructed with {@code client == null}). - * Reuses the same retry+backoff machinery as {@link #fail(Throwable)} — - * connect failures are retried indefinitely (Invariant B), and a - * terminal upgrade reject is delivered through the dispatcher, not - * thrown to the producer. + * Reuses the same retry+backoff machinery as {@link #fail(Throwable)}. + * Transport, authentication, upgrade and capability failures are all + * retried indefinitely here (Invariant B); none is surfaced to the + * foreground producer. */ private void attemptInitialConnect() { connectLoop(new LineSenderException( @@ -1104,6 +1547,15 @@ private void clearDurableAckTracking() { lastFrameOrPingNanos = 0L; } + private static boolean isCatchUpCapGap(Throwable t) { + return t instanceof CatchUpSendException && ((CatchUpSendException) t).isCapGap; + } + + private void resetCatchUpCapGapEpisode() { + catchUpCapGapAttempts = 0; + catchUpCapGapFirstNanos = -1L; + } + /** * Shared per-outage retry loop. Used by {@link #fail(Throwable)} for * mid-flight wire failures (phase="reconnect") and by @@ -1118,16 +1570,24 @@ private void connectLoop(Throwable initial, String phase, long paceFirstAttemptM recordFatal(initial); return; } + // A wire/role state that interrupted an orphan cap-gap run is not evidence that + // the cluster's batch cap stayed incompatible during the outage. Start a fresh + // episode; the cap-gap exception itself is the one reconnect cause that preserves + // the existing attempt count and dwell anchor. + if (!isCatchUpCapGap(initial)) { + resetCatchUpCapGapEpisode(); + } LOG.warn("cursor I/O loop entering {} loop: {}", phase, initial.getMessage()); long outageStartNanos = System.nanoTime(); - // INVARIANT B: a store-and-forward drainer must NEVER terminate on a + // INVARIANT B: a store-and-forward loop must NEVER terminate on a // wall-clock reconnect budget. A replica-only / all-endpoints-replica // window is TRANSIENT -- a replica gets promoted, a primary reappears -- // so this background loop retries for as long as it is running, backing - // off between attempts. The ONLY terminal conditions are a genuinely - // non-retriable upgrade (auth / non-421 upgrade / durable-ack capability - // gap), which return directly below, or the sender being stopped. SF + // off between attempts. Endpoint-policy failures (auth / non-421 + // upgrade / durable-ack capability gap) are terminal only for orphan + // drainers. Foreground senders retry them from asynchronous startup onward + // so a credential or cluster capability rotation cannot stop the producer. SF // exhaustion is surfaced to the PRODUCER as append backpressure, never // here. reconnect_max_duration_millis is intentionally NOT consulted: it // bounds only the blocking (non-lazy) initial connect in @@ -1187,66 +1647,81 @@ private void connectLoop(Throwable initial, String phase, long paceFirstAttemptM phase, elapsedMs, attempts, fsnAtZero); return; } + // A null factory result is an unsuccessful connect state, not a cap-gap + // observation. Do not let the time spent retrying it satisfy orphan dwell. + resetCatchUpCapGapEpisode(); } catch (QwpAuthFailedException | WebSocketUpgradeException e) { - // Terminal across all configured endpoints per spec sf-client.md - // section 13.3: auth (401/403) bypasses reconnect and surfaces as - // SECURITY_ERROR. WebSocketUpgradeException reaching here is always - // non-421: QwpUpgradeFailures.classify upstream converts a - // 421-with-X-QuestDB-Role to QwpIngressRoleRejectedException, and a - // 421 without that header walks the transport-error path in - // buildAndConnect and lands as a LineSenderException, falling into - // the Throwable branch below. - LOG.error("terminal upgrade error during {} -- won't retry: {}", - phase, e.getMessage()); - long fromFsn = engine.ackedFsn() + 1L; - long toFsn = Math.max(fromFsn, engine.publishedFsn()); - SenderError err = new SenderError( - SenderError.Category.SECURITY_ERROR, - SenderError.Policy.TERMINAL, - SenderError.NO_STATUS_BYTE, - "ws-upgrade-failed: " + e.getMessage(), - SenderError.NO_MESSAGE_SEQUENCE, - fromFsn, - toFsn, - null, - System.nanoTime() - ); - totalServerErrors.incrementAndGet(); - recordFatal(new LineSenderServerException(err)); - dispatchError(err); - return; + if (endpointPolicyFailureIsTerminal()) { + // Orphans return control to their quarantine owner. + // WebSocketUpgradeException reaching here is always non-421: + // role rejects are classified into the transient branch below. + LOG.error("terminal upgrade error during {} -- won't retry: {}", + phase, e.getMessage()); + long fromFsn = engine.ackedFsn() + 1L; + long toFsn = Math.max(fromFsn, engine.publishedFsn()); + SenderError err = new SenderError( + SenderError.Category.SECURITY_ERROR, + SenderError.Policy.TERMINAL, + SenderError.NO_STATUS_BYTE, + "ws-upgrade-failed: " + e.getMessage(), + SenderError.NO_MESSAGE_SEQUENCE, + fromFsn, + toFsn, + null, + System.nanoTime() + ); + totalServerErrors.incrementAndGet(); + recordFatal(new LineSenderServerException(err)); + dispatchError(err); + return; + } + resetCatchUpCapGapEpisode(); + lastReconnectError = e; + long now = System.nanoTime(); + if (now - lastLogNanos >= RECONNECT_LOG_THROTTLE_NANOS) { + LOG.warn("{} attempt {}: foreground auth/upgrade policy rejected the connection; " + + "retrying while store-and-forward owns the buffered data -- {}", + phase, attempts, e.getMessage()); + lastLogNanos = now; + } } catch (QwpDurableAckMismatchException e) { - // Per spec sf-client.md section 8.1: the client opted into durable - // ack but the cluster cannot honour it. Loud fail at connect rather - // than silently waiting for ack frames that will never arrive. - // Classified as PROTOCOL_VIOLATION (config/capability mismatch), - // not SECURITY_ERROR -- this is not an auth failure. - LOG.error("durable-ack mismatch during {} -- won't retry: {}", - phase, e.getMessage()); - if (terminalError == null) { - // Mirror recordFatal's first-writer-wins latch: only the - // sweep that owns the terminal may mark the gap, and the - // marker must be visible before the terminalError volatile - // write that checkError() keys on. - capabilityGapTerminal = e; + if (endpointPolicyFailureIsTerminal()) { + // Orphans hand a capability gap back to BackgroundDrainer's + // settle budget. + LOG.error("durable-ack mismatch during {} -- won't retry: {}", + phase, e.getMessage()); + if (terminalError == null) { + // Publish the marker before terminalError, which is the + // volatile first-writer-wins latch observed by the owner. + capabilityGapTerminal = e; + } + long fromFsn = engine.ackedFsn() + 1L; + long toFsn = Math.max(fromFsn, engine.publishedFsn()); + SenderError err = new SenderError( + SenderError.Category.PROTOCOL_VIOLATION, + SenderError.Policy.TERMINAL, + SenderError.NO_STATUS_BYTE, + "durable-ack-mismatch: " + e.getMessage(), + SenderError.NO_MESSAGE_SEQUENCE, + fromFsn, + toFsn, + null, + System.nanoTime() + ); + totalServerErrors.incrementAndGet(); + recordFatal(new LineSenderServerException(err)); + dispatchError(err); + return; + } + resetCatchUpCapGapEpisode(); + lastReconnectError = e; + long now = System.nanoTime(); + if (now - lastLogNanos >= RECONNECT_LOG_THROTTLE_NANOS) { + LOG.warn("{} attempt {}: foreground durable-ack capability is temporarily " + + "unavailable; retrying while store-and-forward owns the buffered data -- {}", + phase, attempts, e.getMessage()); + lastLogNanos = now; } - long fromFsn = engine.ackedFsn() + 1L; - long toFsn = Math.max(fromFsn, engine.publishedFsn()); - SenderError err = new SenderError( - SenderError.Category.PROTOCOL_VIOLATION, - SenderError.Policy.TERMINAL, - SenderError.NO_STATUS_BYTE, - "durable-ack-mismatch: " + e.getMessage(), - SenderError.NO_MESSAGE_SEQUENCE, - fromFsn, - toFsn, - null, - System.nanoTime() - ); - totalServerErrors.incrementAndGet(); - recordFatal(new LineSenderServerException(err)); - dispatchError(err); - return; } catch (QwpRoleMismatchException | QwpIngressRoleRejectedException e) { // Role mismatch: every reachable endpoint role-rejected the // upgrade -- right now they are all replicas / primary-catchup. @@ -1263,6 +1738,7 @@ private void connectLoop(Throwable initial, String phase, long paceFirstAttemptM // endpoint, forever. Growing to reconnectMaxBackoffMillis // mirrors the orphan drainer's role-reject path and honours the // documented capped-exponential-backoff contract. + resetCatchUpCapGapEpisode(); lastReconnectError = e; long now = System.nanoTime(); if (now - lastLogNanos >= RECONNECT_LOG_THROTTLE_NANOS) { @@ -1286,6 +1762,9 @@ private void connectLoop(Throwable initial, String phase, long paceFirstAttemptM recordFatal(e); throw (Error) e; } + if (!isCatchUpCapGap(e)) { + resetCatchUpCapGapEpisode(); + } lastReconnectError = e; long now = System.nanoTime(); if (now - lastLogNanos >= RECONNECT_LOG_THROTTLE_NANOS) { @@ -1331,6 +1810,33 @@ private void connectLoop(Throwable initial, String phase, long paceFirstAttemptM phase, elapsedMs, attempts, lastMsg); } + /** + * Whether an endpoint-policy failure (auth, non-421 upgrade, durable-ack + * capability gap) latches a terminal instead of retrying under Invariant B. + *

+ * Two callers own a terminal, for different reasons: + *

+ * Once a FOREGROUND sender has reached the server even once, initialization is + * over and store-and-forward owns the buffered data: every endpoint-policy + * failure is then a transient the drainer must ride out, never a producer-fatal + * terminal (Invariant B). + */ + private boolean endpointPolicyFailureIsTerminal() { + return reconnectPolicy == ReconnectPolicy.ORPHAN || !hasEverConnected; + } + /** * Send {@code err} to the async-delivery dispatcher if one is configured. * Producer-side typed throw (TERMINAL) goes through {@code recordFatal} + @@ -1480,8 +1986,8 @@ private void enqueuePendingOk(long wireSeq) { * listener both non-null), enters the per-outage retry loop: capped * exponential backoff with jitter, retried for as long as the loop is * running -- there is NO wall-clock give-up (Invariant B: a store-and- - * forward drainer only terminates on SF exhaustion or a genuinely non- - * retriable auth/upgrade reject). On the first successful reconnect the + * forward drainer does not give up on endpoint or transport state). On the + * first successful reconnect the * I/O loop resumes with reset wire state and replays from * {@code engine.ackedFsn() + 1}. *

@@ -1587,11 +2093,10 @@ private void ioLoop() { // Async-initial-connect path: ctor accepted a null client because // a reconnect factory is wired. Drive the very first connect on // this thread so the producer thread never blocks on it. - // attemptInitialConnect either sets `client` (success) or records - // a terminal failure and clears `running` (auth/upgrade reject; - // plain connect failures retry indefinitely under Invariant B). - // Either way, the main loop below sees the outcome via the - // `running` and `client` fields. + // attemptInitialConnect retries every endpoint/transport failure + // indefinitely under Invariant B and returns only after it installs + // a client or close() clears `running`. The main loop below sees the + // outcome via the `running` and `client` fields. if (client == null && running) { attemptInitialConnect(); } @@ -1669,6 +2174,12 @@ private void ioLoop() { // best-effort } } + // The symbol-dict mirror is I/O-thread-owned; free it here, on the + // owning thread's exit path, after the last send that could touch it. + if (sentDictBytesAddr != 0) { + releaseSentDictBytes(); + } + freeCatchUpFrameBuffer(); shutdownLatch.countDown(); // Failed-stop hand-off (see delegateEngineClose): the owner could // not free the engine safely while this thread was alive, so the @@ -1691,8 +2202,11 @@ private void ioLoop() { /** * Walk the engine's segments to find the one containing {@code targetFsn}, * and set {@code sendOffset} to the byte offset of that frame within it. - * This is called at startup and after every reconnect, after fsnAtZero has - * already been reset to {@code targetFsn} and nextWireSeq to 0. + * This is called at startup and after every reconnect, once + * {@link #setWireBaselineWithCatchUp} has anchored the wire baseline + * ({@code fsnAtZero} / {@code nextWireSeq}) -- which may leave {@code nextWireSeq} + * past the catch-up frames it emitted. This method only positions the byte + * cursor at {@code targetFsn}; it does not touch the wire mapping. *

* If {@code targetFsn} is already published, the method positions the byte * cursor exactly at that frame. If {@code targetFsn} is not published yet, @@ -1832,9 +2346,12 @@ private void sendDurableAckKeepaliveIfDue() { private void swapClient(WebSocketClient newClient) { WebSocketClient old = this.client; this.client = newClient; - // Sticky: once the wire is up, we've reached the server at least - // once for this sender's lifetime. Used downstream to classify a - // subsequent terminal failure as transient vs config-likely. + // Sticky: once the wire is up, we've reached the server at least once + // for this sender's lifetime. Exposed to the owning sender for + // connection-state observability, and it ends initialization: from here + // on endpointPolicyFailureIsTerminal() stops latching endpoint-policy + // failures on a foreground sender and rides them out instead, because + // store-and-forward now owns the buffered data (Invariant B). this.hasEverConnected = true; if (old != null) { try { @@ -1849,8 +2366,6 @@ private void swapClient(WebSocketClient newClient) { // past the tail instead of replaying into it. tryRetireOrphanTail(); long replayStart = engine.ackedFsn() + 1L; - this.fsnAtZero = replayStart; - this.nextWireSeq = 0L; // Snapshot publishedFsn at swap time — frames at FSN ≤ this value // were already on the wire before the drop and will be replayed. // trySendOne resets replayTargetFsn to -1 once we cross the boundary. @@ -1862,9 +2377,693 @@ private void swapClient(WebSocketClient newClient) { // carrying stale state across the wire boundary would either // double-trim or starve the queue. clearDurableAckTracking(); + setWireBaselineWithCatchUp(replayStart); positionCursorAt(replayStart); } + /** + * Sets the wire-sequence baseline for a fresh connection and, when the symbol + * dictionary mirror is non-empty, emits a full-dictionary catch-up frame first + * so the fresh server (whose dictionary starts empty) can resolve the + * non-self-sufficient delta frames that replay next. + *

+ * The catch-up occupies wire seq 0, which maps to the already-acked FSN just + * below {@code replayStart} (a harmless re-ack); real replay frames then follow + * from wire seq 1. With nothing to catch up (fresh sender, or full-dict mode), + * or before a client exists (async initial connect), keep the plain 1:1 + * {@code fsnAtZero == replayStart} mapping; the catch-up then happens on the + * first real connection via swapClient. + */ + private void setWireBaselineWithCatchUp(long replayStart) { + // Fresh connection: no data frame has been sent on it yet. Reset before the + // catch-up (which sends only dictionary frames) so onClose / + // handleServerRejection can tell "only the catch-up went out" from "the + // head data frame went out". + dataFrameSentThisConnection = false; + // Do not gate solely on isDeltaDictEnabled(): a recovered delta slot can + // lose access to its side-file while its on-disk frames still start above + // zero. hasReplayDictionaryDependency preserves that distinction while + // keeping full-dictionary fallback catch-up-free across every reconnect. + if (client != null && sentDictCount > 0 && hasReplayDictionaryDependency) { + this.nextWireSeq = 0L; + // The catch-up may span several frames when the dictionary exceeds the + // server's batch cap; each consumes a wire sequence (0 .. n-1) that maps + // to an already-acked FSN, so the first real frame still lands on + // replayStart. + int catchUpFrames = sendDictCatchUp(); + this.fsnAtZero = replayStart - catchUpFrames; + } else { + this.fsnAtZero = replayStart; + this.nextWireSeq = 0L; + } + } + + /** + * Returns the symbol-dictionary delta start id of a frame, or -1 when the + * frame carries no delta section. Used by the pre-send torn-dictionary guard. + */ + private int frameDeltaStart(long payloadAddr, int payloadLen) { + if (!isDeltaFrame(payloadAddr, payloadLen)) { + return -1; + } + return (int) readVarintAt(payloadAddr + QwpConstants.HEADER_SIZE, payloadAddr + payloadLen); + } + + // True only for a well-formed QWP frame this encoder produced that carries a + // delta symbol-dict section. The magic check keeps the dict logic from + // misreading non-QWP payloads (e.g. synthetic frames injected by tests) whose + // bytes happen to set the delta flag. + private static boolean isDeltaFrame(long payloadAddr, int payloadLen) { + if (payloadLen < QwpConstants.HEADER_SIZE + || Unsafe.getUnsafe().getInt(payloadAddr) != QwpConstants.MAGIC_MESSAGE) { + return false; + } + byte flags = Unsafe.getUnsafe().getByte(payloadAddr + QwpConstants.HEADER_OFFSET_FLAGS); + return (flags & QwpConstants.FLAG_DELTA_SYMBOL_DICT) != 0; + } + + /** + * Copies the symbol-dictionary delta a just-sent frame carries into the + * loop-local mirror ({@link #sentDictBytesAddr}) so a future reconnect can + * re-register it. Frames are sent in FSN order carrying monotonically + * extending deltas, so a frame whose delta starts exactly at + * {@link #sentDictCount} extends the mirror; a replayed or empty-delta frame + * (nothing new) is skipped. Only ever called in delta mode, for a frame the + * pre-send guard already classified as a delta frame. + * + * @param payloadAddr address of the QWP message (12-byte header first) + * @param payloadLen message length in bytes + * @param deltaStart the frame's delta start id, already decoded by the + * pre-send guard ({@link #frameDeltaStart}) -- passed in so + * the magic/flags and start-id varint are not re-parsed + */ + private void accumulateSentDict(long payloadAddr, int payloadLen, int deltaStart) { + long limit = payloadAddr + payloadLen; + // deltaStart is known (the guard decoded it); locate deltaCount just past + // its canonical LEB128 encoding rather than re-reading the header and the + // start-id varint. + long p = payloadAddr + QwpConstants.HEADER_SIZE + NativeBufferWriter.varintSize(deltaStart); + long deltaCount = readVarintAt(p, limit); + p = varintEnd; + // The mirror holds ids [0, sentDictCount). Accumulate ONLY the part of this + // frame's delta [deltaStart, deltaStart+deltaCount) that extends past the + // tip -- ids [sentDictCount, deltaStart+deltaCount). Cases: + // - deltaStart > sentDictCount: a gap. trySendOne's torn-dictionary guard + // rejects it before send, so it never reaches here; bail defensively + // rather than accumulate past a hole. + // - deltaEnd <= sentDictCount: a pure replay/overlap we already hold -- + // nothing new. + // - deltaStart <= sentDictCount < deltaEnd: extend the mirror by the tail. + // deltaStart == sentDictCount is the steady-state case (skip == 0). + // Handling the partial overlap explicitly -- rather than dropping the whole + // frame whenever deltaStart != sentDictCount -- keeps the mirror complete + // even if a future producer ever emits a delta that overlaps the tip; + // silently dropping the new tail would leave the reconnect catch-up + // incomplete and shift server-side ids. + long deltaEnd = deltaStart + deltaCount; + if (deltaCount <= 0 || deltaStart > sentDictCount || deltaEnd <= sentDictCount) { + return; + } + // Deliberately does NOT touch the entry-ends index. Only sendDictCatchUp reads + // it, so maintaining it here would put a store per new symbol on the per-frame + // I/O path -- and on the workload this feature targets (a new symbol per ROW) + // that is a store per row, plus 4 bytes per symbol retained for the connection's + // whole life, to serve a reconnect that may never happen. sendDictCatchUp calls + // ensureSentDictEntryIndex(), which notices sentDictIndexedCount has fallen + // behind sentDictCount and rebuilds; that O(n) walk runs at most once per + // reconnect and is dwarfed by shipping the same n bytes over the wire. A + // connection that never reconnects now never allocates the index at all. + // + // Walk past the already-held prefix [deltaStart, sentDictCount), then copy + // the new tail [sentDictCount, deltaEnd). + int skip = sentDictCount - deltaStart; + for (int i = 0; i < skip; i++) { + long len = readVarintAt(p, limit); + p = varintEnd + len; + if (p > limit) { + return; // malformed -- bail rather than corrupt the mirror + } + } + long regionStart = p; + long newCount = deltaEnd - sentDictCount; + // Walk the new tail only to find where it ends; the entry boundaries are not + // recorded here (see above -- the catch-up rebuilds them on demand). + for (long i = 0; i < newCount; i++) { + long len = readVarintAt(p, limit); + p = varintEnd + len; + if (p > limit) { + // Malformed -- never happens for frames we encoded; bail rather + // than corrupt the mirror. + return; + } + } + int regionBytes = (int) (p - regionStart); + // long sum: sentDictBytesLen + regionBytes can exceed Integer.MAX_VALUE on + // a very-high-cardinality lifetime connection; ensureSentDictCapacity then + // fails loudly rather than overflowing to a negative int (which would make + // the capacity check pass and copyMemory scribble past the buffer). + ensureSentDictCapacity((long) sentDictBytesLen + regionBytes); + Unsafe.getUnsafe().copyMemory(regionStart, sentDictBytesAddr + sentDictBytesLen, regionBytes); + sentDictBytesLen += regionBytes; + sentDictCount += (int) newCount; + } + + private void ensureSentDictCapacity(long required) { + if (sentDictBytesCapacity >= required) { + return; + } + if (required > MAX_SENT_DICT_BYTES) { + // Latch a terminal, do NOT just throw: accumulateSentDict runs AFTER + // the frame's sendBinary, so a bare throw unwinds to ioLoop -> fail() + // -> connectLoop, which (running still true) reconnects and replays the + // same frame, which re-overflows the never-shrinking mirror -- an + // unbounded reconnect livelock rather than the "fails loudly" the + // MAX_SENT_DICT_BYTES ceiling promises. recordFatal flips running=false + // so connectLoop's !running guard winds the loop down and checkError() + // surfaces the terminal, matching sendCatchUpChunk's guard for the same + // ceiling. The throw still unwinds past the pending copyMemory. + LineSenderException err = new LineSenderException("symbol dictionary mirror exceeds the maximum size [" + + "required=" + required + ", max=" + MAX_SENT_DICT_BYTES + ']'); + recordFatal(err); + throw err; + } + // Grow in long to avoid the capacity*2 int overflow (negative) that would + // otherwise degrade the doubling near 1 GB; clamp to the int ceiling. + long newCap = Math.max((long) sentDictBytesCapacity * 2, Math.max(4096L, required)); + if (newCap > MAX_SENT_DICT_BYTES) { + newCap = MAX_SENT_DICT_BYTES; + } + if (sentDictBytesOwned) { + sentDictBytesAddr = Unsafe.realloc( + sentDictBytesAddr, + sentDictBytesCapacity, + (int) newCap, + MemoryTag.NATIVE_DEFAULT); + } else { + long newAddr = Unsafe.malloc((int) newCap, MemoryTag.NATIVE_DEFAULT); + if (sentDictBytesLen > 0) { + Unsafe.getUnsafe().copyMemory(sentDictBytesAddr, newAddr, sentDictBytesLen); + } + sentDictBytesAddr = newAddr; + sentDictBytesOwned = true; + } + sentDictBytesCapacity = (int) newCap; + } + + private void releaseSentDictBytes() { + if (sentDictBytesAddr != 0 && sentDictBytesOwned) { + Unsafe.free(sentDictBytesAddr, sentDictBytesCapacity, MemoryTag.NATIVE_DEFAULT); + } + if (sentDictEntryEndsAddr != 0L) { + Unsafe.free( + sentDictEntryEndsAddr, + sentDictEntryEndsCapacity * Integer.BYTES, + MemoryTag.NATIVE_DEFAULT); + } + sentDictBytesAddr = 0; + sentDictBytesCapacity = 0; + sentDictBytesLen = 0; + sentDictBytesOwned = false; + sentDictCount = 0; + sentDictEntryEndsAddr = 0L; + sentDictEntryEndsCapacity = 0; + sentDictIndexedCount = 0; + } + + private void ensureSentDictEntryEndsCapacity(long required) { + if (required <= sentDictEntryEndsCapacity) { + return; + } + if (required > MAX_SENT_DICT_ENTRIES) { + LineSenderException err = new LineSenderException( + "symbol dictionary entry index exceeds the maximum size [required=" + + required + ", max=" + MAX_SENT_DICT_ENTRIES + ']'); + recordFatal(err); + throw err; + } + long newCapacity = Math.max( + (long) sentDictEntryEndsCapacity * 2L, + Math.max(1024L, required)); + if (newCapacity > MAX_SENT_DICT_ENTRIES) { + newCapacity = MAX_SENT_DICT_ENTRIES; + } + sentDictEntryEndsAddr = Unsafe.realloc( + sentDictEntryEndsAddr, + sentDictEntryEndsCapacity * Integer.BYTES, + (int) newCapacity * Integer.BYTES, + MemoryTag.NATIVE_DEFAULT); + sentDictEntryEndsCapacity = (int) newCapacity; + } + + private void ensureSentDictEntryIndex() { + if (sentDictIndexedCount != sentDictCount) { + rebuildSentDictEntryIndex(); + } + } + + private void rebuildSentDictEntryIndex() { + ensureSentDictEntryEndsCapacity(sentDictCount); + long p = sentDictBytesAddr; + long limit = sentDictBytesAddr + sentDictBytesLen; + for (int i = 0; i < sentDictCount; i++) { + long len = readVarintAt(p, limit); + p = varintEnd + len; + if (p > limit) { + throw new LineSenderException( + "invalid symbol dictionary mirror while building the entry index [entry=" + + i + ", count=" + sentDictCount + ']'); + } + Unsafe.getUnsafe().putInt( + sentDictEntryEndsAddr + (long) i * Integer.BYTES, + (int) (p - sentDictBytesAddr)); + } + if (p != limit) { + throw new LineSenderException( + "invalid symbol dictionary mirror length while building the entry index [indexed=" + + (p - sentDictBytesAddr) + ", length=" + sentDictBytesLen + ']'); + } + sentDictIndexedCount = sentDictCount; + catchUpEntryIndexBuildCount++; + } + + private long readVarintAt(long p, long limit) { + long value = 0; + int shift = 0; + long cur = p; + while (cur < limit) { + byte b = Unsafe.getUnsafe().getByte(cur++); + value |= (long) (b & 0x7F) << shift; + if ((b & 0x80) == 0) { + break; + } + shift += 7; + if (shift > 35) { + // Defensive bound, matching PersistedSymbolDict.decodeVarint: a + // canonical entry-length / delta varint is <= 5 bytes. Every caller + // reads freshly-encoded, CRC- or openExisting-validated bytes, so + // this is unreachable, but it stops a corrupt continuation run from + // over-shifting into a garbage length. + break; + } + } + varintEnd = cur; + return value; + } + + /** + * Re-registers the whole symbol dictionary on a fresh connection, split into + * as many table-less frames as the server's advertised batch cap requires so + * no single frame exceeds it (a large dictionary would otherwise be rejected). + * Each chunk carries a contiguous id range {@code [start .. start+count)}, in + * order, so the server accumulates them exactly as it would the original + * per-frame deltas. Returns the number of frames sent (each consumed a wire + * sequence), so the caller can align {@code fsnAtZero}. Throws {@link + * CatchUpSendException} on a send error (retriable -- the caller reconnects); + * a single entry too large for the cap is non-retriable, so it latches a + * terminal before throwing. + */ + private int sendDictCatchUp() { + int cap = client.getServerMaxBatchSize(); + long fullFrameLen = QwpConstants.HEADER_SIZE + + NativeBufferWriter.varintSize(0) + + NativeBufferWriter.varintSize(sentDictCount) + + (long) sentDictBytesLen; + if (cap <= 0 && fullFrameLen <= MAX_SENT_DICT_BYTES) { + sendCatchUpChunk(0, sentDictCount, sentDictBytesAddr, sentDictBytesLen); + resetCatchUpCapGapEpisode(); + return 1; + } + ensureSentDictEntryIndex(); + // The frame ceiling a catch-up chunk must not exceed: the server's + // advertised cap, or -- when the server advertises none (cap <= 0) -- + // MAX_SENT_DICT_BYTES so sendCatchUpChunk's int frameLen (HEADER_SIZE + + // varints + symbolsLen) cannot overflow on a pathological multi-GB + // dictionary (unreachable at real cardinality; defensive). Used by the + // single-entry terminal below, which measures the real solo frame. + int frameLimit = cap > 0 ? cap : MAX_SENT_DICT_BYTES; + // Symbol-bytes budget for PACKING several entries into one chunk, leaving + // room for the 12-byte header and the two delta-section varints. Kept + // deliberately conservative (reserving 16 for the varints): it only makes a + // multi-entry chunk split marginally earlier, never over the cap. It must + // NOT gate the single-entry terminal -- that reserve is larger than the + // minimal data-frame overhead, so an entry the producer already shipped + // under this cap could exceed the reserve yet still fit its own catch-up + // frame; the terminal tests the real solo frame against frameLimit instead. + int budget = cap > 0 + ? Math.max(1, cap - QwpConstants.HEADER_SIZE - 16) + : MAX_SENT_DICT_BYTES - QwpConstants.HEADER_SIZE - 16; + int framesSent = 0; + int chunkStartId = 0; + long chunkStartAddr = sentDictBytesAddr; + int chunkSymbols = 0; + long chunkBytes = 0; + for (int entryId = 0; entryId < sentDictCount; entryId++) { + int entryStartOffset = entryId == 0 + ? 0 + : Unsafe.getUnsafe().getInt( + sentDictEntryEndsAddr + (long) (entryId - 1) * Integer.BYTES); + int entryEndOffset = Unsafe.getUnsafe().getInt( + sentDictEntryEndsAddr + (long) entryId * Integer.BYTES); + long entryStart = sentDictBytesAddr + entryStartOffset; + long entryBytes = entryEndOffset - entryStartOffset; + // The exact table-less frame sendCatchUpChunk would build for THIS entry + // alone: header + deltaStart varint (the entry's own global id) + + // deltaCount varint (1) + the entry bytes. A cap gap exists only when even + // that solo frame exceeds the cap -- i.e. the entry genuinely cannot be + // re-registered. Testing the real solo frame (not the conservative + // packing budget above) is what keeps a HOMOGENEOUS cluster + // livelock-free: an entry the producer already shipped in a data frame + // under this cap (header + delta varints + entry + schema + >=1 row) is + // strictly larger than its bare catch-up frame, so it always fits here. + long soloFrameLen = QwpConstants.HEADER_SIZE + + NativeBufferWriter.varintSize(chunkStartId + chunkSymbols) + + NativeBufferWriter.varintSize(1) + + entryBytes; + if (soloFrameLen > frameLimit) { + // Cap gap: this entry cannot be re-registered under the fresh + // server's advertised cap. A HOMOGENEOUS cluster never reaches here + // (an entry that fit its data frame under a cap always fits its bare + // catch-up frame under that same cap), so the only way in is a + // heterogeneous / rolling-cap failover to a smaller-cap node. + // + // A foreground sender retries indefinitely: store-and-forward must + // contain a transient heterogeneous-cluster window instead of surfacing + // it to the producer. Only an orphan drainer may apply the settle budget + // below and quarantine its slot after a persistent gap. Under budget the + // throw is RETRIABLE (no recordFatal) -- connectLoop reconnects with + // backoff and re-runs the catch-up, which resets the episode on a node + // that accepts it. An unrelated transport/upgrade/role state also resets + // the episode, so its downtime cannot satisfy the orphan dwell. On + // exhaustion latch via recordFatal, NOT fail() -- + // failing from inside the catch-up would re-enter connectLoop (see + // CatchUpSendException); the data must be resent after the cap is raised. + // Escalation needs BOTH the strike count AND a minimum wall-clock dwell + // (see DEFAULT_CATCHUP_CAP_GAP_MIN_ESCALATION_WINDOW_MILLIS). A count + // alone measures how often we looked, not how long the gap has held -- + // and 16 attempts at the capped backoff take only ~2 minutes, less than + // an ordinary rolling restart of the larger-cap node. Escalating on the + // count alone would therefore hard-fail a live store-and-forward + // producer during a routine cluster operation. + // + // The STRIKE COUNT -- never the anchor's sign -- says whether an episode is + // already open. A System.nanoTime() instant is only meaningful as a + // difference: its origin is arbitrary and the spec permits negative values, + // so a sign test cannot tell an unset anchor from a real one. Wherever it + // misreads a real anchor as unset it re-anchors to now on EVERY strike, + // pinning episodeNanos at ~0, and the dwell can then never be satisfied -- + // the terminal never latches, however long the gap truly persists. + // recordHeadRejectionStrike keys its episode off poisonStrikes for the same + // reason. + if (catchUpCapGapPolicy == CatchUpCapGapPolicy.RETRY_FOREVER) { + throw new CatchUpSendException(new LineSenderException( + "symbol dictionary entry too large for the server batch cap during catch-up [" + + "frameLen=" + soloFrameLen + ", cap=" + cap + ']' + + "; retrying indefinitely -- a larger-cap node may return"), true); + } + + long nowNanos = System.nanoTime(); + if (catchUpCapGapAttempts == 0) { + catchUpCapGapFirstNanos = nowNanos; + } + catchUpCapGapAttempts++; + long episodeNanos = nowNanos - catchUpCapGapFirstNanos; + boolean exhausted = catchUpCapGapAttempts >= MAX_CATCHUP_CAP_GAP_ATTEMPTS + && episodeNanos >= catchUpCapGapMinEscalationWindowNanos; + LineSenderException err = new LineSenderException( + "symbol dictionary entry too large for the server batch cap during catch-up [" + + "frameLen=" + soloFrameLen + ", cap=" + cap + ", attempt=" + + catchUpCapGapAttempts + '/' + MAX_CATCHUP_CAP_GAP_ATTEMPTS + + ", episodeMillis=" + (episodeNanos / 1_000_000L) + + '/' + (catchUpCapGapMinEscalationWindowNanos / 1_000_000L) + ']' + + (exhausted + ? "; the data must be resent after the cap is raised" + : "; retrying -- a larger-cap node may return")); + if (exhausted) { + recordFatal(err); + } + throw new CatchUpSendException(err, true); + } + if (chunkSymbols > 0 && chunkBytes + entryBytes > budget) { + sendCatchUpChunk(chunkStartId, chunkSymbols, chunkStartAddr, (int) chunkBytes); + framesSent++; + chunkStartId += chunkSymbols; + chunkStartAddr = entryStart; + chunkSymbols = 0; + chunkBytes = 0; + } + chunkSymbols++; + chunkBytes += entryBytes; + } + if (chunkSymbols > 0) { + sendCatchUpChunk(chunkStartId, chunkSymbols, chunkStartAddr, (int) chunkBytes); + framesSent++; + } + // The whole dictionary re-registered without a cap gap: this node accepts + // every entry, so the cap-gap episode (if any) is over -- reset BOTH the strike + // count and the wall-clock anchor. Resetting only the count would leave a stale + // anchor from an old episode, so the very first strike of a LATER episode would + // already satisfy the dwell. + resetCatchUpCapGapEpisode(); + return framesSent; + } + + /** + * Sends one table-less catch-up frame carrying dictionary ids + * {@code [deltaStart .. deltaStart+deltaCount)}. Throws {@link + * CatchUpSendException} on a send error instead of calling {@link #fail} + * (see that type for why the catch-up must not re-enter the reconnect loop); + * the caller turns it into a single, non-re-entrant reconnect. + */ + private void sendCatchUpChunk(int deltaStart, int deltaCount, long symbolsAddr, int symbolsLen) { + // Compute the frame size in long and fail loud if it would overflow the int + // size math into a negative Unsafe.malloc. sendDictCatchUp already caps each + // chunk's symbol bytes under the budget, so this is unreachable at real + // cardinality -- but the mirror-side ensureSentDictCapacity guards the same + // math, and a future caller must not be able to overflow this one silently. + long payloadLenL = (long) NativeBufferWriter.varintSize(deltaStart) + + NativeBufferWriter.varintSize(deltaCount) + + symbolsLen; + long frameLenL = QwpConstants.HEADER_SIZE + payloadLenL; + if (frameLenL > MAX_SENT_DICT_BYTES) { + LineSenderException err = new LineSenderException( + "symbol dictionary catch-up frame exceeds the maximum size [" + + "frameLen=" + frameLenL + ", max=" + MAX_SENT_DICT_BYTES + ']'); + recordFatal(err); + throw new CatchUpSendException(err); + } + int payloadLen = (int) payloadLenL; + int prefixLen = QwpConstants.HEADER_SIZE + + NativeBufferWriter.varintSize(deltaStart) + + NativeBufferWriter.varintSize(deltaCount); + ensureCatchUpFrameCapacity(prefixLen); + long frame = catchUpFrameAddr; + try { + Unsafe.getUnsafe().putByte(frame, (byte) 'Q'); + Unsafe.getUnsafe().putByte(frame + 1, (byte) 'W'); + Unsafe.getUnsafe().putByte(frame + 2, (byte) 'P'); + Unsafe.getUnsafe().putByte(frame + 3, (byte) '1'); + Unsafe.getUnsafe().putByte(frame + 4, (byte) client.getServerQwpVersion()); + // FLAG_DEFER_COMMIT: the catch-up carries dictionary entries but NO + // rows, so it must never trigger a server-side commit. Today it is + // always the first frame on a fresh (empty-transaction) connection, so + // committing nothing is a no-op -- but that invariant is load-bearing + // and unasserted. Deferring the (empty) commit removes the dependency: + // a future mid-stream catch-up cannot prematurely commit an in-flight + // deferred transaction. The dictionary delta still registers (as any + // deferred data frame's does); only the row commit is deferred, and the + // next real frame commits it. + Unsafe.getUnsafe().putByte(frame + QwpConstants.HEADER_OFFSET_FLAGS, + (byte) (QwpConstants.FLAG_GORILLA | QwpConstants.FLAG_DELTA_SYMBOL_DICT + | QwpConstants.FLAG_DEFER_COMMIT)); + Unsafe.getUnsafe().putShort(frame + 6, (short) 0); // tableCount + Unsafe.getUnsafe().putInt(frame + 8, payloadLen); + long q = NativeBufferWriter.writeVarint(frame + QwpConstants.HEADER_SIZE, deltaStart); + NativeBufferWriter.writeVarint(q, deltaCount); + client.sendBinary(frame, prefixLen, symbolsAddr, symbolsLen); + } catch (Throwable t) { + // Do NOT fail() here -- see CatchUpSendException. Signal the failure + // up so exactly one non-re-entrant reconnect follows. A JVM Error is + // never a transient reconnect case; let it propagate as-is so the + // I/O loop latches it terminal rather than looping on it. + if (t instanceof Error) { + throw (Error) t; + } + throw new CatchUpSendException(t); + } + nextWireSeq++; // this catch-up chunk consumed a wire sequence + lastFrameOrPingNanos = System.nanoTime(); + totalFramesSent.incrementAndGet(); + } + + @TestOnly + public int catchUpEntryIndexBuildCount() { + return catchUpEntryIndexBuildCount; + } + + @TestOnly + public int catchUpFrameGrowthCount() { + return catchUpFrameGrowthCount; + } + + @TestOnly + public int catchUpCapGapAttempts() { + return catchUpCapGapAttempts; + } + + @TestOnly + public long catchUpCapGapFirstNanos() { + return catchUpCapGapFirstNanos; + } + + @TestOnly + public long catchUpCapGapMinEscalationWindowNanos() { + return catchUpCapGapMinEscalationWindowNanos; + } + + @TestOnly + public long fsnAtZero() { + return fsnAtZero; + } + + @TestOnly + public long poisonFsn() { + return poisonFsn; + } + + @TestOnly + public int poisonStrikes() { + return poisonStrikes; + } + + @TestOnly + public long sentDictBytesAddr() { + return sentDictBytesAddr; + } + + @TestOnly + public int sentDictBytesLen() { + return sentDictBytesLen; + } + + @TestOnly + public boolean sentDictBytesOwned() { + return sentDictBytesOwned; + } + + @TestOnly + public int sentDictCount() { + return sentDictCount; + } + + @TestOnly + public int zeroProgressRecycles() { + return zeroProgressRecycles; + } + + @TestOnly + public void accumulateSentDictForTest(long payloadAddr, int payloadLen, int deltaStart) { + accumulateSentDict(payloadAddr, payloadLen, deltaStart); + } + + @TestOnly + public void connectLoopForTest(Throwable initial, String phase, long paceFirstAttemptMillis) { + connectLoop(initial, phase, paceFirstAttemptMillis); + } + + @TestOnly + public void deliverCloseForTest(int code, String reason) { + responseHandler.onClose(code, reason); + } + + @TestOnly + public void deliverResponseForTest(long payloadPtr, int payloadLen) { + responseHandler.onBinaryMessage(payloadPtr, payloadLen); + } + + @TestOnly + public void ensureSentDictCapacityForTest(long required) { + ensureSentDictCapacity(required); + } + + @TestOnly + public void positionCursorForStartForTest() { + positionCursorForStart(); + } + + @TestOnly + public void seedSentDictMirrorForTest(long addr, int bytes, int symbolCount) { + sentDictBytesAddr = addr; + sentDictBytesCapacity = bytes; + sentDictBytesLen = bytes; + sentDictCount = symbolCount; + sentDictBytesOwned = true; + } + + @TestOnly + public void sendCatchUpChunkForTest(int deltaStart, int deltaCount, long symbolsAddr, int symbolsLen) { + sendCatchUpChunk(deltaStart, deltaCount, symbolsAddr, symbolsLen); + } + + @TestOnly + public void setCatchUpCapGapFirstNanosForTest(long nanos) { + catchUpCapGapFirstNanos = nanos; + } + + @TestOnly + public void setDataFrameSentThisConnectionForTest(boolean value) { + dataFrameSentThisConnection = value; + } + + @TestOnly + public void setFsnAtZeroForTest(long value) { + fsnAtZero = value; + } + + @TestOnly + public void setNextWireSeqForTest(long value) { + nextWireSeq = value; + } + + @TestOnly + public void setRunningForTest(boolean value) { + running = value; + } + + @TestOnly + public void setWireBaselineWithCatchUpForTest(long replayStart) { + setWireBaselineWithCatchUp(replayStart); + } + + @TestOnly + public boolean trySendOneForTest() { + return trySendOne(); + } + + private void ensureCatchUpFrameCapacity(int required) { + if (catchUpFrameCapacity >= required) { + return; + } + long newCapacity = Math.max(required, Math.max(4096L, (long) catchUpFrameCapacity * 2L)); + if (newCapacity > MAX_SENT_DICT_BYTES) { + newCapacity = MAX_SENT_DICT_BYTES; + } + catchUpFrameAddr = Unsafe.realloc( + catchUpFrameAddr, + catchUpFrameCapacity, + (int) newCapacity, + MemoryTag.NATIVE_DEFAULT); + catchUpFrameCapacity = (int) newCapacity; + catchUpFrameGrowthCount++; + } + + private void freeCatchUpFrameBuffer() { + if (catchUpFrameAddr != 0L) { + Unsafe.free(catchUpFrameAddr, catchUpFrameCapacity, MemoryTag.NATIVE_DEFAULT); + catchUpFrameAddr = 0L; + catchUpFrameCapacity = 0; + } + } + private boolean tryReceiveAcks() { boolean any = false; try { @@ -1898,7 +3097,20 @@ private boolean trySendOne() { // Nothing sent on this connection yet: re-anchor in place past // the retired tail. The wireSeq<->FSN mapping is untouched // because no wire sequence has been consumed. - positionCursorForStart(); + try { + positionCursorForStart(); + } catch (CatchUpSendException e) { + // Re-anchor's catch-up send failed. fail() here is a fresh, + // non-re-entrant connectLoop entry from the I/O loop body -- + // the same recovery a normal trySendOne send failure takes. + // Preserve the wrapper on a cap gap so connectLoop's + // isCatchUpCapGap keeps the orphan settle episode (attempt count + // + dwell anchor) alive across the re-anchor recycle; an ordinary + // failure unwraps to the raw cause and restarts the episode, like + // any normal send failure. + fail(isCatchUpCapGap(e) ? e : e.getCause()); + return false; + } return true; } // Frames were already sent on this connection: the linear @@ -1949,12 +3161,55 @@ private boolean trySendOne() { if (frameEnd > pub) { return false; // payload not fully published yet } + long frameAddr = base + sendOffset + MmapSegment.FRAME_HEADER_SIZE; + // Torn-dictionary guard. sentDictCount is this loop's model of how many ids + // the CURRENT server has been told about: seeded from the persisted + // dictionary, re-registered by the catch-up, and extended by + // accumulateSentDict as each frame goes out. A frame whose delta starts ABOVE + // that coverage references ids the server was never given; replaying it would + // make the server null-pad the hole (QwpMessageCursor grows the dict with + // nulls to deltaStart+deltaCount) and land rows with SILENTLY NULL symbols. + // Fail terminally instead; the unreplayable data must be resent. + // + // The guard, the mirror and the catch-up are ONE mechanism and are all + // ungated (see the constructor). A frame at deltaStart == sentDictCount is + // contiguous and extends the coverage, so a slot whose frames start at id 0 + // replays cleanly with no persisted dictionary at all -- which is exactly why + // the mirror below must keep advancing even when the dictionary is missing. + // A gap here therefore means genuine loss: a host/power crash tore the + // unsynced .symbol-dict below the ids the surviving frames still reference + // (SF is process-crash but not host-crash durable). + int deltaStart = frameDeltaStart(frameAddr, payloadLen); + if (deltaStart > sentDictCount) { + recordFatal(new LineSenderException( + "recovered store-and-forward symbol dictionary is incomplete (likely a host crash): " + + "frame delta start " + deltaStart + " exceeds recovered dictionary size " + + sentDictCount + "; cannot replay without corrupting data -- resend required")); + return false; + } try { - client.sendBinary(base + sendOffset + MmapSegment.FRAME_HEADER_SIZE, payloadLen); + client.sendBinary(frameAddr, payloadLen); } catch (Throwable t) { fail(t); return false; } + // A real ring frame (data or commit) has now gone out on this connection, + // as opposed to only the dictionary catch-up. onClose / handleServerRejection + // key their poison-strike vs pre-send decision off this, not off nextWireSeq + // (which the catch-up advances). + dataFrameSentThisConnection = true; + if (deltaStart >= 0) { + // Mirror the symbols this frame just registered on the server, so a later + // reconnect can rebuild the whole dictionary and the guard above keeps an + // accurate view of the server's coverage. Idempotent on replay: a frame + // whose delta we already hold advances nothing. + // + // Ungated (see the constructor): this is the ONLY thing that advances + // sentDictCount from the frames themselves, so gating it while leaving the + // guard ungated froze the coverage at 0 and terminal'd a slot that replays + // perfectly from id 0. + accumulateSentDict(frameAddr, payloadLen, deltaStart); + } lastFrameOrPingNanos = System.nanoTime(); sendOffset = frameEnd; long fsnSent = fsnAtZero + nextWireSeq; @@ -1984,8 +3239,11 @@ void positionCursorForStart() { // starts past it. Zero wire cost, no recycle. tryRetireOrphanTail(); long replayStart = engine.ackedFsn() + 1L; - this.fsnAtZero = replayStart; - this.nextWireSeq = 0L; + // Recovery / orphan-drain seed the dictionary mirror, so the initial + // connection may also need a catch-up (client is non-null in the + // sync-start and drainer paths; null in async-initial, where swapClient + // handles it on the first connect). + setWireBaselineWithCatchUp(replayStart); positionCursorAt(replayStart); } @@ -2011,31 +3269,79 @@ private boolean tryRetireOrphanTail() { if (orphanSkipTipFsn < 0) { return true; } - if (engine.ackedFsn() < orphanSkipStartFsn - 1L) { + if (!engine.retireRecoveredOrphanTailIfReady()) { return false; } - LOG.warn("retiring orphaned deferred tail: {} frame(s) [fsn {}..{}] belong to a transaction " - + "whose commit was never published; aborting them (never transmitted, slots trimmed)", - orphanSkipTipFsn - orphanSkipStartFsn + 1, orphanSkipStartFsn, orphanSkipTipFsn); - engine.acknowledge(orphanSkipTipFsn); orphanSkipStartFsn = -1L; orphanSkipTipFsn = -1L; return true; } + /** + * Determines whether an oversized symbol-dictionary catch-up entry is always + * retriable or may become terminal after the orphan settle budget. + */ + public enum CatchUpCapGapPolicy { + /** Keep reconnecting until a node with a compatible batch cap returns. */ + RETRY_FOREVER, + /** Quarantine an orphan slot after both the attempt and dwell limits expire. */ + TERMINAL_AFTER_SETTLE_BUDGET + } + + /** Identifies who owns data while the reconnect loop is unavailable. */ + public enum ReconnectPolicy { + /** A live producer keeps buffering and retries endpoint-policy failures. */ + FOREGROUND, + /** An orphan drainer returns terminal states to its quarantine owner. */ + ORPHAN + } + /** * Factory used by the I/O loop to build a fresh, connected, upgraded * {@link WebSocketClient} after a wire failure. Implementations close * the old client (if needed), build a new one with the same auth/TLS * config, connect, perform the WebSocket upgrade, and return it ready - * to send. Throw on a terminal failure (auth rejection, etc.) — the - * I/O loop will treat the throw as fatal. + * to send. The loop's {@link ReconnectPolicy} decides whether endpoint-policy + * failures are retried or returned as terminal. */ @FunctionalInterface public interface ReconnectFactory { WebSocketClient reconnect() throws Exception; } + /** + * Signals that a symbol-dictionary catch-up frame could not be sent on the + * current connection. Thrown by {@link #sendDictCatchUp}/{@link + * #sendCatchUpChunk} instead of calling {@link #fail}: the catch-up runs + * inside {@link #connectLoop} (via {@link #swapClient}) and, on the initial + * connect, inside {@link #start()} / {@link #trySendOne} on the caller + * thread. Calling {@code fail()} from there would re-enter {@code + * connectLoop} -- corrupting the {@code fsnAtZero}/{@code nextWireSeq} wire + * mapping (a subsequent ACK then trims un-acked frames) and growing the + * stack until it overflows into a terminal, breaking the "retry a transient + * outage forever" invariant -- or run {@code connectLoop} on the caller + * thread and block {@code Sender} construction indefinitely. Each catch + * site instead turns it into ONE non-re-entrant reconnect: {@code + * connectLoop}'s own retry catch (swapClient path), a fresh {@code fail()} + * from the I/O loop body (trySendOne path), or dropping the dead client so + * the I/O thread reconnects (start path). A JVM {@code Error} is never + * wrapped -- it must stay terminal. The {@code capGap} marker lets the reconnect + * loop preserve only consecutive incompatible-cap observations; ordinary catch-up + * send failures restart the orphan settle episode. + */ + private static final class CatchUpSendException extends RuntimeException { + private final boolean isCapGap; + + CatchUpSendException(Throwable cause) { + this(cause, false); + } + + CatchUpSendException(Throwable cause, boolean isCapGap) { + super(cause); + this.isCapGap = isCapGap; + } + } + /** * One slot in the pendingDurable FIFO. Holds a wireSeq plus the per-table * (name, seqTxn) pairs from its OK frame. Empty entries (tableCount = 0) @@ -2173,7 +3479,7 @@ public void onClose(int code, String reason) { || code == WebSocketCloseCode.GOING_AWAY; LineSenderException cause = new LineSenderException( "WebSocket closed by server: code=" + code + " reason=" + reason); - if (!orderly && nextWireSeq > 0) { + if (!orderly && dataFrameSentThisConnection) { if (recordHeadRejectionStrike(Math.max(engine.ackedFsn(), highestOkFsn) + 1L)) { haltOnPoisonedFrame("ws-close[" + code + ' ' + WebSocketCloseCode.describe(code) + "]: " + reason, @@ -2280,17 +3586,21 @@ private void handleServerRejection(long wireSeq) { // value is only used to attribute an FSN to the error report -- // a rejection never advances the watermark. long highestSent = nextWireSeq - 1L; - if (highestSent < 0L) { - // Pre-send rejection: server emitted an error frame before - // we sent anything on this connection (typical after a - // fresh swapClient — auth failure, server-initiated halt, - // etc.). The server-named wireSeq does not correspond to - // any frame we sent, so clamping it to 0 and acknowledging - // fsnAtZero would silently advance ackedFsn past a real - // unsent batch (fsnAtZero == ackedFsn + 1 right after a - // swap). Skip the watermark advance entirely; still surface - // the error so the user's handler sees it and HALT errors - // remain producer-observable. + if (!dataFrameSentThisConnection) { + // Pre-send rejection: the server emitted an error frame before we + // sent any DATA frame on this connection (typical after a fresh + // swapClient -- auth failure, server-initiated halt, or a rejection + // of the dictionary catch-up itself). nextWireSeq may be > 0 here + // because the catch-up consumed wire sequences, so this keys off + // dataFrameSentThisConnection, not highestSent >= 0 -- otherwise a + // transient NACK of a catch-up frame would take the post-send + // poison-strike path and could escalate a transient outage to a + // terminal. The server-named wireSeq does not correspond to any + // data frame we sent, so clamping it to 0 and acknowledging + // fsnAtZero would silently advance ackedFsn past a real unsent + // batch. Skip the watermark advance entirely; still surface the + // error so the user's handler sees it and HALT errors remain + // producer-observable. handlePreSendRejection(wireSeq, status, category, policy); return; } @@ -2300,6 +3610,32 @@ private void handleServerRejection(long wireSeq) { wireSeq, highestSent); } long fsn = fsnAtZero + cappedSeq; + if (fsn <= engine.ackedFsn()) { + // The clamped wire seq maps at or below the replay head, so this + // NACK is for a dictionary catch-up frame -- which occupies the + // already-acked wire sequences below replayStart -- not a data frame + // this connection sent. (dataFrameSentThisConnection can be true here + // because trySendOne ships the head data frame before tryReceiveAcks + // reads the catch-up's NACK in the same loop iteration.) Attributing + // it a data FSN would key recordHeadRejectionStrike() off a + // below-baseline FSN -- negative when replayStart < catchUpFrames -- + // colliding with the poisonFsn == -1 "no suspect" sentinel and + // laundering a genuine poison run, and would report a bogus FSN. + // Treat it exactly like a pre-send rejection: surface + recycle, no + // poison strike, no watermark advance. Symmetric with the success + // path, where engine.acknowledge() no-ops at or below ackedFsn. A + // real replayed data frame is at fsn > ackedFsn, so it is never + // caught here. + // + // Catch-up frames therefore sit OUTSIDE the poison detector: a + // deterministically-NACKed catch-up recycles forever (paced, so no + // busy-loop). That is acceptable -- a catch-up only re-registers + // symbols the cluster already accepted, so a persistent NACK of one + // is a server bug, not a poison-frame the client can quarantine, and + // Invariant B's "retry a transient outage forever" applies. + handlePreSendRejection(wireSeq, status, category, policy); + return; + } // Best-effort table attribution: the parser populates // response.tableNames on error frames the same way it does on // STATUS_OK. If exactly one table was named, surface it; if diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java index 78e5db9f..406526c4 100644 --- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java +++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/MmapSegment.java @@ -44,7 +44,7 @@ *

* On-disk layout — header and frame format: * 

- *   [u32 magic 'SF01'] [u8 ver=1] [u8 flags=0] [u16 reserved=0]
+ *   [u32 magic 'SF01'] [u8 ver]   [u8 flags=0] [u16 reserved=0]
  *   [u64 baseSeq]       [u64 createdMicros]                       24-byte header
  *   frame, frame, ...                                              each frame:
  *                                                                  [u32 crc32c]
@@ -63,8 +63,8 @@ public final class MmapSegment implements QuietCloseable {
     public static final int FILE_MAGIC = 0x31304653; // 'SF01' little-endian
     public static final int FRAME_HEADER_SIZE = 8;   // u32 crc + u32 payloadLen
     public static final int HEADER_SIZE = 24;
-    public static final byte VERSION = 1;
-    private static final int[] CRC32C_TABLE = buildCrc32cTable();
+    public static final byte LEGACY_VERSION = 1;
+    public static final byte VERSION = 2;
     private static final Logger LOG = LoggerFactory.getLogger(MmapSegment.class);
 
     private final String path;
@@ -74,6 +74,7 @@ public final class MmapSegment implements QuietCloseable {
     // memory-backed segments — same cursor architecture, no disk involvement.
     // close() and msync() branch on this flag.
     private final boolean memoryBacked;
+    private final byte version;
     // appendCursor: written only by the producer thread, never read by anyone else
     // — it's the reservation cursor. Plain field is fine.
     private long appendCursor;
@@ -104,7 +105,7 @@ public final class MmapSegment implements QuietCloseable {
 
     private MmapSegment(String path, int fd, long mmapAddress, long sizeBytes,
                         long baseSeq, long initialCursor, long frameCount,
-                        boolean memoryBacked, long tornTailBytes) {
+                        boolean memoryBacked, long tornTailBytes, byte version) {
         this.path = path;
         this.fd = fd;
         this.mmapAddress = mmapAddress;
@@ -115,6 +116,7 @@ private MmapSegment(String path, int fd, long mmapAddress, long sizeBytes,
         this.frameCount = frameCount;
         this.memoryBacked = memoryBacked;
         this.tornTailBytes = tornTailBytes;
+        this.version = version;
     }
 
     /**
@@ -162,6 +164,59 @@ public static MmapSegment create(FilesFacade ff, String path, long baseSeq, long
      * per-call {@code byte[]} + native-malloc the way the String overload does.
      */
     public static MmapSegment create(FilesFacade ff, long pathPtr, String displayPath, long baseSeq, long sizeBytes) {
+        return create(ff, pathPtr, displayPath, baseSeq, sizeBytes, VERSION);
+    }
+
+    /**
+     * Creates a tiny v1 segment containing one deliberately non-QWP payload.
+     * Two such files with non-contiguous base sequences form the rollback
+     * barrier installed by {@link SegmentRing}: a v1-only reader opens both
+     * and fails its mandatory FSN-contiguity check before replay. If a process
+     * dies after creating only one guard, that reader still adopts a non-empty
+     * log and sends the invalid one-byte head frame first, so it cannot silently
+     * discard the v2 files and start a fresh slot.
+     */
+    static void createLegacyReaderGuard(String path, long baseSeq) {
+        long pathPtr = FilesFacade.INSTANCE.allocNativePath(path);
+        long payload = 0L;
+        try (MmapSegment segment = create(
+                FilesFacade.INSTANCE,
+                pathPtr,
+                path,
+                baseSeq,
+                HEADER_SIZE + FRAME_HEADER_SIZE + 1L,
+                LEGACY_VERSION
+        )) {
+            payload = Unsafe.malloc(1, MemoryTag.NATIVE_DEFAULT);
+            Unsafe.getUnsafe().putByte(payload, (byte) 0);
+            if (segment.tryAppend(payload, 1) != HEADER_SIZE) {
+                throw new MmapSegmentException("could not append legacy-reader guard frame: " + path);
+            }
+            // Unlike the segment log this file guards, the guard IS msync'd. It is a
+            // rollback barrier, so it has to be durable before the data it protects
+            // becomes durable -- a guard still sitting in the page cache when the host
+            // dies leaves zeroes on disk beside written-back v2 segments, and a
+            // rolled-back v1 reader skips the guard (bad magic) and the segments (bad
+            // version) alike, sees an empty slot, and truncates the log. It is 33 bytes
+            // written once per slot lifetime, not per flush, so the sync costs nothing
+            // measurable and buys the barrier the durability its whole purpose assumes.
+            segment.msync();
+        } finally {
+            if (payload != 0L) {
+                Unsafe.free(payload, 1, MemoryTag.NATIVE_DEFAULT);
+            }
+            FilesFacade.INSTANCE.freeNativePath(pathPtr);
+        }
+    }
+
+    private static MmapSegment create(
+            FilesFacade ff,
+            long pathPtr,
+            String displayPath,
+            long baseSeq,
+            long sizeBytes,
+            byte version
+    ) {
         if (sizeBytes < HEADER_SIZE + FRAME_HEADER_SIZE + 1) {
             throw new IllegalArgumentException(
                     "sizeBytes too small for header + one minimal frame: " + sizeBytes);
@@ -192,12 +247,13 @@ public static MmapSegment create(FilesFacade ff, long pathPtr, String displayPat
             }
             // Header goes straight into the mapping — no separate write syscall.
             Unsafe.getUnsafe().putInt(addr, FILE_MAGIC);
-            Unsafe.getUnsafe().putByte(addr + 4, VERSION);
+            Unsafe.getUnsafe().putByte(addr + 4, version);
             Unsafe.getUnsafe().putByte(addr + 5, (byte) 0); // flags
             Unsafe.getUnsafe().putShort(addr + 6, (short) 0); // reserved
             Unsafe.getUnsafe().putLong(addr + 8, baseSeq);
             Unsafe.getUnsafe().putLong(addr + 16, Os.currentTimeMicros());
-            return new MmapSegment(displayPath, fd, addr, sizeBytes, baseSeq, HEADER_SIZE, 0, false, 0L);
+            return new MmapSegment(displayPath, fd, addr, sizeBytes, baseSeq,
+                    HEADER_SIZE, 0, false, 0L, version);
         } catch (Throwable t) {
             if (addr != Files.FAILED_MMAP_ADDRESS) {
                 Files.munmap(addr, sizeBytes, MemoryTag.MMAP_DEFAULT);
@@ -234,7 +290,8 @@ public static MmapSegment createInMemory(long baseSeq, long sizeBytes) {
             Unsafe.getUnsafe().putShort(addr + 6, (short) 0);
             Unsafe.getUnsafe().putLong(addr + 8, baseSeq);
             Unsafe.getUnsafe().putLong(addr + 16, Os.currentTimeMicros());
-            return new MmapSegment(null, -1, addr, sizeBytes, baseSeq, HEADER_SIZE, 0, true, 0L);
+            return new MmapSegment(null, -1, addr, sizeBytes, baseSeq,
+                    HEADER_SIZE, 0, true, 0L, VERSION);
         } catch (Throwable t) {
             Unsafe.free(addr, sizeBytes, MemoryTag.NATIVE_DEFAULT);
             throw t;
@@ -298,7 +355,7 @@ public static MmapSegment openExisting(FilesFacade ff, String path) {
                         "bad magic in " + path + ": 0x" + Integer.toHexString(magic));
             }
             byte version = Unsafe.getUnsafe().getByte(addr + 4);
-            if (version != VERSION) {
+            if (version != LEGACY_VERSION && version != VERSION) {
                 throw new MmapSegmentException("unsupported version in " + path + ": " + version);
             }
             long baseSeq = Unsafe.getUnsafe().getLong(addr + 8);
@@ -313,8 +370,9 @@ public static MmapSegment openExisting(FilesFacade ff, String path) {
                 throw new MmapSegmentException(
                         "bad baseSeq in " + path + ": " + baseSeq);
             }
-            long lastGood = scanFrames(addr, fileSize);
-            long count = countFrames(addr, lastGood);
+            FrameScan scan = scanFrames(addr, fileSize);
+            long lastGood = scan.lastGood;
+            long count = scan.frameCount;
             long tornTail = detectTornTail(addr, lastGood, fileSize);
             if (tornTail > 0) {
                 LOG.warn("SF segment {}: torn tail of {} bytes at offset {} "
@@ -324,7 +382,8 @@ public static MmapSegment openExisting(FilesFacade ff, String path) {
                                 + "Investigate disk health or unexpected writer crash.",
                         path, tornTail, lastGood, fileSize, count);
             }
-            return new MmapSegment(path, fd, addr, fileSize, baseSeq, lastGood, count, false, tornTail);
+            return new MmapSegment(path, fd, addr, fileSize, baseSeq,
+                    lastGood, count, false, tornTail, version);
         } catch (Throwable t) {
             if (addr != Files.FAILED_MMAP_ADDRESS) {
                 Files.munmap(addr, fileSize, MemoryTag.MMAP_DEFAULT);
@@ -438,6 +497,11 @@ public long sizeBytes() {
         return sizeBytes;
     }
 
+    /** On-disk format version read from or written to this segment. */
+    public byte version() {
+        return version;
+    }
+
     /**
      * Appends one frame: writes {@code [crc32c | u32 payloadLen | payload]}
      * starting at the current append cursor, then advances both cursors
@@ -522,6 +586,20 @@ public long findLastFrameFsnWithoutPayloadFlag(int flagsOffset, int flagMask, in
         return best;
     }
 
+    /**
+     * Feeds every recovered frame in this segment to the engine's single-pass
+     * recovery fold. Segments are visited oldest-first by {@link SegmentRing}.
+     */
+    void scanRecovery(RecoveredFrameAnalysis analysis) {
+        long off = HEADER_SIZE;
+        long frames = frameCount;
+        for (long i = 0; i < frames; i++) {
+            int payloadLen = Unsafe.getUnsafe().getInt(mmapAddress + off + 4);
+            analysis.accept(baseSeq + i, mmapAddress + off + FRAME_HEADER_SIZE, payloadLen);
+            off += FRAME_HEADER_SIZE + payloadLen;
+        }
+    }
+
     /**
      * Number of frames written since {@link #create} (or recovered by
      * {@link #openExisting}). Used by {@code SegmentRing} to compute
@@ -581,14 +659,15 @@ private static boolean isMmapAccessFault(Throwable t) {
     }
 
     /**
-     * Forward scan that returns the offset just past the last frame whose
-     * CRC verifies. A torn-tail frame (declared length runs past EOF, or
-     * CRC mismatch) leaves both cursors at the start of that frame; the
-     * next {@link #tryAppend} will overwrite it. The scan only reads from
-     * the mapping — no syscalls.
+     * Forward scan that returns the offset just past the last frame whose CRC
+     * verifies and the number of verified frames. A torn-tail frame (declared
+     * length runs past EOF, or CRC mismatch) leaves both cursors at the start of
+     * that frame; the next {@link #tryAppend} will overwrite it. The scan only
+     * reads from the mapping — no syscalls.
      */
-    private static long scanFrames(long addr, long fileSize) {
+    private static FrameScan scanFrames(long addr, long fileSize) {
         long pos = HEADER_SIZE;
+        long frameCount = 0L;
         try {
             while (pos + FRAME_HEADER_SIZE <= fileSize) {
                 int crcRead = Unsafe.getUnsafe().getInt(addr + pos);
@@ -596,7 +675,7 @@ private static long scanFrames(long addr, long fileSize) {
                 // Defensive: a corrupt length field could be enormous or negative,
                 // both of which would otherwise overrun the mapping.
                 if (payloadLen < 0 || pos + FRAME_HEADER_SIZE + payloadLen > fileSize) {
-                    return pos;
+                    return new FrameScan(pos, frameCount);
                 }
                 // CRC over the contiguous (payloadLen, payload) pair, folded
                 // via Unsafe reads rather than the native Crc32c.update.
@@ -615,13 +694,18 @@ private static long scanFrames(long addr, long fileSize) {
                 // the same scan). Folding over Unsafe keeps every fault
                 // catchable -- handled below as the boundary of recoverable
                 // data; a page that instead reads back as zeroes just fails the
-                // CRC check and ends the scan. Recovery is cold, so the slower
-                // table CRC here is immaterial.
-                int crcCalc = crc32cRecovery(addr + pos + 4, 4L + payloadLen);
+                // CRC check and ends the scan. The shared Java/Unsafe
+                // slice-by-8 path keeps those faults catchable without imposing
+                // a scalar byte-at-a-time scan on large recovery backlogs.
+                int crcCalc = Crc32c.updateUnsafe(
+                        Crc32c.INIT,
+                        addr + pos + 4,
+                        4L + payloadLen);
                 if (crcCalc != crcRead) {
-                    return pos;
+                    return new FrameScan(pos, frameCount);
                 }
                 pos += FRAME_HEADER_SIZE + payloadLen;
+                frameCount++;
             }
         } catch (InternalError e) {
             // The read at `pos` hit a mapped page that is not backed by real
@@ -647,43 +731,7 @@ private static long scanFrames(long addr, long fileSize) {
                             + "if this recurs.",
                     pos, fileSize);
         }
-        return pos;
-    }
-
-    /**
-     * CRC-32C (Castagnoli) of {@code [addr, addr + len)} read through
-     * {@link Unsafe}, seeded like {@code Crc32c.update(Crc32c.INIT, addr, len)}
-     * and bit-identical to it (verified) -- but every byte load is an Unsafe
-     * intrinsic, so a fault on an unbacked mapped page is a catchable
-     * {@link InternalError} instead of the uncatchable JNI SIGBUS the native
-     * {@link Crc32c} would raise. Byte-at-a-time via a precomputed table
-     * ({@code ~0.5 GiB/s}); used only on the cold recovery scan, never on the
-     * append hot path (which stays on the native, hardware-friendly path).
-     */
-    private static int crc32cRecovery(long addr, long len) {
-        int crc = ~Crc32c.INIT;
-        for (long i = 0; i < len; i++) {
-            crc = (crc >>> 8) ^ CRC32C_TABLE[(crc ^ Unsafe.getUnsafe().getByte(addr + i)) & 0xFF];
-        }
-        return ~crc;
-    }
-
-    /**
-     * Standard reflected CRC-32C byte table (polynomial {@code 0x82F63B78}),
-     * matching {@code crc32c_table[0]} in the native {@code crc32c.c}. Computed
-     * at class init to avoid 256 hand-transcribed literals; drives
-     * {@link #crc32cRecovery}.
-     */
-    private static int[] buildCrc32cTable() {
-        int[] table = new int[256];
-        for (int n = 0; n < 256; n++) {
-            int c = n;
-            for (int k = 0; k < 8; k++) {
-                c = (c & 1) != 0 ? (0x82F63B78 ^ (c >>> 1)) : (c >>> 1);
-            }
-            table[n] = c;
-        }
-        return table;
+        return new FrameScan(pos, frameCount);
     }
 
     /**
@@ -723,19 +771,13 @@ private static long detectTornTail(long addr, long lastGood, long fileSize) {
         return 0L;
     }
 
-    /**
-     * Counts frames in {@code [HEADER_SIZE, lastGood)}. Walks the framing in
-     * lockstep with {@link #scanFrames} (which already validated CRCs); so
-     * this is just length-driven traversal, no CRC re-check.
-     */
-    private static long countFrames(long addr, long lastGood) {
-        long pos = HEADER_SIZE;
-        long count = 0;
-        while (pos < lastGood) {
-            int payloadLen = Unsafe.getUnsafe().getInt(addr + pos + 4);
-            pos += FRAME_HEADER_SIZE + payloadLen;
-            count++;
+    private static final class FrameScan {
+        private final long frameCount;
+        private final long lastGood;
+
+        private FrameScan(long lastGood, long frameCount) {
+            this.lastGood = lastGood;
+            this.frameCount = frameCount;
         }
-        return count;
     }
 }
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/OrphanScanner.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/OrphanScanner.java
index 833960ec..49b70674 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/OrphanScanner.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/OrphanScanner.java
@@ -327,7 +327,17 @@ private static boolean hasAnySegmentFile(String slotPath) {
             while (rc > 0) {
                 String name = Files.utf8ToString(Files.findName(find));
                 rc = Files.findNext(find);
-                if (name != null && name.endsWith(".sfa")) {
+                // The legacy-reader barriers are named .sfa on purpose (a rolled-back v1
+                // reader must not skip them), but they are a barrier, not data. Counting
+                // them here would call a slot that never held a byte an orphan with
+                // unacked data: CursorSendEngine plants them FIRST, before recovery or
+                // any segment is created, and its cleanup does not unlink them, so a
+                // construction that fails afterwards (ENOSPC on the initial segment)
+                // leaves a directory holding nothing else. A drainer would then adopt it,
+                // fail its own build under the same disk pressure, and quarantine an
+                // empty slot with a permanent .failed sentinel plus an ERROR for an
+                // operator to chase.
+                if (name != null && name.endsWith(".sfa") && !SegmentRing.isLegacyReaderGuard(name)) {
                     return true;
                 }
             }
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java
new file mode 100644
index 00000000..54ba8f39
--- /dev/null
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/PersistedSymbolDict.java
@@ -0,0 +1,1063 @@
+/*******************************************************************************
+ *     ___                  _   ____  ____
+ *    / _ \ _   _  ___  ___| |_|  _ \| __ )
+ *   | | | | | | |/ _ \/ __| __| | | |  _ \
+ *   | |_| | |_| |  __/\__ \ |_| |_| | |_) |
+ *    \__\_\\__,_|\___||___/\__|____/|____/
+ *
+ *  Copyright (c) 2014-2019 Appsicle
+ *  Copyright (c) 2019-2026 QuestDB
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ ******************************************************************************/
+
+package io.questdb.client.cutlass.qwp.client.sf.cursor;
+
+import io.questdb.client.cutlass.qwp.client.GlobalSymbolDictionary;
+import io.questdb.client.cutlass.qwp.client.NativeBufferWriter;
+import io.questdb.client.std.Crc32c;
+import io.questdb.client.std.Files;
+import io.questdb.client.std.FilesFacade;
+import io.questdb.client.std.MemoryTag;
+import io.questdb.client.std.ObjList;
+import io.questdb.client.std.QuietCloseable;
+import io.questdb.client.std.Unsafe;
+import io.questdb.client.std.str.Utf8s;
+import org.jetbrains.annotations.TestOnly;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Append-only, per-slot persistence of the global symbol dictionary that a
+ * store-and-forward sender ships to the server with delta encoding. Lives at
+ * {@code /.symbol-dict} alongside the segment files, the slot lock and
+ * {@code .ack-watermark}.
+ * 

+ * Delta-encoded SF frames are NOT self-sufficient: a frame carries only the
+ * symbols it introduces, so recovering (process restart) or draining (orphan
+ * adoption) a slot requires re-registering the whole dictionary on the fresh
+ * server before those frames replay. This file is that dictionary. Unlike
+ * {@link AckWatermark} -- a discardable optimization protected by a
+ * {@code max()} clamp -- this file is load-bearing: a surviving frame
+ * that references an id missing from it is unrecoverable. It is therefore held
+ * to a stronger durability contract, and {@link #open} never destroys it (see
+ * "Never recreate over an existing file" below).
+ * 

+ * Layout (little-endian):
+ * 
+ *   offset 0: u32 magic = 'SYD1'
+ *   offset 4: u8  version = 3
+ *   offset 5: 3 bytes reserved (zero)
+ *   offset 8: chunks, each
+ *             [entryCount: varint][entryBytes: varint][entries][crc32c: u32]
+ *             where entries = [len: varint][utf8] repeated entryCount times,
+ *             occupying exactly entryBytes bytes, and the CRC-32C covers the
+ *             two header varints AND the entry region.
+ * 

+ * A chunk is one append -- i.e. exactly the set of symbols one frame
+ * introduces, since the producer persists a frame's new symbols in a single call
+ * before publishing it. Symbol id {@code i} is the {@code i}-th entry across all
+ * chunks (ids are dense and assigned sequentially from 0), so no id is stored.
+ * 

+ * Why the checksum is per chunk, not per entry. The only consumer of the
+ * recovered prefix is the send loop's replay guard, which compares a surviving
+ * frame's {@code deltaStart} against the recovered dictionary size -- and every
+ * {@code deltaStart} is a chunk boundary, because chunks and frame deltas are
+ * written one-for-one. A tear inside a chunk therefore invalidates exactly the
+ * frames a per-entry checksum would have invalidated anyway: per-entry
+ * granularity buys no extra recoverable prefix. It costs a great deal, though.
+ * {@link Crc32c#update} is a native call, so checksumming per entry put one JNI
+ * transition -- plus one sub-cache-line copy and one redundant varint decode --
+ * on the producer thread for every new symbol. On the high-cardinality batch this
+ * feature exists to serve (one new symbol per row), that is a thousand native
+ * calls per flush where the chunk needs one.
+ * 

+ * Durability / write-ahead ordering: the producer appends the symbols a
+ * frame introduces BEFORE that frame is published to the ring, but does NOT
+ * fsync -- matching the rest of store-and-forward, which is page-cache (not
+ * disk) durable. This ordering is sufficient for a process/JVM crash: the
+ * page cache survives, so both the dictionary and the frames survive and the
+ * dictionary is a superset of every recoverable frame's references. It is NOT
+ * sufficient for a host/power crash, where unflushed pages can be lost out
+ * of order and the dictionary may end up torn relative to the frames it serves --
+ * exactly as the segment frames themselves may be lost on a host crash. Two
+ * layers keep a host-crash tear from silently corrupting data:
+ * 

+ * Together these turn every detectable host-crash tear into a fail-clean
+ * "resend required" instead of a silent symbol misattribution -- the same
+ * CRC-32C protection the segment frames carry. A tear that happened to leave a
+ * byte run whose CRC still matches is not distinguished, but that is a 1-in-2^32
+ * collision per corrupted chunk, no weaker than the frames' own checksum.
+ * 

+ * A torn trailing chunk from a crash mid-append is self-healing: {@link #open}
+ * stops parsing at the first incomplete chunk and the next append overwrites it.
+ * 

+ * Never recreate over an existing file. {@link #open} -- the RECOVERY
+ * entry point -- returns {@code null} when an existing file cannot be read or
+ * parsed, and NEVER falls back to recreating it empty. Recreating would mean
+ * {@code O_TRUNC} over the only copy of load-bearing state, so a single transient
+ * read error (an EIO on a flaky disk, a short read) would permanently destroy the
+ * dictionary the surviving delta frames reference -- turning a recoverable outage
+ * into unrecoverable data. A {@code null} instead degrades the sender to full
+ * self-sufficient frames and leaves every byte on disk, so a later attempt, once
+ * the transient clears, can still recover the slot in full. Only
+ * {@link #openClean} -- the FRESH-slot path, where discarding is the whole point
+ * -- truncates.
+ * 

+ * Lifecycle: single-writer (the producer / user thread) for appends. Read
+ * once at {@link #open} to seed in-memory state on recovery or orphan-drain. The
+ * owner (the engine) closes it, and {@code close()} is callable from any thread
+ * (a shutdown hook, test cleanup). {@code close()} and the append methods are
+ * therefore {@code synchronized}: without that, a close racing an in-flight append
+ * could unmap the production append region (or free the fault-test scratch buffer),
+ * close the fd, and let an in-flight write corrupt memory or land on a descriptor
+ * the OS has reused for another file. Not thread-safe for concurrent writers.
+ */
+public final class PersistedSymbolDict implements QuietCloseable {
+
+    /**
+     * Filename within the slot directory. Dot-prefixed so directory
+     * enumerators that filter by the {@code .sfa} suffix (segment recovery,
+     * OrphanScanner, trim) skip it automatically.
+     */
+    public static final String FILE_NAME = ".symbol-dict";
+    static final int CRC_SIZE = 4; // u32 CRC-32C trailing every chunk
+    static final int FILE_MAGIC = 0x31445953; // 'SYD1' little-endian
+    static final int HEADER_SIZE = 8;
+    // One bounded, segment-sized append window avoids the allocate/unmap/mmap
+    // cycle every 64 KiB without geometrically reserving up to 2x a large
+    // dictionary. close() truncates the unused tail back to appendOffset.
+    static final int APPEND_MAP_CAPACITY = 4 * 1024 * 1024;
+    /**
+     * Upper bound on a chunk's two header varints ({@code entryCount} and
+     * {@code entryBytes}): each is at most 5 bytes for a 32-bit value. The
+     * encoders reserve this much in front of the entry region so the header can
+     * be back-filled once the region's exact size is known, keeping header,
+     * entries and CRC one contiguous run.
+     */
+    static final int MAX_CHUNK_HEADER_SIZE = 10;
+    /**
+     * Ceiling for the append scratch buffer, mirroring
+     * {@code CursorWebSocketSendLoop.MAX_SENT_DICT_BYTES}: the capacity math is
+     * int-typed, so a larger buffer cannot be addressed. Exceeding it throws --
+     * {@link #ensureScratch} never silently under-allocates.
+     */
+    static final int MAX_SCRATCH_BYTES = Integer.MAX_VALUE - 8;
+    static final byte VERSION = 3; // v3 moved the CRC-32C from per-entry to per-chunk
+    private static final Logger LOG = LoggerFactory.getLogger(PersistedSymbolDict.class);
+    private final int fd;
+    // Filesystem seam. Production is FilesFacade.INSTANCE (straight to Files);
+    // tests inject a fault facade to exercise recovery I/O failures (a truncate
+    // that cannot drop a torn tail, a short write) without a real broken disk.
+    private final FilesFacade ff;
+    // Production writes directly into segmented append mappings. Wrapping facades retain the
+    // positioned-write path by default so fault tests can inject short writes through ff.write;
+    // mmap-specific fault facades opt in through FilesFacade.isMmapAllowed().
+    private final boolean mappedAppend;
+    // True only when recovery parsed the file through a temporary read-only
+    // mmap instead of allocating a second native buffer as large as the file.
+    // Test-visible so the peak-memory regression has an observable contract.
+    private final boolean mappedRecoveryInput;
+    private long appendMapAddr;
+    private long appendMapCapacity;
+    private long appendMapOffset;
+    private int appendMapGrowthCount;
+    private long appendOffset;
+    private long appendWriteCount;
+    private boolean closed;
+    // In-memory copy of the WIRE entry region [len][utf8]... -- chunk headers and
+    // CRCs stripped -- populated only when open() recovered existing chunks
+    // (recovery / orphan-drain). Zero/empty for a freshly created file. READ (not
+    // consumed) to seed the producer's id map and to seed the send loop's catch-up
+    // mirror. Foreground construction transfers ownership to its sole loop after
+    // producer seeding; orphan-drainer loops borrow it because one engine may create
+    // several wire sessions. If ownership was not transferred, close() frees it.
+    private long loadedEntriesAddr;
+    private int loadedEntriesLen;
+    private long scratchAddr;
+    private int scratchCap;
+    private int size;
+
+    private PersistedSymbolDict(
+            FilesFacade ff,
+            int fd,
+            long appendOffset,
+            int size,
+            long loadedEntriesAddr,
+            int loadedEntriesLen,
+            boolean mappedRecoveryInput
+    ) {
+        this.ff = ff;
+        this.fd = fd;
+        this.mappedAppend = ff.isMmapAllowed();
+        this.mappedRecoveryInput = mappedRecoveryInput;
+        this.appendOffset = appendOffset;
+        this.size = size;
+        this.loadedEntriesAddr = loadedEntriesAddr;
+        this.loadedEntriesLen = loadedEntriesLen;
+    }
+
+    /**
+     * Opens the dictionary file in {@code slotDir} for RECOVERY, creating it only
+     * when it does not already exist. An existing file is parsed and its complete,
+     * CRC-valid chunks are loaded into memory (see {@link #loadedEntriesAddr()}).
+     * 

+     * Returns {@code null} on any I/O or parse failure -- including an existing file
+     * that cannot be read, carries an unknown version, or fails its checksums. The
+     * caller then falls back to full-dictionary (self-sufficient) frames for this
+     * slot, so a broken side-file degrades gracefully rather than aborting the
+     * sender. Crucially, a {@code null} return NEVER destroys the file: see the
+     * class-level "Never recreate over an existing file" note.
+     */
+    public static PersistedSymbolDict open(String slotDir) {
+        return open(FilesFacade.INSTANCE, slotDir);
+    }
+
+    /**
+     * Facade-aware variant of {@link #open(String)}. Production passes
+     * {@link FilesFacade#INSTANCE}; tests inject a fault facade to drive recovery
+     * I/O failures (e.g. a truncate that cannot drop a torn tail).
+     */
+    public static PersistedSymbolDict open(FilesFacade ff, String slotDir) {
+        String filePath = slotDir + "/" + FILE_NAME;
+        boolean exists = ff.exists(filePath);
+        long existing = exists ? ff.length(filePath) : -1L;
+        if (exists && existing < 0) {
+            // The file is present but its length could not be stat'd (a transient EIO
+            // on a flaky disk). Do NOT fall through to openFresh below, which O_TRUNCs:
+            // truncating the only copy of load-bearing state on a TRANSIENT fault is the
+            // exact destruction the class-level "Never recreate over an existing file"
+            // note forbids -- and unlike the openExisting read path, this routing check
+            // otherwise has no guard. A genuine sub-header stub reports a length in
+            // [0, HEADER_SIZE); only a stat error reports < 0, so the two are
+            // distinguishable. Degrade to full self-sufficient frames and leave every
+            // byte on disk for a later attempt, once the transient clears.
+            LOG.warn("symbol dict {} exists but its length could not be read; "
+                    + "falling back to full-dictionary frames (file left intact)", filePath);
+            return null;
+        }
+        if (existing >= HEADER_SIZE) {
+            // Chunk lengths and the retained contiguous entry region are int-sized,
+            // so a dictionary at or past Integer.MAX_VALUE cannot be represented
+            // safely even though production recovery maps rather than reads it.
+            if (existing >= Integer.MAX_VALUE) {
+                LOG.warn("symbol dict {} too large ({} bytes) to reopen; "
+                        + "falling back to full-dictionary frames (file left intact)", filePath, existing);
+                return null;
+            }
+            // NEVER recreate over an existing file on the recovery path: openFresh
+            // truncates, and these bytes are the only copy of state the surviving
+            // delta frames reference. A null degrades this slot to full
+            // self-sufficient frames and preserves the file for a later attempt.
+            return openExisting(ff, filePath, existing);
+        }
+        // Absent, or a sub-header stub left by a crash inside openFresh: no
+        // load-bearing content to lose, so create it.
+        return openFresh(ff, filePath);
+    }
+
+    /**
+     * Opens the dictionary in {@code slotDir} as a FRESH, EMPTY file, discarding
+     * any surviving content. This is the fresh-start counterpart to {@link #open}:
+     * a slot with no recovered segments must start with an empty dictionary, so a
+     * dictionary left by a prior lifecycle -- a fully-drained slot whose
+     * best-effort delete failed, or a crash in the close window -- must NOT be
+     * inherited. Unlike {@link #open}, which parses and TRUSTS an existing file for
+     * recovery/orphan-drain replay and never destroys it, this truncates: the
+     * fresh-start producer is not seeded from the dictionary, so trusting a survivor
+     * would leave the producer's ids diverged from the dictionary the send loop
+     * replays and silently misattribute symbols on the next reconnect. Truncating
+     * (rather than relying on an unlink succeeding first) closes the gap even when
+     * the delete is refused -- e.g. a Windows share lock. Returns {@code null} on
+     * I/O failure, so the caller falls back to full self-sufficient frames exactly
+     * as {@link #open} does.
+     */
+    public static PersistedSymbolDict openClean(String slotDir) {
+        return openClean(FilesFacade.INSTANCE, slotDir);
+    }
+
+    /**
+     * Facade-aware variant of {@link #openClean(String)}.
+     */
+    public static PersistedSymbolDict openClean(FilesFacade ff, String slotDir) {
+        return openFresh(ff, slotDir + "/" + FILE_NAME);
+    }
+
+    /**
+     * Best-effort removal of a stale dictionary file. Used at fully-drained close
+     * (the slot is empty, nothing references the dictionary any more), mirroring
+     * {@link AckWatermark#removeOrphan}. The fresh-start path deliberately does NOT
+     * use this -- it opens a clean dictionary via {@link #openClean} instead, so a
+     * failed delete cannot leave a stale dictionary a new session would trust.
+     */
+    public static void removeOrphan(String slotDir) {
+        removeOrphan(FilesFacade.INSTANCE, slotDir);
+    }
+
+    /**
+     * Facade-aware variant of {@link #removeOrphan(String)}.
+     */
+    public static void removeOrphan(FilesFacade ff, String slotDir) {
+        ff.remove(slotDir + "/" + FILE_NAME);
+    }
+
+    /**
+     * Appends {@code count} wire entries -- {@code [len varint][utf8]...}, the
+     * symbol-dict delta section the frame encoder just wrote -- as ONE chunk.
+     * 

+     * The consistency walk below decodes each entry's length varint, but the bytes
+     * themselves are copied in a SINGLE {@code copyMemory} and checksummed by a
+     * SINGLE {@link Crc32c#update} covering the whole chunk. A per-entry checksum
+     * would put one JNI transition, one sub-cache-line copy and one redundant varint
+     * decode on the producer thread per new symbol; the chunk needs one of each.
+     * 

+     * Advances {@code size} by {@code count}. Same durability/idempotency contract
+     * as {@link #appendSymbols}: no fsync, and a short write throws WITHOUT
+     * advancing {@code size}/{@code appendOffset}, so a retry keyed off
+     * {@link #size()} re-persists the same range at the same offset. No-op when the
+     * range is empty or the dictionary is closed.
+     */
+    public synchronized void appendRawEntries(long addr, int len, int count) {
+        if (closed || count <= 0 || len <= 0) {
+            return;
+        }
+        // Validate the (addr, len, count) triple BEFORE writing anything: an
+        // inconsistent triple would record a chunk whose stored entryCount disagreed
+        // with the entries it holds, shifting the dense id->symbol map on recovery.
+        // The sole caller derives count and len from one beginMessage, so this cannot
+        // fire today -- but the file this guards is the one the "resend required"
+        // contract rests on, so keep the structural guard. Gated behind assert: it
+        // re-walks every entry's length prefix on the common flush path, and the client
+        // library runs without -ea in production (embedded in user apps), so this holds
+        // the guard in the client's own -ea test suite without the per-flush cost in
+        // production.
+        assert validateRawEntries(addr, len, count);
+        int hdrLen = NativeBufferWriter.varintSize(count) + NativeBufferWriter.varintSize(len);
+        if (mappedAppend) {
+            long recLen = (long) hdrLen + len + CRC_SIZE;
+            ensureAppendMap(checkedRequiredOffset(recLen));
+            long recStart = appendMapAddr + appendOffset - appendMapOffset;
+            long p = NativeBufferWriter.writeVarint(recStart, count);
+            NativeBufferWriter.writeVarint(p, len);
+            Unsafe.getUnsafe().copyMemory(addr, recStart + hdrLen, len);
+            commitMappedChunk(recStart, hdrLen, len, count);
+            return;
+        }
+        ensureScratch((long) hdrLen + len + CRC_SIZE);
+        long p = NativeBufferWriter.writeVarint(scratchAddr, count);
+        NativeBufferWriter.writeVarint(p, len);
+        Unsafe.getUnsafe().copyMemory(addr, scratchAddr + hdrLen, len);
+        flushChunk(scratchAddr, hdrLen, len, count);
+    }
+
+    /**
+     * Appends one symbol as a single-entry chunk, extending the on-disk dictionary.
+     * The caller appends a frame's new symbols BEFORE publishing that frame, so the
+     * write ordering (dictionary entry before referencing frame) holds; no fsync is
+     * performed (see the class-level durability note). Assigns the next dense id
+     * implicitly (the entry's position).
+     * 

+     * Test-only: production persists a frame's whole new-symbol range in one chunk
+     * via {@link #appendSymbols} / {@link #appendRawEntries}. Tests use this to
+     * build a dictionary one entry at a time.
+     */
+    @TestOnly
+    public synchronized void appendSymbol(CharSequence symbol) {
+        if (closed) {
+            return;
+        }
+        int utf8Len = Utf8s.utf8Bytes(symbol);
+        int wireLen = NativeBufferWriter.varintSize(utf8Len) + utf8Len; // [len][utf8]
+        if (mappedAppend) {
+            int hdrLen = NativeBufferWriter.varintSize(1) + NativeBufferWriter.varintSize(wireLen);
+            long recLen = (long) hdrLen + wireLen + CRC_SIZE;
+            ensureAppendMap(checkedRequiredOffset(recLen));
+            long recStart = appendMapAddr + appendOffset - appendMapOffset;
+            long p = NativeBufferWriter.writeVarint(recStart, 1);
+            p = NativeBufferWriter.writeVarint(p, wireLen);
+            p = NativeBufferWriter.writeVarint(p, utf8Len);
+            if (utf8Len > 0) {
+                Utf8s.strCpyUtf8(symbol, p, utf8Len);
+            }
+            commitMappedChunk(recStart, hdrLen, wireLen, 1);
+            return;
+        }
+        ensureScratch((long) MAX_CHUNK_HEADER_SIZE + wireLen + CRC_SIZE);
+        long entryStart = scratchAddr + MAX_CHUNK_HEADER_SIZE;
+        long p = NativeBufferWriter.writeVarint(entryStart, utf8Len);
+        if (utf8Len > 0) {
+            Utf8s.strCpyUtf8(symbol, p, utf8Len);
+        }
+        writeChunkFromScratch(wireLen, 1);
+    }
+
+    /**
+     * Appends the dense id range {@code [from .. to]} as one chunk with one
+     * checksum. This is the RE-ENCODE path: the steady-state persist ships a frame's
+     * pre-encoded delta bytes through {@link #appendRawEntries}, and only a retry
+     * after a failed publish rebuilds a range straight from the dictionary. The
+     * mapped path stages the entry region in the scratch buffer with a single UTF-8
+     * walk per symbol, then bulk-copies it into the append window after the header,
+     * and still commits WITHOUT a positioned-write syscall. A direct encode into the
+     * window would walk each symbol's UTF-8 length twice -- the exact entriesLen sizes
+     * both the header varint and the mmap reserve -- and the back-to-back chunk format
+     * leaves no room to reserve-and-back-fill the header in place. The
+     * positioned-write fallback runs only behind an injected filesystem facade so
+     * short-write recovery tests retain their fault seam. Callers pass the dictionary
+     * and the range so the ids resolve to their symbol strings.
+     * 

+     * Same durability and idempotency contract as {@link #appendSymbol}: no fsync,
+     * and a short write throws WITHOUT advancing {@code size}/{@code appendOffset},
+     * so a retry keyed off {@link #size()} re-encodes the same range and overwrites
+     * at the same offset. No-op when the range is empty or the dictionary is closed.
+     */
+    public synchronized void appendSymbols(GlobalSymbolDictionary dict, int from, int to) {
+        if (closed || to < from) {
+            return;
+        }
+        int count = to - from + 1;
+        if (mappedAppend) {
+            // Stage the entry region in scratch with ONE UTF-8 walk per symbol, then
+            // bulk-copy it into the append window after the header (see the method
+            // javadoc for why a direct in-window encode would have to walk each symbol
+            // twice). ensureScratch enforces the same MAX_SCRATCH_BYTES ceiling the old
+            // sizing pass did, throwing before size/appendOffset advance.
+            int entriesLen = 0;
+            for (int id = from; id <= to; id++) {
+                CharSequence symbol = dict.getSymbol(id);
+                int utf8Len = Utf8s.utf8Bytes(symbol);
+                int wireLen = NativeBufferWriter.varintSize(utf8Len) + utf8Len; // [len][utf8]
+                ensureScratch((long) entriesLen + wireLen);
+                long q = NativeBufferWriter.writeVarint(scratchAddr + entriesLen, utf8Len);
+                if (utf8Len > 0) {
+                    Utf8s.strCpyUtf8(symbol, q, utf8Len);
+                }
+                entriesLen += wireLen;
+            }
+            int hdrLen = NativeBufferWriter.varintSize(count)
+                    + NativeBufferWriter.varintSize(entriesLen);
+            long recLen = (long) hdrLen + entriesLen + CRC_SIZE;
+            ensureAppendMap(checkedRequiredOffset(recLen));
+            long recStart = appendMapAddr + appendOffset - appendMapOffset;
+            long p = NativeBufferWriter.writeVarint(recStart, count);
+            NativeBufferWriter.writeVarint(p, entriesLen);
+            if (entriesLen > 0) {
+                Unsafe.getUnsafe().copyMemory(scratchAddr, recStart + hdrLen, entriesLen);
+            }
+            commitMappedChunk(recStart, hdrLen, entriesLen, count);
+            return;
+        }
+        int entriesLen = 0;
+        for (int id = from; id <= to; id++) {
+            CharSequence symbol = dict.getSymbol(id);
+            int utf8Len = Utf8s.utf8Bytes(symbol);
+            int wireLen = NativeBufferWriter.varintSize(utf8Len) + utf8Len; // [len][utf8]
+            ensureScratch((long) MAX_CHUNK_HEADER_SIZE + entriesLen + wireLen + CRC_SIZE);
+            long entryStart = scratchAddr + MAX_CHUNK_HEADER_SIZE + entriesLen;
+            long p = NativeBufferWriter.writeVarint(entryStart, utf8Len);
+            if (utf8Len > 0) {
+                Utf8s.strCpyUtf8(symbol, p, utf8Len);
+            }
+            entriesLen += wireLen;
+        }
+        writeChunkFromScratch(entriesLen, count);
+    }
+
+    @Override
+    public synchronized void close() {
+        if (closed) {
+            return;
+        }
+        closed = true;
+        if (loadedEntriesAddr != 0L) {
+            Unsafe.free(loadedEntriesAddr, loadedEntriesLen, MemoryTag.NATIVE_DEFAULT);
+            // Null after freeing (like scratchAddr below) so a future accessor that
+            // reads loadedEntriesAddr()/loadedEntriesLen() post-close cannot
+            // dereference freed native memory; the getters are not closed-guarded.
+            loadedEntriesAddr = 0L;
+            loadedEntriesLen = 0;
+        }
+        if (appendMapAddr != 0L) {
+            ff.munmap(appendMapAddr, appendMapCapacity, MemoryTag.MMAP_DEFAULT);
+            appendMapAddr = 0L;
+            appendMapCapacity = 0L;
+            appendMapOffset = 0L;
+            // The active window reserves space past the logical end. Return that tail on
+            // orderly close; after a crash open() treats the zero-filled reserve as
+            // a torn trailing chunk and truncates it to the same appendOffset.
+            if (!ff.truncate(fd, appendOffset)) {
+                LOG.warn("symbol dict {} could not trim mmap reserve to {}; recovery will "
+                                + "discard the zero-filled tail on the next open",
+                        FILE_NAME, appendOffset);
+            }
+        }
+        if (scratchAddr != 0L) {
+            Unsafe.free(scratchAddr, scratchCap, MemoryTag.NATIVE_DEFAULT);
+            scratchAddr = 0L;
+            scratchCap = 0;
+        }
+        if (fd >= 0) {
+            ff.close(fd);
+        }
+    }
+
+    @TestOnly
+    public synchronized int appendMapGrowthCount() {
+        return appendMapGrowthCount;
+    }
+
+    @TestOnly
+    public synchronized long appendWriteCount() {
+        return appendWriteCount;
+    }
+
+    /**
+     * Base address of the loaded entry region -- the concatenated
+     * {@code [len][utf8]} bytes of every recovered symbol in id order, exactly as a
+     * delta section carries them (chunk headers and CRCs stripped). Zero when
+     * nothing was recovered.
+     * 

+     * Construction-phase only. This hands out a raw pointer into native
+     * memory that {@link #close()} frees and nulls, with no closed-guard and no
+     * synchronization. It is safe to read only BEFORE the slot's I/O thread and
+     * any producer append start -- i.e. while the send loop is being constructed
+     * or an orphan-drain is seeding its mirror, both of which happen-before those
+     * threads. A caller that reads it from a running thread races {@code close()}
+     * and can dereference freed memory (use-after-free).
+     */
+    public long loadedEntriesAddr() {
+        return loadedEntriesAddr;
+    }
+
+    /**
+     * Byte length of {@link #loadedEntriesAddr()}. Construction-phase only, for
+     * the same reason -- see {@link #loadedEntriesAddr()}.
+     */
+    public int loadedEntriesLen() {
+        return loadedEntriesLen;
+    }
+
+    /**
+     * Transfers the recovered entry buffer to the foreground send loop. The
+     * symbol count remains unchanged because it is also the append baseline;
+     * only the native-buffer ownership moves. Construction-phase only.
+     */
+    synchronized long takeLoadedEntries() {
+        long addr = loadedEntriesAddr;
+        loadedEntriesAddr = 0L;
+        loadedEntriesLen = 0;
+        return addr;
+    }
+
+    @TestOnly
+    public boolean usedMappedRecoveryInput() {
+        return mappedRecoveryInput;
+    }
+
+    /**
+     * Decodes the recovered entries directly into {@code target} in ascending-id
+     * order. This avoids materialising a cardinality-sized temporary list that
+     * the producer would immediately copy into the global dictionary.
+     * Construction-phase only; see {@link #loadedEntriesAddr()}.
+     */
+    public void addLoadedSymbolsTo(GlobalSymbolDictionary target) {
+        decodeLoadedSymbols(target, null);
+    }
+
+    /**
+     * Materialises the loaded entries as symbol strings in ascending-id order.
+     * Retained for recovery-format tests; production decodes directly through
+     * {@link #addLoadedSymbolsTo(GlobalSymbolDictionary)}.
+     */
+    @TestOnly
+    public ObjList readLoadedSymbols() {
+        ObjList out = new ObjList<>(Math.max(size, 1));
+        decodeLoadedSymbols(null, out);
+        return out;
+    }
+
+    private void decodeLoadedSymbols(GlobalSymbolDictionary target, ObjList out) {
+        long p = loadedEntriesAddr;
+        long limit = p + loadedEntriesLen;
+        for (int i = 0; i < size && p < limit; i++) {
+            long len = 0;
+            int shift = 0;
+            while (p < limit) {
+                byte b = Unsafe.getUnsafe().getByte(p++);
+                len |= (long) (b & 0x7F) << shift;
+                if ((b & 0x80) == 0) {
+                    break;
+                }
+                shift += 7;
+                if (shift > 35) {
+                    // Bound the varint like the other decoders: a canonical length is
+                    // <= 5 bytes. open() already CRC-validated these bytes, so this is
+                    // defensive only; the p + len > limit check below rejects the
+                    // over-long run.
+                    break;
+                }
+            }
+            if (p + len > limit) {
+                break; // defensive: torn tail (should not happen past parse in open)
+            }
+            String symbol = Utf8s.stringFromUtf8Bytes(p, p + len);
+            if (target != null) {
+                target.addRecoveredSymbol(symbol);
+            } else {
+                out.add(symbol);
+            }
+            p += len;
+        }
+    }
+
+    /**
+     * Number of symbols the dictionary holds (highest id + 1).
+     */
+    public int size() {
+        return size;
+    }
+
+    private static PersistedSymbolDict openExisting(FilesFacade ff, String filePath, long fileLen) {
+        int fd = ff.openRW(filePath);
+        if (fd < 0) {
+            LOG.warn("symbol dict {} could not be opened (rc={}); "
+                    + "falling back to full-dictionary frames (file left intact)", filePath, fd);
+            return null;
+        }
+        int len = (int) fileLen; // open() bounds fileLen to [HEADER_SIZE, Integer.MAX_VALUE)
+        boolean mappedInput = ff.isMmapAllowed();
+        long inputAddr = 0L;
+        long entriesAddr = 0L;
+        int entriesLen = 0;
+        try {
+            if (mappedInput) {
+                inputAddr = ff.mmap(fd, fileLen, 0L, Files.MAP_RO, MemoryTag.MMAP_DEFAULT);
+                if (inputAddr == Files.FAILED_MMAP_ADDRESS) {
+                    inputAddr = 0L;
+                    throw new IllegalStateException("could not mmap symbol dictionary for recovery");
+                }
+            } else {
+                // Fault-injection facades retain the positioned-read path so tests
+                // can still force short reads without mapping around the seam.
+                inputAddr = Unsafe.malloc(len, MemoryTag.NATIVE_DEFAULT);
+                long read = ff.read(fd, inputAddr, len, 0);
+                if (read != len) {
+                    throw new IllegalStateException("short read while recovering symbol dictionary "
+                            + "[expected=" + len + ", actual=" + read + ']');
+                }
+            }
+            if (Unsafe.getUnsafe().getInt(inputAddr) != FILE_MAGIC
+                    || Unsafe.getUnsafe().getByte(inputAddr + 4) != VERSION) {
+                throw new IllegalStateException("bad magic or unknown symbol dictionary version");
+            }
+            // First pass validates chunks and totals only the entry bytes that
+            // survive. Production reads through a file-backed mapping, so this
+            // pass does not allocate an anonymous whole-file copy. The second
+            // pass copies directly into one exact-sized retained allocation.
+            RecoveryScan scan = scanRecoveredChunks(inputAddr, len);
+            entriesLen = scan.entriesLen;
+            if (entriesLen > 0) {
+                entriesAddr = Unsafe.malloc(entriesLen, MemoryTag.NATIVE_DEFAULT);
+                copyRecoveredEntries(inputAddr, scan.validLen, entriesAddr, entriesLen);
+            }
+            if (mappedInput) {
+                ff.munmap(inputAddr, fileLen, MemoryTag.MMAP_DEFAULT);
+            } else {
+                Unsafe.free(inputAddr, len, MemoryTag.NATIVE_DEFAULT);
+            }
+            inputAddr = 0L;
+            // Drop any torn/stale trailing bytes so a LATER, shorter append cannot
+            // leave residue past its own end. The truncate result IS checked: a file
+            // we cannot trim could still expose stale post-end bytes whose
+            // (self-consistent) chunk CRC the parse would accept at a shifted
+            // position, so a failed truncate makes the file untrusted -- return null
+            // (the sender falls back to full self-sufficient frames) and, per the
+            // never-destroy contract, leave every byte on disk.
+            if (scan.validLen < len && !ff.truncate(fd, scan.validLen)) {
+                throw new IllegalStateException("could not drop torn/stale symbol dictionary tail");
+            }
+            return new PersistedSymbolDict(
+                    ff, fd, scan.validLen, scan.count, entriesAddr, entriesLen, mappedInput);
+        } catch (Throwable t) {
+            if (inputAddr != 0L) {
+                if (mappedInput) {
+                    ff.munmap(inputAddr, fileLen, MemoryTag.MMAP_DEFAULT);
+                } else {
+                    Unsafe.free(inputAddr, len, MemoryTag.NATIVE_DEFAULT);
+                }
+            }
+            if (entriesAddr != 0L) {
+                Unsafe.free(entriesAddr, entriesLen, MemoryTag.NATIVE_DEFAULT);
+            }
+            if (fd >= 0) {
+                ff.close(fd);
+            }
+            // Pass the throwable as a trailing argument with no matching placeholder so
+            // slf4j prints the stack trace: this WARN is the only forensic record of why
+            // a load-bearing dictionary was abandoned.
+            LOG.warn("symbol dict {} recovery failed; falling back to full-dictionary frames "
+                    + "(file left intact)", filePath, t);
+            return null;
+        }
+    }
+
+    private static void copyRecoveredEntries(long inputAddr, int validLen, long entriesAddr, int entriesLen) {
+        Varint v = new Varint();
+        int diskPos = HEADER_SIZE;
+        int dstPos = 0;
+        while (diskPos < validLen) {
+            if (!v.decode(inputAddr, diskPos, validLen)
+                    || !v.decode(inputAddr, v.end, validLen)) {
+                throw new IllegalStateException("validated symbol dictionary chunk could not be decoded");
+            }
+            int entryBytes = (int) v.value;
+            int entriesStart = v.end;
+            Unsafe.getUnsafe().copyMemory(inputAddr + entriesStart, entriesAddr + dstPos, entryBytes);
+            dstPos += entryBytes;
+            diskPos = entriesStart + entryBytes + CRC_SIZE;
+        }
+        assert dstPos == entriesLen;
+    }
+
+    private static RecoveryScan scanRecoveredChunks(long inputAddr, int len) {
+        Varint v = new Varint();
+        int count = 0;
+        int diskPos = HEADER_SIZE;
+        long entriesLen = 0L;
+        while (diskPos < len) {
+            if (!v.decode(inputAddr, diskPos, len)) {
+                break;
+            }
+            long entryCount = v.value;
+            if (!v.decode(inputAddr, v.end, len)) {
+                break;
+            }
+            long entryBytes = v.value;
+            int entriesStart = v.end;
+            long chunkEnd = (long) entriesStart + entryBytes;
+            if (chunkEnd + CRC_SIZE > len) {
+                break;
+            }
+            int chunkEndI = (int) chunkEnd;
+            int crcStored = Unsafe.getUnsafe().getInt(inputAddr + chunkEndI);
+            int crcCalc = Crc32c.updateUnsafe(
+                    Crc32c.INIT,
+                    inputAddr + diskPos,
+                    chunkEndI - diskPos);
+            if (crcCalc != crcStored
+                    || entryCount <= 0
+                    || (long) count + entryCount > Integer.MAX_VALUE
+                    || entriesLen + entryBytes > Integer.MAX_VALUE) {
+                break;
+            }
+            entriesLen += entryBytes;
+            diskPos = chunkEndI + CRC_SIZE;
+            count += (int) entryCount;
+        }
+        return new RecoveryScan(count, (int) entriesLen, diskPos);
+    }
+
+    private static PersistedSymbolDict openFresh(FilesFacade ff, String filePath) {
+        int fd = ff.openCleanRW(filePath);
+        if (fd < 0) {
+            LOG.warn("symbol dict {} could not be created (rc={}); proceeding without it", filePath, fd);
+            return null;
+        }
+        long hdr = 0L;
+        try {
+            hdr = Unsafe.malloc(HEADER_SIZE, MemoryTag.NATIVE_DEFAULT);
+            Unsafe.getUnsafe().putInt(hdr, FILE_MAGIC);
+            Unsafe.getUnsafe().putByte(hdr + 4, VERSION);
+            Unsafe.getUnsafe().putByte(hdr + 5, (byte) 0);
+            Unsafe.getUnsafe().putByte(hdr + 6, (byte) 0);
+            Unsafe.getUnsafe().putByte(hdr + 7, (byte) 0);
+            long written = ff.write(fd, hdr, HEADER_SIZE, 0);
+            if (written != HEADER_SIZE) {
+                int fdToClose = fd;
+                fd = -1; // relinquish before close so the catch cannot double-close if close throws
+                ff.close(fdToClose);
+                ff.remove(filePath); // drop the headerless stub rather than litter
+                LOG.warn("symbol dict {} header write failed; proceeding without it", filePath);
+                return null;
+            }
+        } catch (Throwable t) {
+            // Unreachable with FilesFacade.INSTANCE (Files.write is native and returns
+            // -1 rather than throwing; the Unsafe puts target a valid 8-byte buffer and
+            // an 8-byte malloc cannot realistically OOM), but the ff seam exists so
+            // tests CAN inject a throwing facade -- close the fd and drop the stub so
+            // neither leaks.
+            if (fd >= 0) { // the header-write branch relinquished fd to -1 before closing
+                int fdToClose = fd;
+                fd = -1;
+                ff.close(fdToClose);
+                ff.remove(filePath);
+            }
+            LOG.warn("symbol dict {} creation failed; proceeding without it", filePath, t);
+            return null;
+        } finally {
+            if (hdr != 0L) {
+                Unsafe.free(hdr, HEADER_SIZE, MemoryTag.NATIVE_DEFAULT);
+            }
+        }
+        return new PersistedSymbolDict(ff, fd, HEADER_SIZE, 0, 0L, 0, false);
+    }
+
+    /**
+     * Verifies that {@code count} wire entries ({@code [len varint][utf8]}) occupy
+     * exactly {@code len} bytes from {@code addr}. Returns {@code true} when the triple
+     * is consistent; throws {@link IllegalStateException} (naming the offending entry /
+     * count / consumed bytes) otherwise. Called only from an {@code assert} in
+     * {@link #appendRawEntries}: it guards an internal invariant the sole caller cannot
+     * violate today, so it runs under the test suite's {@code -ea} but is elided from
+     * the per-flush production path (client apps run without {@code -ea}).
+     */
+    private static boolean validateRawEntries(long addr, int len, int count) {
+        long src = addr;
+        long srcLimit = addr + len;
+        for (int i = 0; i < count; i++) {
+            long symLen = 0;
+            int shift = 0;
+            while (src < srcLimit) {
+                byte b = Unsafe.getUnsafe().getByte(src++);
+                symLen |= (long) (b & 0x7F) << shift;
+                if ((b & 0x80) == 0) {
+                    break;
+                }
+                shift += 7;
+                if (shift > 35) {
+                    // A canonical entry-length varint is <= 5 bytes; a longer
+                    // continuation run is corrupt. The bound check below rejects it.
+                    break;
+                }
+            }
+            src += symLen; // src was just past the len varint
+            if (src > srcLimit) {
+                throw new IllegalStateException("malformed raw symbol-dict entries to " + FILE_NAME
+                        + " [entry=" + i + ", count=" + count + ']');
+            }
+        }
+        if (src != srcLimit) {
+            throw new IllegalStateException("raw symbol-dict entries under-filled the buffer to "
+                    + FILE_NAME + " [count=" + count + ", len=" + len
+                    + ", consumed=" + (int) (src - addr) + ']');
+        }
+        return true;
+    }
+
+    private long checkedRequiredOffset(long recordLen) {
+        long required = appendOffset + recordLen;
+        if (recordLen < 0L || required < appendOffset) {
+            throw new IllegalStateException("symbol dict append offset overflow to "
+                    + FILE_NAME + " [offset=" + appendOffset + ", recordLen=" + recordLen + ']');
+        }
+        return required;
+    }
+
+    /**
+     * Commits a chunk already assembled directly in the append mmap. The logical
+     * offset and symbol count advance last, after the CRC and a store fence, so an
+     * interrupted process leaves either a complete chunk or a tail that open()
+     * rejects and truncates.
+     */
+    private void commitMappedChunk(long recStart, int hdrLen, int entriesLen, int count) {
+        long bodyLen = (long) hdrLen + entriesLen;
+        long recLen = bodyLen + CRC_SIZE;
+        Unsafe.getUnsafe().putInt(
+                recStart + bodyLen,
+                Crc32c.update(Crc32c.INIT, recStart, bodyLen));
+        Unsafe.getUnsafe().storeFence();
+        appendOffset += recLen;
+        size += count;
+    }
+
+    /**
+     * Ensures the production append mmap covers the absolute file offset
+     * {@code required}. The log is mapped in 4 MiB-aligned windows: small flushes
+     * share a segment-sized window, while a large existing dictionary does not force
+     * the process to reserve and remap its whole prefix or geometrically over-allocate it.
+     */
+    private void ensureAppendMap(long required) {
+        if (appendMapAddr != 0L
+                && required >= appendMapOffset
+                && required <= appendMapOffset + appendMapCapacity) {
+            return;
+        }
+        long newOffset = appendOffset - appendOffset % APPEND_MAP_CAPACITY;
+        long needed = required - newOffset;
+        long newCapacity = Math.max(APPEND_MAP_CAPACITY, needed);
+        long remainder = newCapacity % APPEND_MAP_CAPACITY;
+        if (remainder != 0L) {
+            long padding = APPEND_MAP_CAPACITY - remainder;
+            if (newCapacity > Long.MAX_VALUE - padding) {
+                throw new IllegalStateException("symbol dict mmap capacity overflow to "
+                        + FILE_NAME + " [required=" + required + ']');
+            }
+            newCapacity += padding;
+        }
+        if (newOffset > Long.MAX_VALUE - newCapacity) {
+            throw new IllegalStateException("symbol dict mmap file offset overflow to "
+                    + FILE_NAME + " [offset=" + newOffset + ", capacity=" + newCapacity + ']');
+        }
+        long newFileSize = newOffset + newCapacity;
+        if (!ff.allocate(fd, newFileSize)) {
+            throw new IllegalStateException("could not grow mmap append region for "
+                    + FILE_NAME + " [required=" + required + ", fileSize=" + newFileSize + ']');
+        }
+        if (appendMapAddr != 0L) {
+            long oldAddr = appendMapAddr;
+            long oldCapacity = appendMapCapacity;
+            appendMapAddr = 0L;
+            appendMapCapacity = 0L;
+            appendMapOffset = 0L;
+            ff.munmap(oldAddr, oldCapacity, MemoryTag.MMAP_DEFAULT);
+        }
+        long newAddr = ff.mmap(
+                fd, newCapacity, newOffset, Files.MAP_RW, MemoryTag.MMAP_DEFAULT);
+        if (newAddr == Files.FAILED_MMAP_ADDRESS) {
+            throw new IllegalStateException("could not mmap append region for "
+                    + FILE_NAME + " [offset=" + newOffset + ", capacity=" + newCapacity + ']');
+        }
+        appendMapAddr = newAddr;
+        appendMapCapacity = newCapacity;
+        appendMapOffset = newOffset;
+        appendMapGrowthCount++;
+    }
+
+    /**
+     * Grows the append scratch buffer to hold at least {@code required} bytes.
+     * 

+     * Throws when {@code required} exceeds {@link #MAX_SCRATCH_BYTES} rather than
+     * clamping to it: a clamp would hand back a buffer SMALLER than the caller asked
+     * for and return normally, and every caller then writes {@code required} bytes
+     * into it -- turning a clean out-of-memory into a silent native-heap overflow, on
+     * the very write path the dictionary's integrity rests on. This is the same
+     * loud-failure shape {@code CursorWebSocketSendLoop.ensureSentDictCapacity} uses
+     * on the same bound. Unreachable at any realistic cardinality (it needs a ~2 GiB
+     * dictionary section in a single frame, itself bounded by the server's batch
+     * cap), but a size guard on a data-integrity write path must never
+     * under-allocate.
+     */
+    private void ensureScratch(long required) {
+        if (scratchCap >= required) {
+            return;
+        }
+        if (required > MAX_SCRATCH_BYTES) {
+            throw new IllegalStateException("symbol dict scratch buffer exceeds the maximum size to "
+                    + FILE_NAME + " [required=" + required + ", max=" + MAX_SCRATCH_BYTES + ']');
+        }
+        // Double in long: scratchCap * 2 as an int overflows negative past ~1 GB and
+        // would make the realloc size negative.
+        long newCap = Math.max(required, Math.max(256L, (long) scratchCap * 2));
+        if (newCap > MAX_SCRATCH_BYTES) {
+            newCap = MAX_SCRATCH_BYTES;
+        }
+        scratchAddr = Unsafe.realloc(scratchAddr, scratchCap, (int) newCap, MemoryTag.NATIVE_DEFAULT);
+        scratchCap = (int) newCap;
+    }
+
+    /**
+     * Checksums {@code [recStart, recStart + hdrLen + entriesLen)} in ONE native
+     * call, appends the CRC, and issues ONE positioned write. Advances
+     * {@code size}/{@code appendOffset} only on a complete write, so a short write
+     * throws and a retry keyed off {@link #size()} re-persists the same range at the
+     * same offset.
+     */
+    private void flushChunk(long recStart, int hdrLen, int entriesLen, int count) {
+        int bodyLen = hdrLen + entriesLen;
+        int recLen = bodyLen + CRC_SIZE;
+        Unsafe.getUnsafe().putInt(recStart + bodyLen, Crc32c.update(Crc32c.INIT, recStart, bodyLen));
+        appendWriteCount++;
+        long written = ff.write(fd, recStart, recLen, appendOffset);
+        if (written != recLen) {
+            throw new IllegalStateException("short write to " + FILE_NAME
+                    + " [expected=" + recLen + ", actual=" + written + ']');
+        }
+        appendOffset += recLen;
+        size += count;
+    }
+
+    /**
+     * Writes one chunk whose entry region is ALREADY encoded in scratch at offset
+     * {@link #MAX_CHUNK_HEADER_SIZE}. Back-fills the header immediately in front of
+     * the entries -- the header is at most {@code MAX_CHUNK_HEADER_SIZE} bytes, so it
+     * always fits the reserve -- leaving header, entries and CRC one contiguous run
+     * for a single checksum and a single write.
+     */
+    private void writeChunkFromScratch(int entriesLen, int count) {
+        int hdrLen = NativeBufferWriter.varintSize(count) + NativeBufferWriter.varintSize(entriesLen);
+        long recStart = scratchAddr + MAX_CHUNK_HEADER_SIZE - hdrLen;
+        long p = NativeBufferWriter.writeVarint(recStart, count);
+        NativeBufferWriter.writeVarint(p, entriesLen);
+        flushChunk(recStart, hdrLen, entriesLen, count);
+    }
+
+    private static final class RecoveryScan {
+        private final int count;
+        private final int entriesLen;
+        private final int validLen;
+
+        private RecoveryScan(int count, int entriesLen, int validLen) {
+            this.count = count;
+            this.entriesLen = entriesLen;
+            this.validLen = validLen;
+        }
+    }
+
+    /**
+     * Zero-allocation LEB128 decoder, one instance per recovery pass -- not one
+     * per chunk. The previous {@code long[]}-returning decoder allocated once per
+     * entry, so a million-symbol recovery churned a million throwaway arrays.
+     */
+    private static final class Varint {
+        int end;
+        long value;
+
+        /**
+         * Decodes the varint at {@code buf[pos..limit)}. Returns false -- leaving
+         * {@code value}/{@code end} undefined -- when the varint is truncated (a torn
+         * tail) or runs longer than a canonical 5-byte length.
+         */
+        boolean decode(long buf, int pos, int limit) {
+            long v = 0;
+            int shift = 0;
+            int cur = pos;
+            while (cur < limit) {
+                byte b = Unsafe.getUnsafe().getByte(buf + cur);
+                cur++;
+                v |= (long) (b & 0x7F) << shift;
+                if ((b & 0x80) == 0) {
+                    value = v;
+                    end = cur;
+                    return true;
+                }
+                shift += 7;
+                if (shift > 35) {
+                    return false; // implausible for a chunk header; treat as torn
+                }
+            }
+            return false;
+        }
+    }
+}
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/RecoveredFrameAnalysis.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/RecoveredFrameAnalysis.java
new file mode 100644
index 00000000..0977c833
--- /dev/null
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/RecoveredFrameAnalysis.java
@@ -0,0 +1,366 @@
+/*******************************************************************************
+ *     ___                  _   ____  ____
+ *    / _ \ _   _  ___  ___| |_|  _ \| __ )
+ *   | | | | | | |/ _ \/ __| __| | | |  _ \
+ *   | |_| | |_| |  __/\__ \ |_| |_| | |_) |
+ *    \__\_\\__,_|\___||___/\__|____/|____/
+ *
+ *  Copyright (c) 2014-2019 Appsicle
+ *  Copyright (c) 2019-2026 QuestDB
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ ******************************************************************************/
+
+package io.questdb.client.cutlass.qwp.client.sf.cursor;
+
+import io.questdb.client.cutlass.qwp.client.GlobalSymbolDictionary;
+import io.questdb.client.cutlass.qwp.protocol.QwpConstants;
+import io.questdb.client.std.MemoryTag;
+import io.questdb.client.std.QuietCloseable;
+import io.questdb.client.std.Unsafe;
+import io.questdb.client.std.str.Utf8s;
+
+/**
+ * Engine-owned result of the one ordered frame walk performed during recovery.
+ * It folds commit-boundary detection, delta extrema, gap detection and the raw
+ * symbol suffix into the same pass. Running state is checkpointed only at a
+ * commit-bearing frame, so a deferred orphan tail can be scanned without ever
+ * leaking its metadata or symbols into the committed result.
+ */
+final class RecoveredFrameAnalysis implements QuietCloseable {
+
+    private static final int MAX_RAW_BYTES = Integer.MAX_VALUE - 8;
+    private final long ackedFsn;
+    private final int baseline;
+    private long committedBoundaryFsn = -1L;
+    private long committedCoverage;
+    private boolean committedGap;
+    private long committedMaxDeltaEnd;
+    private long committedMaxDeltaStart;
+    private int committedRawCount;
+    private int committedRawLen;
+    private long framesVisited;
+    private boolean runningGap;
+    private boolean runningUnackedGap;
+    private long runningMaxDeltaEnd;
+    private long runningMaxDeltaStart;
+    private long symbolEntriesVisited;
+    private long rawAddr;
+    private int rawCapacity;
+    private int runningRawCount;
+    private int runningRawLen;
+    private long runningCoverage;
+
+    RecoveredFrameAnalysis(int baseline, long ackedFsn) {
+        this.baseline = baseline;
+        this.ackedFsn = ackedFsn;
+        this.runningCoverage = baseline;
+        this.committedCoverage = baseline;
+    }
+
+    void accept(long fsn, long payload, int payloadLen) {
+        framesVisited++;
+        boolean isQwp = payloadLen >= QwpConstants.HEADER_SIZE
+                && payloadLen > QwpConstants.HEADER_OFFSET_FLAGS
+                && Unsafe.getUnsafe().getInt(payload) == QwpConstants.MAGIC_MESSAGE;
+        byte flags = isQwp
+                ? Unsafe.getUnsafe().getByte(payload + QwpConstants.HEADER_OFFSET_FLAGS)
+                : 0;
+        if (isQwp && (flags & QwpConstants.FLAG_DELTA_SYMBOL_DICT) != 0) {
+            foldDelta(fsn, payload + QwpConstants.HEADER_SIZE, payload + payloadLen);
+        }
+
+        // Only a positively identified deferred QWP frame can belong to an
+        // uncommitted tail. Short, foreign or otherwise non-QWP payloads remain
+        // retirement barriers, matching findLastFrameFsnWithoutPayloadFlag.
+        if (!isQwp || (flags & QwpConstants.FLAG_DEFER_COMMIT) == 0) {
+            committedBoundaryFsn = fsn;
+            committedCoverage = runningCoverage;
+            committedGap = runningGap;
+            committedMaxDeltaEnd = runningMaxDeltaEnd;
+            committedMaxDeltaStart = runningMaxDeltaStart;
+            committedRawLen = runningRawLen;
+            committedRawCount = runningRawCount;
+        }
+    }
+
+    void addDecodedSymbolsTo(GlobalSymbolDictionary target) {
+        decodeSymbols(target);
+    }
+
+    private void decodeSymbols(GlobalSymbolDictionary target) {
+        long p = rawAddr;
+        long limit = rawAddr + committedRawLen;
+        for (int i = 0; i < committedRawCount; i++) {
+            long encoded = readVarint(p, limit);
+            if (encoded < 0L) {
+                throw new IllegalStateException("malformed cached symbol dictionary suffix");
+            }
+            int varintLen = (int) (encoded & 7L);
+            long symbolLen = encoded >>> 3;
+            p += varintLen;
+            if (symbolLen > limit - p) {
+                throw new IllegalStateException("truncated cached symbol dictionary suffix");
+            }
+            target.addRecoveredSymbol(Utf8s.stringFromUtf8Bytes(p, p + symbolLen));
+            p += symbolLen;
+        }
+        if (p != limit) {
+            throw new IllegalStateException("overfilled cached symbol dictionary suffix");
+        }
+    }
+
+    int baseline() {
+        return baseline;
+    }
+
+    long commitBoundaryFsn() {
+        return committedBoundaryFsn;
+    }
+
+    long coverage() {
+        return committedGap ? -1L : committedCoverage;
+    }
+
+    long framesVisited() {
+        return framesVisited;
+    }
+
+    /**
+     * Discards native bytes accumulated only while scanning an uncommitted
+     * deferred tail. Call exactly once after the ordered recovery walk.
+     */
+    void finish() {
+        runningRawLen = committedRawLen;
+        runningRawCount = committedRawCount;
+        if (rawCapacity == committedRawLen) {
+            return;
+        }
+        if (committedRawLen == 0) {
+            Unsafe.free(rawAddr, rawCapacity, MemoryTag.NATIVE_DEFAULT);
+            rawAddr = 0L;
+            rawCapacity = 0;
+            return;
+        }
+        rawAddr = Unsafe.realloc(
+                rawAddr,
+                rawCapacity,
+                committedRawLen,
+                MemoryTag.NATIVE_DEFAULT);
+        rawCapacity = committedRawLen;
+    }
+
+    long maxDeltaEnd() {
+        return committedMaxDeltaEnd;
+    }
+
+    long maxDeltaStart() {
+        return committedMaxDeltaStart;
+    }
+
+    long rawAddr() {
+        return rawAddr;
+    }
+
+    int rawCount() {
+        return committedRawCount;
+    }
+
+    int rawLen() {
+        return committedRawLen;
+    }
+
+    int rawCapacity() {
+        return rawCapacity;
+    }
+
+    /**
+     * Releases the cached suffix after a foreground producer and its one send
+     * loop have both consumed it. Recovery metadata remains available, but the
+     * raw entries must not be requested again.
+     */
+    void releaseRawStorage() {
+        if (rawAddr != 0L) {
+            Unsafe.free(rawAddr, rawCapacity, MemoryTag.NATIVE_DEFAULT);
+            rawAddr = 0L;
+            rawCapacity = 0;
+        }
+        runningRawLen = 0;
+        runningRawCount = 0;
+        committedRawLen = 0;
+        committedRawCount = 0;
+    }
+
+    long symbolEntriesVisited() {
+        return symbolEntriesVisited;
+    }
+
+    @Override
+    public void close() {
+        releaseRawStorage();
+    }
+
+    /**
+     * Appends one contiguous run of {@code count} wire entries -- {@code [len][utf8]}
+     * repeated, exactly as a delta section carries them -- in a single copy. Callers
+     * pass a whole frame's new-symbol suffix at once; see {@link #foldDelta} for why
+     * that suffix is always contiguous.
+     */
+    private void appendRaw(long addr, int len, int count) {
+        long required = (long) runningRawLen + len;
+        if (required > MAX_RAW_BYTES) {
+            throw new IllegalStateException("recovered symbol dictionary suffix exceeds maximum size "
+                    + "[required=" + required + ", max=" + MAX_RAW_BYTES + ']');
+        }
+        if (required > rawCapacity) {
+            long newCapacity = Math.max(required, Math.max(4_096L, (long) rawCapacity * 2L));
+            if (newCapacity > MAX_RAW_BYTES) {
+                newCapacity = MAX_RAW_BYTES;
+            }
+            rawAddr = Unsafe.realloc(rawAddr, rawCapacity, (int) newCapacity, MemoryTag.NATIVE_DEFAULT);
+            rawCapacity = (int) newCapacity;
+        }
+        Unsafe.getUnsafe().copyMemory(addr, rawAddr + runningRawLen, len);
+        runningRawLen += len;
+        runningRawCount += count;
+    }
+
+    private void foldDelta(long fsn, long p, long limit) {
+        long encodedStart = readVarint(p, limit);
+        if (encodedStart < 0L) {
+            markGap(fsn);
+            return;
+        }
+        int startLen = (int) (encodedStart & 7L);
+        long deltaStart = encodedStart >>> 3;
+        p += startLen;
+        if (deltaStart > runningMaxDeltaStart) {
+            runningMaxDeltaStart = deltaStart;
+        }
+
+        long encodedCount = readVarint(p, limit);
+        if (encodedCount < 0L) {
+            markGap(fsn);
+            return;
+        }
+        int countLen = (int) (encodedCount & 7L);
+        long deltaCount = encodedCount >>> 3;
+        p += countLen;
+        long deltaEnd = deltaCount > Long.MAX_VALUE - deltaStart
+                ? Long.MAX_VALUE
+                : deltaStart + deltaCount;
+        if (deltaEnd > runningMaxDeltaEnd) {
+            runningMaxDeltaEnd = deltaEnd;
+        }
+        if (runningGap) {
+            // A full dictionary is a new self-sufficient epoch, but it may only
+            // repair a gap that is entirely behind the durable ACK watermark.
+            // If any gapped frame will replay first, accepting this reset would
+            // hide the unsafe wire-order gap and let the server observe missing
+            // ids before it reaches the full frame.
+            if (deltaStart != 0L || runningUnackedGap) {
+                return;
+            }
+            runningGap = false;
+            runningCoverage = baseline;
+            runningRawLen = 0;
+            runningRawCount = 0;
+        }
+        if (deltaStart > runningCoverage) {
+            markGap(fsn);
+            return;
+        }
+        // The segment scan already CRC-validated this frame. When its entire
+        // range is covered, entry parsing cannot extend recovery state, so skip
+        // the cardinality-proportional payload walk.
+        if (deltaEnd <= runningCoverage) {
+            return;
+        }
+
+        // runningCoverage is loop-invariant here -- it only advances after the walk --
+        // and id ascends from deltaStart, so `id >= runningCoverage` is a step
+        // predicate: the entries this frame contributes are always ONE contiguous run
+        // at the tail of its delta section. Note where that run starts and copy it in a
+        // single memcpy once the walk succeeds, rather than paying a bound check, a
+        // capacity check and a ~12-byte copyMemory per symbol. Recovery walks the whole
+        // backlog, and the workload this feature exists for introduces a new symbol per
+        // ROW, so that is millions of stub-dispatched small copies where one bulk copy
+        // per frame does the job.
+        //
+        // Deferring the copy past the markGap bail-outs also drops the partial prefix
+        // the per-entry version used to leave behind. That residue was already
+        // unreachable -- a gap pins coverage() at -1, and a later self-sufficient frame
+        // resets runningRawLen/runningRawCount -- so not writing it is equivalent, and
+        // leaves less state to reason about.
+        long id = deltaStart;
+        long suffixStart = 0L;
+        int suffixCount = 0;
+        for (long i = 0; i < deltaCount; i++, id++) {
+            symbolEntriesVisited++;
+            long entryStart = p;
+            long encodedLen = readVarint(p, limit);
+            if (encodedLen < 0L) {
+                markGap(fsn);
+                return;
+            }
+            int varintLen = (int) (encodedLen & 7L);
+            long symbolLen = encodedLen >>> 3;
+            p += varintLen;
+            if (symbolLen > limit - p) {
+                markGap(fsn);
+                return;
+            }
+            p += symbolLen;
+            if (id >= runningCoverage) {
+                if (suffixCount == 0) {
+                    suffixStart = entryStart;
+                }
+                suffixCount++;
+            }
+        }
+        if (suffixCount > 0) {
+            appendRaw(suffixStart, (int) (p - suffixStart), suffixCount);
+        }
+        if (deltaEnd > runningCoverage) {
+            runningCoverage = deltaEnd;
+        }
+    }
+
+    private void markGap(long fsn) {
+        runningGap = true;
+        if (fsn > ackedFsn) {
+            runningUnackedGap = true;
+        }
+    }
+
+    /**
+     * Returns {@code (value << 3) | encodedByteCount}, or {@code -1} for an
+     * unterminated/oversized 32-bit protocol varint.
+     */
+    private static long readVarint(long p, long limit) {
+        long value = 0L;
+        int shift = 0;
+        int bytes = 0;
+        while (p < limit && bytes < 5) {
+            byte b = Unsafe.getUnsafe().getByte(p++);
+            value |= (long) (b & 0x7F) << shift;
+            bytes++;
+            if ((b & 0x80) == 0) {
+                return (value << 3) | bytes;
+            }
+            shift += 7;
+        }
+        return -1L;
+    }
+}
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java
index c8516c4c..6e438ac1 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SegmentRing.java
@@ -62,6 +62,8 @@ public final class SegmentRing implements QuietCloseable {
     public static final long BACKPRESSURE_NO_SPARE = -1L;
     /** Sentinel: append failed because the payload doesn't fit in a fresh segment. */
     public static final long PAYLOAD_TOO_LARGE = -2L;
+    static final String LEGACY_READER_GUARD_A = ".qwp-v2-guard-a.sfa";
+    static final String LEGACY_READER_GUARD_B = ".qwp-v2-guard-b.sfa";
     private static final Logger LOG = LoggerFactory.getLogger(SegmentRing.class);
     // Tally of baseSeq comparisons performed by sortByBaseSeq across every
     // openExisting() recovery on this JVM. Used by SegmentRingTest to
@@ -69,6 +71,7 @@ public final class SegmentRing implements QuietCloseable {
     // (CI runner variance makes elapsed-millisecond bounds flaky). Cheap
     // in production: one volatile-free add per partition pass, dwarfed by
     // the mmap I/O the recovery does on every segment.
+    private static final int SEALED_PREFIX_COMPACTION_THRESHOLD = 64;
     private static long sortComparisons;
     private final long maxBytesPerSegment;
     // Sealed segments in baseSeq order, oldest first. Active is held separately.
@@ -78,6 +81,16 @@ public final class SegmentRing implements QuietCloseable {
     // looks at sealedSegments after observing a higher ackedFsn, by which
     // point the producer thread's add to sealedSegments has retired.
     private final ObjList sealedSegments = new ObjList<>();
+    // Logical deque head. Trimming nulls and advances this cursor instead of
+    // shifting the remaining ObjList on every ACK. When the dead prefix is at
+    // least half the backing list (and non-trivially large), one bulk compaction
+    // reclaims it. Each live reference is therefore moved at most once per
+    // halving, making sequential trim O(N) overall instead of O(N^2).
+    private int sealedHead;
+    // Deterministic performance probes: unlike elapsed-time assertions these
+    // let tests pin binary successor search and amortized prefix compaction.
+    private int lastNextSealedSearchComparisons;
+    private long sealedCompactionMoves;
     // High-water byte offset within the active segment at which we proactively
     // ask the segment manager to provision a spare (if one isn't already
     // installed). Computed once as 3/4 of segment capacity -- leaves the manager
@@ -178,7 +191,9 @@ public static SegmentRing openExisting(String sfDir, long maxBytesPerSegment) {
                 int rc = 1;
                 while (rc > 0) {
                     String name = Files.utf8ToString(Files.findName(find));
-                    if (name != null && name.endsWith(".sfa")) {
+                    if (name != null
+                            && name.endsWith(".sfa")
+                            && !isLegacyReaderGuard(name)) {
                         String path = sfDir + "/" + name;
                         MmapSegment seg = null;
                         try {
@@ -325,6 +340,58 @@ public long ackedFsn() {
         return ackedFsn;
     }
 
+    /**
+     * Installs an on-disk rollback barrier before any v2 segment can be
+     * created or appended. The guards are valid v1 segments so a legacy
+     * reader cannot skip them as an unknown format. Their deliberately
+     * non-contiguous FSN ranges make its recovery fail closed before replay;
+     * current readers recognize the reserved names and omit them from the
+     * logical ring.
+     */
+    static void installLegacyReaderBarrier(String sfDir) {
+        installLegacyReaderGuardIfDamaged(sfDir + '/' + LEGACY_READER_GUARD_A, Long.MAX_VALUE - 4L);
+        installLegacyReaderGuardIfDamaged(sfDir + '/' + LEGACY_READER_GUARD_B, Long.MAX_VALUE - 1L);
+    }
+
+    /**
+     * Verifies the guard at {@code path} and rewrites it only when it is missing or
+     * damaged.
+     * 

+     * A guard's content never changes, so re-creating one that is already intact buys
+     * nothing -- and costs the only window in which the barrier can be lost. Creation
+     * goes through {@code openCleanRW} (which truncates) and does not fsync, so an
+     * unconditional rewrite on every engine open re-opens that window every time. A host
+     * crash inside it leaves a zero-filled guard next to v2 segment data that is already
+     * durable; a rolled-back v1 reader then skips the guard (bad magic) AND the v2
+     * segments (bad version), concludes the slot is empty, and truncates the unacked log
+     * the barrier exists to protect. Keeping an intact guard means the steady state never
+     * enters that window at all.
+     */
+    private static void installLegacyReaderGuardIfDamaged(String path, long baseSeq) {
+        if (isIntactLegacyReaderGuard(path, baseSeq)) {
+            return;
+        }
+        MmapSegment.createLegacyReaderGuard(path, baseSeq);
+    }
+
+    private static boolean isIntactLegacyReaderGuard(String path, long baseSeq) {
+        if (!Files.exists(path)) {
+            return false;
+        }
+        try (MmapSegment guard = MmapSegment.openExisting(path)) {
+            // Anything short of the exact shape createLegacyReaderGuard writes -- a
+            // zero-filled survivor of a torn creation included -- is rewritten.
+            return guard != null
+                    && guard.version() == MmapSegment.LEGACY_VERSION
+                    && guard.baseSeq() == baseSeq
+                    && guard.frameCount() == 1
+                    && guard.tornTailBytes() == 0L;
+        } catch (Throwable t) {
+            LOG.warn("legacy-reader guard {} could not be verified ({}); rewriting it", path, t.toString());
+            return false;
+        }
+    }
+
     /**
      * I/O thread (or anyone tracking ACK) advances the ACK cursor. {@code seq}
      * is cumulative -- the server has confirmed every FSN up to and including
@@ -439,13 +506,14 @@ public synchronized void close() {
             hotSpare.close();
             hotSpare = null;
         }
-        for (int i = 0, n = sealedSegments.size(); i < n; i++) {
+        for (int i = sealedHead, n = sealedSegments.size(); i < n; i++) {
             MmapSegment s = sealedSegments.get(i);
             if (s != null) {
                 s.close();
             }
         }
         sealedSegments.clear();
+        sealedHead = 0;
     }
 
     /**
@@ -462,9 +530,10 @@ public synchronized ObjList drainTrimmable() {
         // Sealed segments are in baseSeq order, oldest first; once we hit one
         // that isn't fully acked, none of the later ones can be either.
         // Synchronized so the I/O thread's snapshotSealedSegments() can't
-        // race against the remove(0) shuffling slots underneath it.
-        while (sealedSegments.size() > 0) {
-            MmapSegment s = sealedSegments.get(0);
+        // race against logical-head advancement or a bulk prefix compaction.
+        int n = sealedSegments.size();
+        while (sealedHead < n) {
+            MmapSegment s = sealedSegments.get(sealedHead);
             long lastSeq = s.baseSeq() + s.frameCount() - 1;
             if (lastSeq > acked) {
                 break;
@@ -473,8 +542,9 @@ public synchronized ObjList drainTrimmable() {
                 out = new ObjList<>();
             }
             out.add(s);
-            sealedSegments.remove(0);
+            sealedSegments.setQuick(sealedHead++, null);
         }
+        compactSealedPrefixIfNeeded();
         return out;
     }
 
@@ -490,7 +560,7 @@ public synchronized ObjList drainTrimmable() {
      * scan cost doesn't matter.
      */
     public synchronized MmapSegment findSegmentContaining(long fsn) {
-        for (int i = 0, n = sealedSegments.size(); i < n; i++) {
+        for (int i = sealedHead, n = sealedSegments.size(); i < n; i++) {
             MmapSegment s = sealedSegments.get(i);
             long base = s.baseSeq();
             if (fsn >= base && fsn < base + s.frameCount()) {
@@ -513,7 +583,7 @@ public synchronized MmapSegment findSegmentContaining(long fsn) {
      * fallback -- see {@link #nextSealedAfter(MmapSegment)}.
      */
     public synchronized MmapSegment firstSealed() {
-        return sealedSegments.size() > 0 ? sealedSegments.get(0) : null;
+        return sealedHead < sealedSegments.size() ? sealedSegments.get(sealedHead) : null;
     }
 
     /** Active segment -- exposed for the I/O thread's "send next batch" path. */
@@ -536,7 +606,7 @@ public synchronized MmapSegment firstSealed() {
      */
     public synchronized long findLastFsnWithoutPayloadFlag(int flagsOffset, int flagMask, int headerMagic, int minPayloadLen) {
         long best = -1L;
-        for (int i = 0, n = sealedSegments.size(); i < n; i++) {
+        for (int i = sealedHead, n = sealedSegments.size(); i < n; i++) {
             long fsn = sealedSegments.get(i).findLastFrameFsnWithoutPayloadFlag(flagsOffset, flagMask, headerMagic, minPayloadLen);
             if (fsn > best) {
                 best = fsn;
@@ -546,6 +616,25 @@ public synchronized long findLastFsnWithoutPayloadFlag(int flagsOffset, int flag
         return Math.max(best, fsn);
     }
 
+    /**
+     * Performs the one ordered recovery fold across sealed segments and the
+     * active segment. The returned native suffix remains owned by the caller.
+     */
+    synchronized RecoveredFrameAnalysis analyzeRecovery(int symbolBaseline) {
+        RecoveredFrameAnalysis analysis = new RecoveredFrameAnalysis(symbolBaseline, ackedFsn);
+        try {
+            for (int i = sealedHead, n = sealedSegments.size(); i < n; i++) {
+                sealedSegments.get(i).scanRecovery(analysis);
+            }
+            active.scanRecovery(analysis);
+            analysis.finish();
+            return analysis;
+        } catch (Throwable t) {
+            analysis.close();
+            throw t;
+        }
+    }
+
     public MmapSegment getActive() {
         return active;
     }
@@ -556,7 +645,8 @@ public MmapSegment getActive() {
      * concurrent rotation. Cross-thread readers (typically the I/O loop)
      * should use {@link #snapshotSealedSegments(MmapSegment[])} instead.
      */
-    public ObjList getSealedSegments() {
+    public synchronized ObjList getSealedSegments() {
+        compactSealedPrefix();
         return sealedSegments;
     }
 
@@ -608,13 +698,20 @@ public boolean needsHotSpare() {
      */
     public synchronized MmapSegment nextSealedAfter(MmapSegment current) {
         long currentBase = current.baseSeq();
-        for (int i = 0, n = sealedSegments.size(); i < n; i++) {
-            MmapSegment s = sealedSegments.get(i);
-            if (s.baseSeq() > currentBase) {
-                return s;
+        int lo = sealedHead;
+        int hi = sealedSegments.size();
+        int comparisons = 0;
+        while (lo < hi) {
+            int mid = (lo + hi) >>> 1;
+            comparisons++;
+            if (sealedSegments.get(mid).baseSeq() <= currentBase) {
+                lo = mid + 1;
+            } else {
+                hi = mid;
             }
         }
-        return null;
+        lastNextSealedSearchComparisons = comparisons;
+        return lo < sealedSegments.size() ? sealedSegments.get(lo) : null;
     }
 
     /**
@@ -663,15 +760,15 @@ public void setManagerWakeup(Runnable wakeup) {
      * the I/O loop is about to do.
      */
     public synchronized int snapshotSealedSegments(MmapSegment[] target) {
-        int n = sealedSegments.size();
+        int n = sealedSegments.size() - sealedHead;
         if (n > target.length) {
             for (int i = 0; i < target.length; i++) {
-                target[i] = sealedSegments.get(i);
+                target[i] = sealedSegments.get(sealedHead + i);
             }
             return -1;
         }
         for (int i = 0; i < n; i++) {
-            target[i] = sealedSegments.get(i);
+            target[i] = sealedSegments.get(sealedHead + i);
         }
         return n;
     }
@@ -689,13 +786,23 @@ public synchronized long totalSegmentBytes() {
         if (a != null) total += a.sizeBytes();
         MmapSegment hs = hotSpare;
         if (hs != null) total += hs.sizeBytes();
-        for (int i = 0, n = sealedSegments.size(); i < n; i++) {
+        for (int i = sealedHead, n = sealedSegments.size(); i < n; i++) {
             MmapSegment s = sealedSegments.get(i);
             if (s != null) total += s.sizeBytes();
         }
         return total;
     }
 
+    @TestOnly
+    public synchronized int getLastNextSealedSearchComparisons() {
+        return lastNextSealedSearchComparisons;
+    }
+
+    @TestOnly
+    public synchronized long getSealedCompactionMoves() {
+        return sealedCompactionMoves;
+    }
+
     /**
      * Returns the cumulative count of baseSeq comparisons performed by
      * {@link #sortByBaseSeq} since the last {@link #resetSortComparisons()}
@@ -717,6 +824,38 @@ public static void resetSortComparisons() {
         sortComparisons = 0;
     }
 
+    private void compactSealedPrefix() {
+        if (sealedHead == 0) {
+            return;
+        }
+        int size = sealedSegments.size();
+        int remaining = size - sealedHead;
+        if (remaining == 0) {
+            // Every retired slot was nulled while draining, so resetting pos
+            // neither retains references nor has to clear the backing array.
+            sealedSegments.setPos(0);
+        } else {
+            sealedCompactionMoves += remaining;
+            for (int i = 0; i < remaining; i++) {
+                sealedSegments.setQuick(i, sealedSegments.get(sealedHead + i));
+            }
+            for (int i = remaining; i < size; i++) {
+                sealedSegments.setQuick(i, null);
+            }
+            sealedSegments.setPos(remaining);
+        }
+        sealedHead = 0;
+    }
+
+    private void compactSealedPrefixIfNeeded() {
+        int size = sealedSegments.size();
+        if (sealedHead == size
+                || (sealedHead >= SEALED_PREFIX_COMPACTION_THRESHOLD
+                && sealedHead >= size - sealedHead)) {
+            compactSealedPrefix();
+        }
+    }
+
     /**
      * In-place quicksort over {@code list[lo, hi)} keyed by ascending
      * {@code baseSeq}. Median-of-three pivot avoids the pathological O(N²)
@@ -767,6 +906,18 @@ private static void sortByBaseSeq(ObjList list, int lo, int hi) {
         }
     }
 
+    /**
+     * Whether {@code name} is one of the reserved rollback-barrier files this class
+     * plants in every disk-mode slot. They carry the {@code .sfa} suffix so a rolled-back
+     * v1 reader cannot skip them, which means every directory walk that reasons about
+     * "does this slot hold data" has to exclude them -- they are a barrier, not a log.
+     * Package-private so {@link OrphanScanner} shares this one definition rather than
+     * matching the names itself.
+     */
+    static boolean isLegacyReaderGuard(String name) {
+        return LEGACY_READER_GUARD_A.equals(name) || LEGACY_READER_GUARD_B.equals(name);
+    }
+
     private static void swap(ObjList list, int i, int j) {
         if (i == j) return;
         MmapSegment tmp = list.get(i);
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SlotLock.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SlotLock.java
index 0b8379de..7902538c 100644
--- a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SlotLock.java
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/SlotLock.java
@@ -26,17 +26,25 @@
 
 import io.questdb.client.std.Compat;
 import io.questdb.client.std.Files;
+import io.questdb.client.std.FilesFacade;
 import io.questdb.client.std.MemoryTag;
 import io.questdb.client.std.QuietCloseable;
 import io.questdb.client.std.Unsafe;
+import org.jetbrains.annotations.TestOnly;
 
 import java.nio.charset.StandardCharsets;
+import java.nio.file.Path;
+import java.nio.file.Paths;
 
 /**
- * Advisory exclusive lock for a single SF slot directory.
+ * Advisory exclusive locks for a single SF slot.
  * 

- * One {@code .lock} file per slot, held via {@code flock}/{@code LockFileEx}
- * for the entire lifetime of the engine that owns the slot. The lock is
+ * {@link #acquire(String)} locks a {@code .lock} file inside the slot directory
+ * for the entire lifetime of the engine that owns it. {@link #acquireLogical(String)}
+ * locks a sibling file under the parent SF directory for short-lived pathname
+ * transitions and orphan adoption; because it is outside the slot directory,
+ * it remains stable if that directory is renamed. Both use
+ * {@code flock}/{@code LockFileEx}. A lock is
  * automatically released when the fd is closed — including on hard process
  * exit, since the kernel cleans up file locks for terminated processes.
  * 

@@ -58,10 +66,13 @@ public final class SlotLock implements QuietCloseable {
 
     private static final String LOCK_FILE_NAME = ".lock";
     private static final String LOCK_PID_FILE_NAME = ".lock.pid";
+    private static final String LOGICAL_LOCK_DIR_NAME = ".slot-locks";
+    private final FilesFacade ff;
     private final String slotDir;
     private int fd;
 
-    private SlotLock(String slotDir, int fd) {
+    private SlotLock(FilesFacade ff, String slotDir, int fd) {
+        this.ff = ff;
         this.slotDir = slotDir;
         this.fd = fd;
     }
@@ -75,42 +86,93 @@ private SlotLock(String slotDir, int fd) {
      *                               or lock contention.
      */
     public static SlotLock acquire(String slotDir) {
-        if (slotDir == null || slotDir.isEmpty()) {
-            throw new IllegalArgumentException("slotDir must not be empty");
-        }
-        if (!Files.exists(slotDir)) {
-            int rc = Files.mkdir(slotDir, Files.DIR_MODE_DEFAULT);
-            if (rc != 0) {
-                throw new IllegalStateException(
-                        "could not create slot dir: " + slotDir + " rc=" + rc);
-            }
-        }
+        validateSlotDir(slotDir);
+        ensureDirectory(FilesFacade.INSTANCE, slotDir, "slot dir");
         String lockPath = slotDir + "/" + LOCK_FILE_NAME;
         String pidPath = slotDir + "/" + LOCK_PID_FILE_NAME;
-        int fd = Files.openRW(lockPath);
+        return acquireAt(FilesFacade.INSTANCE, slotDir, lockPath, pidPath);
+    }
+
+    /**
+     * Acquires the stable logical lock for {@code slotDir}. Unlike the
+     * directory-local {@code .lock}, this lock is anchored in the parent SF
+     * directory, so renaming the slot cannot move the lock inode away from the
+     * logical slot name it guards.
+     * 

+     * Callers use this as a short-lived transition/adoption lock, always before
+     * acquiring the directory-local lock. In particular it must cover an
+     * unreplayable slot's close -> rename -> recreate transition, preventing a
+     * queued orphan drainer from adopting the renamed inode and later touching
+     * the fresh directory through the old pathname.
+     */
+    public static SlotLock acquireLogical(String slotDir) {
+        return acquireLogical(FilesFacade.INSTANCE, slotDir);
+    }
+
+    /** Facade-aware variant used to exercise logical-lock I/O failures. */
+    @TestOnly
+    public static SlotLock acquireLogical(FilesFacade ff, String slotDir) {
+        validateSlotDir(slotDir);
+        Path slotPath = Paths.get(slotDir);
+        Path parentPath = slotPath.getParent();
+        Path slotNamePath = slotPath.getFileName();
+        if (parentPath == null || slotNamePath == null || slotNamePath.toString().isEmpty()) {
+            throw new IllegalArgumentException(
+                    "slotDir must contain a parent and slot name: " + slotDir);
+        }
+        String parentDir = parentPath.toString();
+        String slotName = slotNamePath.toString();
+        String logicalLockDir = parentDir + "/" + LOGICAL_LOCK_DIR_NAME;
+        ensureDirectory(ff, logicalLockDir, "logical slot lock dir");
+        String lockPath = logicalLockDir + "/" + slotName + ".lock";
+        String pidPath = logicalLockDir + "/" + slotName + ".lock.pid";
+        return acquireAt(ff, slotDir, lockPath, pidPath);
+    }
+
+    private static SlotLock acquireAt(FilesFacade ff, String slotDir, String lockPath, String pidPath) {
+        int fd = ff.openRW(lockPath);
         if (fd < 0) {
             throw new IllegalStateException(
                     "could not open slot lock file: " + lockPath);
         }
         boolean ok = false;
         try {
-            int rc = Files.lock(fd);
+            int rc = ff.lock(fd);
             if (rc != 0) {
                 String holder = readHolder(pidPath);
                 throw new IllegalStateException(
                         "sf slot already in use by another process [slot="
                                 + slotDir + ", holder=" + holder + "]");
             }
-            writePid(pidPath);
+            writePid(ff, pidPath);
             ok = true;
-            return new SlotLock(slotDir, fd);
+            return new SlotLock(ff, slotDir, fd);
         } finally {
             if (!ok) {
-                Files.close(fd);
+                ff.close(fd);
             }
         }
     }
 
+    private static void ensureDirectory(FilesFacade ff, String path, String description) {
+        if (!ff.exists(path)) {
+            int rc = ff.mkdir(path, Files.DIR_MODE_DEFAULT);
+            // Multiple senders may create the shared parent lock directory
+            // concurrently. Treat EEXIST as success, just as the builder does
+            // for the SF root itself.
+            if (rc != 0 && !ff.exists(path)) {
+                throw new IllegalStateException(
+                        "could not create " + description + ": " + path + " rc=" + rc);
+            }
+        }
+    }
+
+    private static void validateSlotDir(String slotDir) {
+        if (slotDir == null || slotDir.isEmpty()) {
+            throw new IllegalArgumentException("slotDir must not be empty");
+        }
+    }
+
     /** Slot dir this lock guards. */
     public String slotDir() {
         return slotDir;
@@ -122,7 +184,7 @@ public void close() {
         // file or the .lock.pid sidecar — a stale PID is harmless (next
         // acquirer overwrites .lock.pid on success).
         if (fd >= 0) {
-            Files.close(fd);
+            ff.close(fd);
             fd = -1;
         }
     }
@@ -152,7 +214,7 @@ private static String readHolder(String pidPath) {
         }
     }
 
-    private static void writePid(String pidPath) {
+    private static void writePid(FilesFacade ff, String pidPath) {
         long pid;
         try {
             pid = Compat.currentPid();
@@ -160,25 +222,25 @@ private static void writePid(String pidPath) {
             // Diagnostic-only — never block lock acquisition on it.
             pid = -1L;
         }
-        int wfd = Files.openRW(pidPath);
+        int wfd = ff.openRW(pidPath);
         if (wfd < 0) {
             // Diagnostic-only — never block lock acquisition on it.
             return;
         }
         try {
-            Files.truncate(wfd, 0L);
+            ff.truncate(wfd, 0L);
             byte[] payload = (pid + "\n").getBytes(StandardCharsets.UTF_8);
             long addr = Unsafe.malloc(payload.length, MemoryTag.NATIVE_DEFAULT);
             try {
                 for (int i = 0; i < payload.length; i++) {
                     Unsafe.getUnsafe().putByte(addr + i, payload[i]);
                 }
-                Files.write(wfd, addr, payload.length, 0L);
+                ff.write(wfd, addr, payload.length, 0L);
             } finally {
                 Unsafe.free(addr, payload.length, MemoryTag.NATIVE_DEFAULT);
             }
         } finally {
-            Files.close(wfd);
+            ff.close(wfd);
         }
     }
 }
diff --git a/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/UnreplayableSlotException.java b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/UnreplayableSlotException.java
new file mode 100644
index 00000000..00c0d351
--- /dev/null
+++ b/core/src/main/java/io/questdb/client/cutlass/qwp/client/sf/cursor/UnreplayableSlotException.java
@@ -0,0 +1,51 @@
+/*******************************************************************************
+ *     ___                  _   ____  ____
+ *    / _ \ _   _  ___  ___| |_|  _ \| __ )
+ *   | | | | | | |/ _ \/ __| __| | | |  _ \
+ *   | |_| | |_| |  __/\__ \ |_| |_| | |_) |
+ *    \__\_\\__,_|\___||___/\__|____/|____/
+ *
+ *  Copyright (c) 2014-2019 Appsicle
+ *  Copyright (c) 2019-2026 QuestDB
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ ******************************************************************************/
+
+package io.questdb.client.cutlass.qwp.client.sf.cursor;
+
+import io.questdb.client.cutlass.line.LineSenderException;
+
+/**
+ * A recovered store-and-forward slot whose symbol dictionary cannot be rebuilt at all --
+ * neither from the persisted side-file's intact prefix nor from the surviving frames' own
+ * delta sections. Its frames reference ids that nothing still holds, so replaying them
+ * would null-pad the gap on the server and silently misattribute symbol values.
+ * 

+ * This is the ONE verdict {@code seedGlobalDictionaryFromPersisted} reaches after it has
+ * tried every source of truth it has, so it is the only signal that may quarantine a slot.
+ * A distinct type, rather than a message match, because the difference between "this slot
+ * is unreplayable" and any other {@link LineSenderException} out of the connect path is the
+ * difference between setting a slot aside and silently discarding one that was fine.
+ * 

+ * It stays a {@code LineSenderException} so that a caller which does NOT handle it -- the
+ * background drainer, a test constructing the sender directly -- keeps the old fail-clean
+ * behaviour rather than seeing a new checked type. Only {@code Sender.build()} treats it as
+ * recoverable, by setting the slot aside and starting the producer on a fresh one.
+ */
+public class UnreplayableSlotException extends LineSenderException {
+
+    public UnreplayableSlotException(CharSequence message) {
+        super(message);
+    }
+}
diff --git a/core/src/main/java/io/questdb/client/impl/ConfigSchema.java b/core/src/main/java/io/questdb/client/impl/ConfigSchema.java
index a255e638..7097f3cd 100644
--- a/core/src/main/java/io/questdb/client/impl/ConfigSchema.java
+++ b/core/src/main/java/io/questdb/client/impl/ConfigSchema.java
@@ -77,6 +77,7 @@ public final class ConfigSchema {
         // (one vocabulary, side-owned application).
         str("max_frame_rejections", Side.INGRESS);
         str("poison_min_escalation_window_millis", Side.INGRESS);
+        str("catchup_cap_gap_min_escalation_window_millis", Side.INGRESS);
         str("max_name_len", Side.INGRESS);
         str("reconnect_initial_backoff_millis", Side.INGRESS);
         str("reconnect_max_backoff_millis", Side.INGRESS);
diff --git a/core/src/main/java/io/questdb/client/std/Crc32c.java b/core/src/main/java/io/questdb/client/std/Crc32c.java
index d0a2e6a8..738efb1b 100644
--- a/core/src/main/java/io/questdb/client/std/Crc32c.java
+++ b/core/src/main/java/io/questdb/client/std/Crc32c.java
@@ -46,6 +46,7 @@
 public final class Crc32c {
     /** Seed value to start a fresh CRC-32C accumulation. */
     public static final int INIT = 0;
+    private static final int[] CRC32C_TABLE = buildCrc32cTable();
 
     private Crc32c() {
     }
@@ -65,6 +66,73 @@ private Crc32c() {
      */
     public static native int update(int seed, long addr, long len);
 
+    /**
+     * Java/Unsafe slice-by-8 CRC-32C for memory that can fault while it is read,
+     * such as a recovery mmap over a sparse or concurrently truncated file.
+     * Keeping every load at an {@link Unsafe} intrinsic site lets HotSpot turn an
+     * mmap access fault into a catchable {@link InternalError}; the native
+     * {@link #update} path cannot provide that guarantee because a SIGBUS raised
+     * inside JNI aborts the JVM.
+     * 

+     * The hot loop consumes each 8-byte block with two {@code getInt}s rather than
+     * eight {@code getByte}s -- 9 loads per block instead of 16, for the same table
+     * work. {@code getInt} is an {@link Unsafe} intrinsic exactly as {@code getByte}
+     * is, so the fault-catchability above is unaffected. It does make the loop
+     * little-endian, which costs nothing here: the segment and dictionary formats this
+     * checksums are already little-endian throughout (their headers are read back with
+     * {@code getInt}), and the native twin this must agree with asserts the same.
+     *
+     * @param seed previous CRC value, or {@link #INIT} to start fresh
+     * @param addr off-heap address of at least {@code len} readable bytes
+     * @param len  number of bytes to consume
+     * @return the new CRC value
+     */
+    public static int updateUnsafe(int seed, long addr, long len) {
+        assert len >= 0L : "CRC length must be non-negative";
+        int crc = ~seed;
+        int[] table = CRC32C_TABLE;
+        while (len >= Long.BYTES) {
+            // Little-endian: lo holds bytes 0..3 and hi bytes 4..7, ascending from the
+            // low byte, so the per-byte table lookups below stay in wire order.
+            int lo = Unsafe.getUnsafe().getInt(addr) ^ crc;
+            int hi = Unsafe.getUnsafe().getInt(addr + Integer.BYTES);
+            crc = table[7 * 256 + (lo & 0xFF)]
+                    ^ table[6 * 256 + ((lo >>> 8) & 0xFF)]
+                    ^ table[5 * 256 + ((lo >>> 16) & 0xFF)]
+                    ^ table[4 * 256 + (lo >>> 24)]
+                    ^ table[3 * 256 + (hi & 0xFF)]
+                    ^ table[2 * 256 + ((hi >>> 8) & 0xFF)]
+                    ^ table[256 + ((hi >>> 16) & 0xFF)]
+                    ^ table[hi >>> 24];
+            addr += Long.BYTES;
+            len -= Long.BYTES;
+        }
+        while (len-- > 0L) {
+            crc = (crc >>> 8) ^ table[(crc ^ Unsafe.getUnsafe().getByte(addr++)) & 0xFF];
+        }
+        return ~crc;
+    }
+
+    private static int[] buildCrc32cTable() {
+        int[] table = new int[8 * 256];
+        for (int n = 0; n < 256; n++) {
+            int c = n;
+            for (int k = 0; k < 8; k++) {
+                c = (c & 1) != 0 ? (0x82F63B78 ^ (c >>> 1)) : (c >>> 1);
+            }
+            table[n] = c;
+        }
+        for (int slice = 1; slice < 8; slice++) {
+            int previousOffset = (slice - 1) * 256;
+            int offset = slice * 256;
+            for (int n = 0; n < 256; n++) {
+                int previous = table[previousOffset + n];
+                table[offset + n] = (previous >>> 8) ^ table[previous & 0xFF];
+            }
+        }
+        return table;
+    }
+
     static {
         Os.init();
     }
diff --git a/core/src/main/java/io/questdb/client/std/FilesFacade.java b/core/src/main/java/io/questdb/client/std/FilesFacade.java
index 1b408cf4..c5e6afc7 100644
--- a/core/src/main/java/io/questdb/client/std/FilesFacade.java
+++ b/core/src/main/java/io/questdb/client/std/FilesFacade.java
@@ -85,6 +85,16 @@ public interface FilesFacade {
 
     int fsync(int fd);
 
+    /**
+     * Whether callers should use this facade's mmap path. The production facade
+     * returns {@code true}; wrapping fault facades retain positioned I/O unless
+     * they explicitly opt in, preserving their ability to inject short reads and
+     * writes.
+     */
+    default boolean isMmapAllowed() {
+        return this == INSTANCE;
+    }
+
     long length(int fd);
 
     /**
@@ -107,6 +117,19 @@ public interface FilesFacade {
 
     int mkdir(String path, int mode);
 
+    /**
+     * Maps a file region. Kept on the facade so mmap failures can be injected
+     * without relying on platform-specific filesystem behavior.
+     */
+    default long mmap(int fd, long len, long offset, int flags, int memoryTag) {
+        return Files.mmap(fd, len, offset, flags, memoryTag);
+    }
+
+    /** Releases a region returned by {@link #mmap(int, long, long, int, int)}. */
+    default void munmap(long address, long len, int memoryTag) {
+        Files.munmap(address, len, memoryTag);
+    }
+
     int openCleanRW(String path);
 
     /**
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/http/client/WebSocketClientTest.java b/core/src/test/java/io/questdb/client/test/cutlass/http/client/WebSocketClientTest.java
index a44c57f2..93a380f8 100644
--- a/core/src/test/java/io/questdb/client/test/cutlass/http/client/WebSocketClientTest.java
+++ b/core/src/test/java/io/questdb/client/test/cutlass/http/client/WebSocketClientTest.java
@@ -32,6 +32,7 @@
 import io.questdb.client.network.PlainSocketFactory;
 import io.questdb.client.network.Socket;
 import io.questdb.client.network.SocketReadinessWaiter;
+import io.questdb.client.std.MemoryTag;
 import io.questdb.client.std.Unsafe;
 import org.junit.Assert;
 import org.junit.Test;
@@ -312,6 +313,41 @@ public void onClose(int code, String reason) {
         });
     }
 
+    @Test
+    public void testSendBinaryAssemblesTwoPayloadSlicesIntoOneMaskedFrame() throws Exception {
+        assertMemoryLeak(() -> {
+            CapturingSocket socket = new CapturingSocket();
+            long first = Unsafe.malloc(3, MemoryTag.NATIVE_DEFAULT);
+            long second = Unsafe.malloc(4, MemoryTag.NATIVE_DEFAULT);
+            try (CapturingWebSocketClient client = new CapturingWebSocketClient(socket)) {
+                setUpgradedTrue(client);
+                for (int i = 0; i < 3; i++) {
+                    Unsafe.getUnsafe().putByte(first + i, (byte) (i + 1));
+                }
+                for (int i = 0; i < 4; i++) {
+                    Unsafe.getUnsafe().putByte(second + i, (byte) (i + 4));
+                }
+
+                client.sendBinary(first, 3, second, 4);
+
+                byte[] frame = socket.sent;
+                Assert.assertNotNull(frame);
+                Assert.assertEquals((byte) 0x82, frame[0]);
+                Assert.assertEquals((byte) (0x80 | 7), frame[1]);
+                byte[] payload = new byte[7];
+                for (int i = 0; i < payload.length; i++) {
+                    payload[i] = (byte) (frame[6 + i] ^ frame[2 + (i & 3)]);
+                }
+                Assert.assertArrayEquals(
+                        new byte[]{1, 2, 3, 4, 5, 6, 7},
+                        payload);
+            } finally {
+                Unsafe.free(first, 3, MemoryTag.NATIVE_DEFAULT);
+                Unsafe.free(second, 4, MemoryTag.NATIVE_DEFAULT);
+            }
+        });
+    }
+
     @Test
     public void testSendCloseFrameDoesNotClobberSendBuffer() throws Exception {
         assertMemoryLeak(() -> {
@@ -463,6 +499,34 @@ public boolean wantsTlsWrite() {
         }
     }
 
+    private static class CapturingSocket extends FakeSocket {
+        private byte[] sent;
+
+        @Override
+        public int send(long bufferPtr, int bufferLen) {
+            sent = new byte[bufferLen];
+            for (int i = 0; i < bufferLen; i++) {
+                sent[i] = Unsafe.getUnsafe().getByte(bufferPtr + i);
+            }
+            return bufferLen;
+        }
+    }
+
+    private static class CapturingWebSocketClient extends WebSocketClient {
+
+        CapturingWebSocketClient(CapturingSocket socket) {
+            super(DefaultHttpClientConfiguration.INSTANCE, (nf, log) -> socket);
+        }
+
+        @Override
+        protected void ioWait(int timeout, int op) {
+        }
+
+        @Override
+        protected void setupIoWait() {
+        }
+    }
+
     /**
      * Socket that serves a fixed byte sequence from recv() and reports
      * every send() as fully written (so close-frame sends succeed).
diff --git a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/CloseSafetyNetTest.java b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/CloseSafetyNetTest.java
index 2a266212..639b7c8d 100644
--- a/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/CloseSafetyNetTest.java
+++ b/core/src/test/java/io/questdb/client/test/cutlass/qwp/client/CloseSafetyNetTest.java
@@ -30,6 +30,7 @@
 import io.questdb.client.SenderErrorHandler;
 import io.questdb.client.cutlass.line.LineSenderException;
 import io.questdb.client.cutlass.qwp.client.QwpWebSocketSender;
+import io.questdb.client.cutlass.qwp.client.WebSocketResponse;
 import io.questdb.client.test.cutlass.qwp.websocket.TestWebSocketServer;
 import org.jetbrains.annotations.NotNull;
 import org.junit.Assert;
@@ -37,14 +38,18 @@
 import org.junit.Test;
 import org.junit.rules.TemporaryFolder;
 
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.nio.charset.StandardCharsets;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
 import java.util.concurrent.atomic.AtomicReference;
 
 /**
  * Pins both branches of {@code QwpWebSocketSender.close()}'s safety-net
- * rethrow, strictly — unlike the close assertions in
- * {@link InitialConnectAsyncTest}, which tolerate either outcome:
+ * rethrow using a deterministic server-side parse rejection:
  *