From 21501193244f197446f65822369f89d010c0ac4b Mon Sep 17 00:00:00 2001 From: arnav-chakraborty Date: Sun, 8 Feb 2026 16:15:44 +0530 Subject: [PATCH 1/2] CASSANDRA-21072: Prevent TombstoneOverwhelmingException from being swallowed when tracking warnings When trackWarnings is enabled, ReadCommandVerbHandler catches all RejectException subclasses and sends an empty success response with failure info in MessageParams. TombstoneOverwhelmingException is a hard abort that should propagate as a proper failure, not masquerade as success. Re-throw it regardless of warning tracking state. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../cassandra/db/ReadCommandVerbHandler.java | 3 +- .../db/ReadCommandVerbHandlerTest.java | 50 +++++++++++++++++++ 2 files changed, 52 insertions(+), 1 deletion(-) diff --git a/src/java/org/apache/cassandra/db/ReadCommandVerbHandler.java b/src/java/org/apache/cassandra/db/ReadCommandVerbHandler.java index 8ff89f4e9b5d..958689acc833 100644 --- a/src/java/org/apache/cassandra/db/ReadCommandVerbHandler.java +++ b/src/java/org/apache/cassandra/db/ReadCommandVerbHandler.java @@ -21,6 +21,7 @@ import org.slf4j.LoggerFactory; import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.db.filter.TombstoneOverwhelmingException; import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator; import org.apache.cassandra.dht.AbstractBounds; import org.apache.cassandra.dht.Token; @@ -88,7 +89,7 @@ public void doVerb(Message message) } catch (RejectException e) { - if (!command.isTrackingWarnings()) + if (!command.isTrackingWarnings() || e instanceof TombstoneOverwhelmingException) throw e; // make sure to log as the exception is swallowed diff --git a/test/unit/org/apache/cassandra/db/ReadCommandVerbHandlerTest.java b/test/unit/org/apache/cassandra/db/ReadCommandVerbHandlerTest.java index c00501105138..1b5aa3e4af79 100644 --- a/test/unit/org/apache/cassandra/db/ReadCommandVerbHandlerTest.java +++ b/test/unit/org/apache/cassandra/db/ReadCommandVerbHandlerTest.java @@ -33,6 +33,7 @@ import org.apache.cassandra.db.filter.ColumnFilter; import org.apache.cassandra.db.filter.DataLimits; import org.apache.cassandra.db.filter.RowFilter; +import org.apache.cassandra.db.filter.TombstoneOverwhelmingException; import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator; import org.apache.cassandra.exceptions.InvalidRequestException; import org.apache.cassandra.locator.InetAddressAndPort; @@ -146,6 +147,27 @@ public void rejectsRequestWithNonMatchingTransientness() .build()); } + @Test(expected = TombstoneOverwhelmingException.class) + public void tombstoneExceptionPropagatesWithoutWarningTracking() + { + ReadCommand command = new TombstoneThrowingReadCommand(metadata); + handler.doVerb(Message.builder(READ_REQ, command) + .from(peer()) + .withId(messageId()) + .build()); + } + + @Test(expected = TombstoneOverwhelmingException.class) + public void tombstoneExceptionPropagatesWithWarningTracking() + { + ReadCommand command = new TombstoneThrowingReadCommand(metadata); + handler.doVerb(Message.builder(READ_REQ, command) + .from(peer()) + .withFlag(MessageFlag.TRACK_WARNINGS) + .withId(messageId()) + .build()); + } + private static int messageId() { return random.nextInt(); @@ -199,6 +221,34 @@ public boolean isTrackingRepairedData() } } + private static class TombstoneThrowingReadCommand extends SinglePartitionReadCommand + { + TombstoneThrowingReadCommand(TableMetadata metadata) + { + super(metadata.epoch, + false, + 0, + false, + PotentialTxnConflicts.DISALLOW, + metadata, + FBUtilities.nowInSeconds(), + ColumnFilter.all(metadata), + RowFilter.none(), + DataLimits.NONE, + KEY, + new ClusteringIndexSliceFilter(Slices.ALL, false), + null, + false, + null); + } + + @Override + public UnfilteredPartitionIterator executeLocally(ReadExecutionController executionController) + { + throw new TombstoneOverwhelmingException(1000, "test_query", metadata(), KEY, Clustering.EMPTY); + } + } + private static DecoratedKey key(TableMetadata metadata, int key) { return metadata.partitioner.decorateKey(ByteBufferUtil.bytes(key)); From 6bf2cd8ef6380a6efda90cdf34339d618faf29ab Mon Sep 17 00:00:00 2001 From: arnav-chakraborty Date: Fri, 13 Feb 2026 15:34:17 +0530 Subject: [PATCH 2/2] Add test for RejectException swallow path when tracking warnings Verifies that non-tombstone RejectExceptions (e.g., LocalReadSizeTooLargeException) are correctly swallowed and sent as empty success responses with MessageParams when TRACK_WARNINGS is enabled. Co-Authored-By: Claude Opus 4.6 (1M context) --- .../db/ReadCommandVerbHandlerTest.java | 41 +++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/test/unit/org/apache/cassandra/db/ReadCommandVerbHandlerTest.java b/test/unit/org/apache/cassandra/db/ReadCommandVerbHandlerTest.java index 1b5aa3e4af79..351af987e1d2 100644 --- a/test/unit/org/apache/cassandra/db/ReadCommandVerbHandlerTest.java +++ b/test/unit/org/apache/cassandra/db/ReadCommandVerbHandlerTest.java @@ -33,6 +33,7 @@ import org.apache.cassandra.db.filter.ColumnFilter; import org.apache.cassandra.db.filter.DataLimits; import org.apache.cassandra.db.filter.RowFilter; +import org.apache.cassandra.db.filter.LocalReadSizeTooLargeException; import org.apache.cassandra.db.filter.TombstoneOverwhelmingException; import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator; import org.apache.cassandra.exceptions.InvalidRequestException; @@ -168,6 +169,18 @@ public void tombstoneExceptionPropagatesWithWarningTracking() .build()); } + @Test + public void rejectExceptionSwallowedWithWarningTracking() + { + ReadCommand command = new RejectThrowingReadCommand(metadata); + handler.doVerb(Message.builder(READ_REQ, command) + .from(peer()) + .withFlag(MessageFlag.TRACK_WARNINGS) + .withId(messageId()) + .build()); + // If we reach here without an exception, the RejectException was correctly swallowed + } + private static int messageId() { return random.nextInt(); @@ -249,6 +262,34 @@ public UnfilteredPartitionIterator executeLocally(ReadExecutionController execut } } + private static class RejectThrowingReadCommand extends SinglePartitionReadCommand + { + RejectThrowingReadCommand(TableMetadata metadata) + { + super(metadata.epoch, + false, + 0, + false, + PotentialTxnConflicts.DISALLOW, + metadata, + FBUtilities.nowInSeconds(), + ColumnFilter.all(metadata), + RowFilter.none(), + DataLimits.NONE, + KEY, + new ClusteringIndexSliceFilter(Slices.ALL, false), + null, + false, + null); + } + + @Override + public UnfilteredPartitionIterator executeLocally(ReadExecutionController executionController) + { + throw new LocalReadSizeTooLargeException("test read size too large"); + } + } + private static DecoratedKey key(TableMetadata metadata, int key) { return metadata.partitioner.decorateKey(ByteBufferUtil.bytes(key));