Skip to content

Commit 617cda7

Browse files
rjrudinrjrudin
committed
MLE-26427 Added unhappy test cases for exclusions
And a couple tests for JSON canonicalization
1 parent af01d9a commit 617cda7

File tree

3 files changed

+215
-0
lines changed

3 files changed

+215
-0
lines changed

‎marklogic-client-api/src/main/java/com/marklogic/client/datamovement/filter/IncrementalWriteFilter.java‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,11 +3,13 @@
33
*/
44
package com.marklogic.client.datamovement.filter;
55

6+
import com.fasterxml.jackson.core.JsonPointer;
67
import com.marklogic.client.datamovement.DocumentWriteSetFilter;
78
import com.marklogic.client.document.DocumentWriteOperation;
89
import com.marklogic.client.document.DocumentWriteSet;
910
import com.marklogic.client.impl.DocumentWriteOperationImpl;
1011
import com.marklogic.client.impl.HandleAccessor;
12+
import com.marklogic.client.impl.XmlFactories;
1113
import com.marklogic.client.io.BaseHandle;
1214
import com.marklogic.client.io.DocumentMetadataHandle;
1315
import com.marklogic.client.io.Format;
@@ -16,6 +18,8 @@
1618
import org.slf4j.Logger;
1719
import org.slf4j.LoggerFactory;
1820

21+
import javax.xml.xpath.XPath;
22+
import javax.xml.xpath.XPathExpressionException;
1923
import java.io.IOException;
2024
import java.nio.charset.StandardCharsets;
2125
import java.time.Instant;
@@ -114,11 +118,53 @@ public Builder xmlExclusions(String... xpathExpressions) {
114118
}
115119

116120
public IncrementalWriteFilter build() {
121+
validateJsonExclusions();
122+
validateXmlExclusions();
117123
if (useEvalQuery) {
118124
return new IncrementalWriteEvalFilter(hashKeyName, timestampKeyName, canonicalizeJson, skippedDocumentsConsumer, jsonExclusions, xmlExclusions);
119125
}
120126
return new IncrementalWriteOpticFilter(hashKeyName, timestampKeyName, canonicalizeJson, skippedDocumentsConsumer, jsonExclusions, xmlExclusions);
121127
}
128+
129+
private void validateJsonExclusions() {
130+
if (jsonExclusions == null) {
131+
return;
132+
}
133+
for (String jsonPointer : jsonExclusions) {
134+
if (jsonPointer == null || jsonPointer.trim().isEmpty()) {
135+
throw new IllegalArgumentException(
136+
"Empty JSON Pointer expression is not valid for excluding content from incremental write hash calculation; " +
137+
"it would exclude the entire document. JSON Pointer expressions must start with '/'.");
138+
}
139+
try {
140+
JsonPointer.compile(jsonPointer);
141+
} catch (IllegalArgumentException e) {
142+
throw new IllegalArgumentException(
143+
String.format("Invalid JSON Pointer expression '%s' for excluding content from incremental write hash calculation. " +
144+
"JSON Pointer expressions must start with '/'; cause: %s", jsonPointer, e.getMessage()), e);
145+
}
146+
}
147+
}
148+
149+
private void validateXmlExclusions() {
150+
if (xmlExclusions == null) {
151+
return;
152+
}
153+
XPath xpath = XmlFactories.getXPathFactory().newXPath();
154+
for (String xpathExpression : xmlExclusions) {
155+
if (xpathExpression == null || xpathExpression.trim().isEmpty()) {
156+
throw new IllegalArgumentException(
157+
"Empty XPath expression is not valid for excluding content from incremental write hash calculation.");
158+
}
159+
try {
160+
xpath.compile(xpathExpression);
161+
} catch (XPathExpressionException e) {
162+
throw new IllegalArgumentException(
163+
String.format("Invalid XPath expression '%s' for excluding content from incremental write hash calculation; cause: %s",
164+
xpathExpression, e.getMessage()), e);
165+
}
166+
}
167+
}
122168
}
123169

124170
protected final String hashKeyName;

‎marklogic-client-api/src/test/java/com/marklogic/client/datamovement/filter/ApplyExclusionsToIncrementalWriteTest.java‎

Lines changed: 103 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -139,4 +139,107 @@ void xmlExclusions() {
139139
assertEquals(10, writtenCount.get(), "Documents should be written since non-excluded content changed");
140140
assertEquals(5, skippedCount.get(), "Skip count should remain at 5");
141141
}
142+
143+
/**
144+
* Verifies that JSON Pointer exclusions are only applied to JSON documents and are ignored for XML documents.
145+
* The XML document should use its full content for hashing since no XML exclusions are configured.
146+
*/
147+
@Test
148+
void jsonExclusionsIgnoredForXmlDocuments() {
149+
filter = IncrementalWriteFilter.newBuilder()
150+
.jsonExclusions("/timestamp")
151+
.onDocumentsSkipped(docs -> skippedCount.addAndGet(docs.length))
152+
.build();
153+
154+
// Write one JSON doc and one XML doc
155+
docs = new ArrayList<>();
156+
ObjectNode jsonDoc = objectMapper.createObjectNode();
157+
jsonDoc.put("id", 1);
158+
jsonDoc.put("timestamp", "2025-01-01T10:00:00Z");
159+
docs.add(new DocumentWriteOperationImpl("/incremental/test/mixed-doc.json", METADATA, new JacksonHandle(jsonDoc)));
160+
161+
String xmlDoc = "<doc><id>1</id><timestamp>2025-01-01T10:00:00Z</timestamp></doc>";
162+
docs.add(new DocumentWriteOperationImpl("/incremental/test/mixed-doc.xml", METADATA, new StringHandle(xmlDoc).withFormat(Format.XML)));
163+
164+
writeDocs(docs);
165+
assertEquals(2, writtenCount.get());
166+
assertEquals(0, skippedCount.get());
167+
168+
// Write again with different timestamp values
169+
docs = new ArrayList<>();
170+
jsonDoc = objectMapper.createObjectNode();
171+
jsonDoc.put("id", 1);
172+
jsonDoc.put("timestamp", "2026-01-02T15:30:00Z"); // Changed
173+
docs.add(new DocumentWriteOperationImpl("/incremental/test/mixed-doc.json", METADATA, new JacksonHandle(jsonDoc)));
174+
175+
xmlDoc = "<doc><id>1</id><timestamp>2026-01-02T15:30:00Z</timestamp></doc>"; // Changed
176+
docs.add(new DocumentWriteOperationImpl("/incremental/test/mixed-doc.xml", METADATA, new StringHandle(xmlDoc).withFormat(Format.XML)));
177+
178+
writeDocs(docs);
179+
assertEquals(3, writtenCount.get(), "XML doc should be written since its timestamp changed and no XML exclusions are configured");
180+
assertEquals(1, skippedCount.get(), "JSON doc should be skipped since only the excluded timestamp field changed");
181+
}
182+
183+
/**
184+
* Verifies that when canonicalizeJson is false, documents with logically identical content
185+
* but different key ordering will produce different hashes, causing a write to occur.
186+
*/
187+
@Test
188+
void jsonNotCanonicalizedCausesDifferentHashForReorderedKeys() {
189+
filter = IncrementalWriteFilter.newBuilder()
190+
.canonicalizeJson(false)
191+
.onDocumentsSkipped(docs -> skippedCount.addAndGet(docs.length))
192+
.build();
193+
194+
// Write initial document with keys in a specific order
195+
docs = new ArrayList<>();
196+
String json1 = "{\"name\":\"Test\",\"id\":1,\"value\":100}";
197+
docs.add(new DocumentWriteOperationImpl("/incremental/test/non-canonical.json", METADATA,
198+
new StringHandle(json1).withFormat(Format.JSON)));
199+
200+
writeDocs(docs);
201+
assertEquals(1, writtenCount.get());
202+
assertEquals(0, skippedCount.get());
203+
204+
// Write again with same logical content but different key order
205+
docs = new ArrayList<>();
206+
String json2 = "{\"id\":1,\"value\":100,\"name\":\"Test\"}";
207+
docs.add(new DocumentWriteOperationImpl("/incremental/test/non-canonical.json", METADATA,
208+
new StringHandle(json2).withFormat(Format.JSON)));
209+
210+
writeDocs(docs);
211+
assertEquals(2, writtenCount.get(), "Document should be written because key order differs and JSON is not canonicalized");
212+
assertEquals(0, skippedCount.get(), "No documents should be skipped");
213+
}
214+
215+
/**
216+
* Verifies that with the default canonicalizeJson(true), documents with logically identical content
217+
* but different key ordering will produce the same hash, causing the document to be skipped.
218+
*/
219+
@Test
220+
void jsonCanonicalizedProducesSameHashForReorderedKeys() {
221+
filter = IncrementalWriteFilter.newBuilder()
222+
.onDocumentsSkipped(docs -> skippedCount.addAndGet(docs.length))
223+
.build();
224+
225+
// Write initial document with keys in a specific order
226+
docs = new ArrayList<>();
227+
String json1 = "{\"name\":\"Test\",\"id\":1,\"value\":100}";
228+
docs.add(new DocumentWriteOperationImpl("/incremental/test/canonical.json", METADATA,
229+
new StringHandle(json1).withFormat(Format.JSON)));
230+
231+
writeDocs(docs);
232+
assertEquals(1, writtenCount.get());
233+
assertEquals(0, skippedCount.get());
234+
235+
// Write again with same logical content but different key order
236+
docs = new ArrayList<>();
237+
String json2 = "{\"id\":1,\"value\":100,\"name\":\"Test\"}";
238+
docs.add(new DocumentWriteOperationImpl("/incremental/test/canonical.json", METADATA,
239+
new StringHandle(json2).withFormat(Format.JSON)));
240+
241+
writeDocs(docs);
242+
assertEquals(1, writtenCount.get(), "Document should be skipped because canonicalized JSON produces the same hash");
243+
assertEquals(1, skippedCount.get(), "One document should be skipped");
244+
}
142245
}

‎marklogic-client-api/src/test/java/com/marklogic/client/datamovement/filter/ApplyInvalidExclusionsToIncrementalWriteTest.java‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
/*
2+
* Copyright (c) 2010-2026 Progress Software Corporation and/or its subsidiaries or affiliates. All Rights Reserved.
3+
*/
4+
package com.marklogic.client.datamovement.filter;
5+
6+
import org.junit.jupiter.api.Test;
7+
8+
import static org.junit.jupiter.api.Assertions.assertThrows;
9+
import static org.junit.jupiter.api.Assertions.assertTrue;
10+
11+
class ApplyInvalidExclusionsToIncrementalWriteTest extends AbstractIncrementalWriteTest {
12+
13+
/**
14+
* Verifies that an invalid JSON Pointer expression (missing leading slash) causes the build to fail
15+
* immediately, allowing the user to fix the configuration before any documents are processed.
16+
*/
17+
@Test
18+
void invalidJsonPointerExpression() {
19+
IncrementalWriteFilter.Builder builder = IncrementalWriteFilter.newBuilder()
20+
.jsonExclusions("timestamp"); // Invalid - missing leading slash
21+
22+
IllegalArgumentException ex = assertThrows(IllegalArgumentException.class, builder::build);
23+
24+
assertTrue(ex.getMessage().contains("Invalid JSON Pointer expression 'timestamp'"),
25+
"Error message should include the invalid expression. Actual: " + ex.getMessage());
26+
assertTrue(ex.getMessage().contains("incremental write"),
27+
"Error message should mention incremental write context. Actual: " + ex.getMessage());
28+
assertTrue(ex.getMessage().contains("must start with '/'"),
29+
"Error message should hint at the fix. Actual: " + ex.getMessage());
30+
}
31+
32+
/**
33+
* Verifies that an empty JSON Pointer expression is rejected since it would exclude the entire document,
34+
* leaving nothing to hash.
35+
*/
36+
@Test
37+
void emptyJsonPointerExpression() {
38+
IncrementalWriteFilter.Builder builder = IncrementalWriteFilter.newBuilder()
39+
.jsonExclusions(""); // Invalid - would exclude entire document
40+
41+
IllegalArgumentException ex = assertThrows(IllegalArgumentException.class, builder::build);
42+
43+
assertTrue(ex.getMessage().contains("Empty JSON Pointer expression"),
44+
"Error message should indicate empty expression. Actual: " + ex.getMessage());
45+
assertTrue(ex.getMessage().contains("would exclude the entire document"),
46+
"Error message should explain why it's invalid. Actual: " + ex.getMessage());
47+
}
48+
49+
/**
50+
* Verifies that an invalid XPath expression causes the build to fail immediately,
51+
* allowing the user to fix the configuration before any documents are processed.
52+
*/
53+
@Test
54+
void invalidXPathExpression() {
55+
IncrementalWriteFilter.Builder builder = IncrementalWriteFilter.newBuilder()
56+
.xmlExclusions("[[[invalid xpath"); // Invalid XPath syntax
57+
58+
IllegalArgumentException ex = assertThrows(IllegalArgumentException.class, builder::build);
59+
60+
assertTrue(ex.getMessage().contains("Invalid XPath expression '[[[invalid xpath'"),
61+
"Error message should include the invalid expression. Actual: " + ex.getMessage());
62+
assertTrue(ex.getMessage().contains("incremental write"),
63+
"Error message should mention incremental write context. Actual: " + ex.getMessage());
64+
}
65+
66+
}

0 commit comments

Comments
 (0)