From 3565a6c30f6e3ad3c0d985201fc51654b8bdd6bf Mon Sep 17 00:00:00 2001 From: Aaron Morton Date: Wed, 18 Feb 2026 09:00:52 +1300 Subject: [PATCH 01/89] WIP --- pom.xml | 5 + .../v1/vectorize/EnvironmentalRequest.java | 6 + .../api/v1/vectorize/ITCollection.java | 148 +++++++++++++ .../jsonapi/api/v1/vectorize/ITElement.java | 15 ++ .../sgv2/jsonapi/api/v1/vectorize/ITFile.java | 22 ++ .../jsonapi/api/v1/vectorize/ITMetadata.java | 3 + .../api/v1/vectorize/IntegrationEnv.java | 35 ++++ .../api/v1/vectorize/IntegrationJob.java | 62 ++++++ .../v1/vectorize/IntegrationJobRunner.java | 34 +++ .../api/v1/vectorize/IntegrationTest.java | 42 ++++ .../v1/vectorize/IntegrationTestRunner.java | 197 ++++++++++++++++++ .../api/v1/vectorize/IntegrationWorkflow.java | 11 + .../jsonapi/api/v1/vectorize/RunnerBase.java | 34 +++ .../api/v1/vectorize/TestAssertion.java | 5 + .../jsonapi/api/v1/vectorize/TestItem.java | 12 ++ .../jsonapi/api/v1/vectorize/TestRequest.java | 89 ++++++++ .../jsonapi/api/v1/vectorize/VectorizeIT.java | 24 +++ .../api/v1/vectorize/VectorizeUnit.java | 22 ++ .../assertions/AssertionFactory.java | 12 ++ .../v1/vectorize/assertions/Documents.java | 18 ++ .../v1/vectorize/assertions/ITAssertion.java | 11 + .../api/v1/vectorize/assertions/Response.java | 13 ++ .../integration-tests/vectorize/open-ai.json | 30 +++ .../vectorize/vectorize-base.json | 83 ++++++++ .../vectorize/vectorize-workflow.json | 56 +++++ 25 files changed, 989 insertions(+) create mode 100644 src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/EnvironmentalRequest.java create mode 100644 src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/ITCollection.java create mode 100644 src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/ITElement.java create mode 100644 src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/ITFile.java create mode 100644 src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/ITMetadata.java create mode 100644 src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/IntegrationEnv.java create mode 100644 src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/IntegrationJob.java create mode 100644 src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/IntegrationJobRunner.java create mode 100644 src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/IntegrationTest.java create mode 100644 src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/IntegrationTestRunner.java create mode 100644 src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/IntegrationWorkflow.java create mode 100644 src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/RunnerBase.java create mode 100644 src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestAssertion.java create mode 100644 src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestItem.java create mode 100644 src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestRequest.java create mode 100644 src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/VectorizeIT.java create mode 100644 src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/VectorizeUnit.java create mode 100644 src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/AssertionFactory.java create mode 100644 src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Documents.java create mode 100644 src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/ITAssertion.java create mode 100644 src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Response.java create mode 100644 src/test/resources/integration-tests/vectorize/open-ai.json create mode 100644 src/test/resources/integration-tests/vectorize/vectorize-base.json create mode 100644 src/test/resources/integration-tests/vectorize/vectorize-workflow.json diff --git a/pom.xml b/pom.xml index e64847c597..1bcfa72c12 100644 --- a/pom.xml +++ b/pom.xml @@ -90,6 +90,11 @@ + + com.jayway.jsonpath + json-path + 2.10.0 + io.quarkus quarkus-arc diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/EnvironmentalRequest.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/EnvironmentalRequest.java new file mode 100644 index 0000000000..82587e992b --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/EnvironmentalRequest.java @@ -0,0 +1,6 @@ +package io.stargate.sgv2.jsonapi.api.v1.vectorize; + +public interface EnvironmentalRequest { + + +} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/ITCollection.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/ITCollection.java new file mode 100644 index 0000000000..44268839cb --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/ITCollection.java @@ -0,0 +1,148 @@ +package io.stargate.sgv2.jsonapi.api.v1.vectorize; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.jayway.jsonpath.Configuration; +import com.jayway.jsonpath.JsonPath; +import com.jayway.jsonpath.spi.json.JacksonJsonNodeJsonProvider; +import com.jayway.jsonpath.spi.mapper.JacksonMappingProvider; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.net.URISyntaxException; +import java.net.URL; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Stream; + +public class ITCollection { + + private static final ObjectMapper MAPPER = new ObjectMapper(); + + private final Configuration config = + Configuration.builder() + .jsonProvider(new JacksonJsonNodeJsonProvider()) + .mappingProvider(new JacksonMappingProvider()) + .build(); + + List itFiles; + + private ITCollection(List itFiles) { + this.itFiles = itFiles; + for (ITFile file : itFiles) { + if (file.element() instanceof IntegrationTest it){ + it.expand(this); + } + } + } + + public IntegrationWorkflow workflowFirstByName(String name) { + var path = "$.meta[?(@.name == '%s')]".formatted(name); + + var itFile = + match(ITElement.ITElementKind.WORKFLOW, path).stream() + .findFirst() + .orElseThrow( + () -> + new IllegalArgumentException( + "No IT file found with meta.name == '%s'".formatted(name))); + return (IntegrationWorkflow) itFile.element(); + } + + public IntegrationTest testsFirstByName(String name) { + return testsByName(name).getFirst(); + } + + public List testsByName(String name) { + var path = "$.meta[?(@.name == '%s')]".formatted(name); + + return match(ITElement.ITElementKind.TEST, path).stream() + .map(itFile -> (IntegrationTest) itFile.element()) + .toList(); + } + + private List byKind(ITElement.ITElementKind kind) { + return itFiles.stream().filter(itFile -> itFile.element().kind() == kind).toList(); + } + + private List match(ITElement.ITElementKind kind, String jsonPath) { + var compiled = JsonPath.compile(jsonPath); + + return byKind(kind).stream().filter(itFile -> hasMatch(itFile.root(), compiled)).toList(); + } + + private boolean hasMatch(JsonNode root, JsonPath compiled) { + var pathResult = JsonPath.using(config).parse(root).read(compiled); + + return switch (pathResult) { + case null -> false; + case java.util.Collection c -> !c.isEmpty(); + case java.util.Map m -> !m.isEmpty(); + case ArrayNode a -> !a.isEmpty(); + case ObjectNode o -> !o.isEmpty(); + default -> true; + }; + } + + static ITCollection loadAll(String path) { + final Path dir = resourceDir(path); + + List itFiles = new ArrayList<>(); + + try (Stream s = Files.walk(dir)) { + itFiles = + s.filter(Files::isRegularFile) + .filter(p -> p.getFileName().toString().endsWith(".json")) + .map(ITCollection::loadOne) + .toList(); + } catch (IOException e) { + throw new UncheckedIOException("Failed reading test resources under: " + dir, e); + } + return new ITCollection(itFiles); + } + + private static ITFile loadOne(Path file) { + try { + var root = MAPPER.readTree(file.toFile()); + + JsonNode kindNode = root.path("meta").path("kind"); + if (!kindNode.isTextual()) { + throw new IllegalArgumentException("Missing/invalid meta.kind in " + file); + } + + var kind = kindNode.asText(); + var element = + switch (kind.toUpperCase()) { + case "TEST" -> MAPPER.treeToValue(root, IntegrationTest.class); + case "WORKFLOW" -> MAPPER.treeToValue(root, IntegrationWorkflow.class); + default -> + throw new IllegalArgumentException("Unknown meta.kind '" + kind + "' in " + file); + }; + return new ITFile(element, root); + } catch (IOException e) { + throw new UncheckedIOException("Failed parsing JSON file: " + file, e); + } + } + + private static Path resourceDir(String path) { + String normalized = path.startsWith("/") ? path.substring(1) : path; + + ClassLoader cl = Thread.currentThread().getContextClassLoader(); + URL url = cl.getResource(normalized); + if (url == null) { + throw new IllegalArgumentException("Test resource folder not found: " + path); + } + + try { + // Works for file: URLs; if you run tests from a jar, switch to getResourceAsStream-based + // walking. + return Paths.get(url.toURI()); + } catch (URISyntaxException e) { + throw new IllegalArgumentException("Bad resource URI for: " + path + " -> " + url, e); + } + } +} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/ITElement.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/ITElement.java new file mode 100644 index 0000000000..e2314448be --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/ITElement.java @@ -0,0 +1,15 @@ +package io.stargate.sgv2.jsonapi.api.v1.vectorize; + +public interface ITElement { + + ITElementKind kind(); + + ITMetadata meta(); + + // void setJson(JsonNode json); + + enum ITElementKind { + TEST, + WORKFLOW + } +} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/ITFile.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/ITFile.java new file mode 100644 index 0000000000..85f9087916 --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/ITFile.java @@ -0,0 +1,22 @@ +package io.stargate.sgv2.jsonapi.api.v1.vectorize; + +import com.fasterxml.jackson.databind.JsonNode; + +public class ITFile { + + private final ITElement element; + private final JsonNode root; + + ITFile(ITElement element, JsonNode root) { + this.element = element; + this.root = root; + } + + public ITElement element() { + return element; + } + + public JsonNode root() { + return root; + } +} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/ITMetadata.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/ITMetadata.java new file mode 100644 index 0000000000..828a6dfd77 --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/ITMetadata.java @@ -0,0 +1,3 @@ +package io.stargate.sgv2.jsonapi.api.v1.vectorize; + +public record ITMetadata(String name, String kind) {} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/IntegrationEnv.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/IntegrationEnv.java new file mode 100644 index 0000000000..f8346c2c42 --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/IntegrationEnv.java @@ -0,0 +1,35 @@ +package io.stargate.sgv2.jsonapi.api.v1.vectorize; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.List; +import java.util.Map; + +public record IntegrationEnv(Map vars) { + private static final Logger LOGGER = LoggerFactory.getLogger(IntegrationEnv.class); + + private static final List SCHEMA_IDENTIFIER = List.of("COLLECTION_NAME"); + + public IntegrationEnv { + + for (var name : SCHEMA_IDENTIFIER) { + if (vars.containsKey(name)) { + var oldValue = vars.get(name); + var newValue = oldValue.replaceAll("[^A-Za-z0-9_]", "_"); + + if (newValue.length() > 48){ + throw new RuntimeException("Schema Identifier longer than 48 characters %s=%s".formatted(name,newValue)); + } + LOGGER.info("XXX Updated IntegrationEnv value because it is a Schema Identifier key={}, oldValue={}, newValue={}",name,oldValue,newValue); + vars.put(name, newValue); + } + } + } + public String requiredValue(String name){ + if (vars.containsKey(name)){ + return vars.get(name); + } + throw new RuntimeException(String.format("Required parameter %s not found", name)); + } +} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/IntegrationJob.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/IntegrationJob.java new file mode 100644 index 0000000000..cc34b490cb --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/IntegrationJob.java @@ -0,0 +1,62 @@ +package io.stargate.sgv2.jsonapi.api.v1.vectorize; + +import io.stargate.sgv2.jsonapi.api.model.command.CommandErrorFactory; +import org.apache.commons.text.StringSubstitutor; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public record IntegrationJob( + ITMetadata meta, + Map fromEnvironment, + Map variables, + Map> matrix, + List tests) { + private static final Logger LOGGER = LoggerFactory.getLogger(IntegrationJob.class); + + + public List allEnvironments() { + + Map fromEnv = new HashMap<>(); + for (Map.Entry entry : fromEnvironment.entrySet()) { + var value = System.getenv(entry.getValue()); + if (value== null) { + throw new RuntimeException("Environment variable " + entry.getValue() + " is undefined"); + } + fromEnv.put(entry.getKey(), value); + } + + // TODO: handle more matrix + var modelList = matrix.get("MODEL"); + + List> allMatrix = new ArrayList<>(); + modelList.forEach( + model -> { + var env = new HashMap(); + env.put("MODEL", model); + allMatrix.add(env); + }); + + List envs = new ArrayList<>(); + allMatrix.forEach( + matrix -> { + matrix.putAll(variables); + matrix.putAll(fromEnv); + + LOGGER.info("XXX ENV BEFORE SUBS {}", matrix); + var subs = new StringSubstitutor(matrix).setEnableUndefinedVariableException(true); + matrix.forEach((key, value) -> { + matrix.put(key, subs.replace(value)); + }); + + LOGGER.info("XXX ENV AFTER SUBS {}", matrix); + var env = new IntegrationEnv(matrix); + envs.add(env); + }); + return envs; + } +} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/IntegrationJobRunner.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/IntegrationJobRunner.java new file mode 100644 index 0000000000..c93ad332d5 --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/IntegrationJobRunner.java @@ -0,0 +1,34 @@ +package io.stargate.sgv2.jsonapi.api.v1.vectorize; + +import java.util.ArrayList; +import java.util.List; + +public class IntegrationJobRunner { + + private final ITCollection itCollection; + private final IntegrationJob job; + + public IntegrationJobRunner(ITCollection itCollection, IntegrationJob job) { + this.itCollection = itCollection; + this.job = job; + } + + public void run() { + + var allEnvs = job.allEnvironments(); + + List allTests = new ArrayList<>(); + job.tests() + .forEach( + testName -> { + allTests.addAll(itCollection.testsByName(testName)); + }); + + for (IntegrationTest test : allTests) { + for (IntegrationEnv env : allEnvs) { + var testRunner = new IntegrationTestRunner(itCollection, test, env); + testRunner.run(); + } + } + } +} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/IntegrationTest.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/IntegrationTest.java new file mode 100644 index 0000000000..b994f46bb2 --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/IntegrationTest.java @@ -0,0 +1,42 @@ +package io.stargate.sgv2.jsonapi.api.v1.vectorize; + +import java.util.ArrayList; +import java.util.List; + +public record IntegrationTest(ITMetadata meta, List setup, List tests) + implements ITElement { + + @Override + public ITElementKind kind() { + return ITElementKind.TEST; + } + + public void expand(ITCollection itCollection) { + + List expandedSetup = new ArrayList<>(); + for (TestRequest request : setup) { + if (request.request().has("$include")){ + var includedTest = itCollection.testsFirstByName(request.request().get("$include").textValue()); + expandedSetup.addAll(includedTest.setup()); + } + else { + expandedSetup.add(request); + } + } + setup.clear(); + setup.addAll(expandedSetup); + + List expandedTests = new ArrayList<>(); + for (TestItem item : tests) { + if (item.include() != null ){ + var includedTest = itCollection.testsFirstByName(item.include()); + expandedTests.addAll(includedTest.tests()); + } + else { + expandedTests.add(item); + } + } + tests.clear(); + tests.addAll(expandedTests); + } +} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/IntegrationTestRunner.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/IntegrationTestRunner.java new file mode 100644 index 0000000000..5ac9b240b1 --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/IntegrationTestRunner.java @@ -0,0 +1,197 @@ +package io.stargate.sgv2.jsonapi.api.v1.vectorize; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import io.restassured.http.ContentType; +import io.restassured.response.ValidatableResponse; +import io.restassured.specification.RequestSpecification; +import io.stargate.sgv2.jsonapi.api.model.command.CommandTarget; +import io.stargate.sgv2.jsonapi.api.v1.CollectionResource; +import io.stargate.sgv2.jsonapi.api.v1.GeneralResource; +import io.stargate.sgv2.jsonapi.api.v1.KeyspaceResource; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions.AssertionFactory; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions.ITAssertion; +import io.stargate.sgv2.jsonapi.config.constants.HttpConstants; +import io.stargate.sgv2.jsonapi.service.embedding.operation.test.CustomITEmbeddingProvider; +import org.apache.commons.lang3.RandomStringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.lang.reflect.Modifier; +import java.util.Arrays; +import java.util.Base64; +import java.util.Map; + +import static io.restassured.RestAssured.given; +import static io.stargate.sgv2.jsonapi.api.v1.ResponseAssertions.responseIsDDLSuccess; +import static io.stargate.sgv2.jsonapi.api.v1.ResponseAssertions.responseIsWriteSuccess; +import static io.stargate.sgv2.jsonapi.api.v1.util.IntegrationTestUtils.getCassandraPassword; +import static io.stargate.sgv2.jsonapi.api.v1.util.IntegrationTestUtils.getCassandraUsername; +import static org.hamcrest.Matchers.*; + +public class IntegrationTestRunner extends RunnerBase { + private static final Logger LOGGER = LoggerFactory.getLogger(IntegrationEnv.class); + + + // keyspace automatically created in this test + protected static final String keyspaceName = + "ks" + RandomStringUtils.insecure().nextAlphanumeric(16); + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + + private final ITCollection itCollection; + private final IntegrationTest test; + private final IntegrationEnv env; + + public IntegrationTestRunner( + ITCollection itCollection, IntegrationTest test, IntegrationEnv env) { + this.itCollection = itCollection; + this.test = test; + this.env = env; + } + + @Override + protected IntegrationEnv integrationEnv() { + return env; + } + + public void run() { + + createKeyspace(keyspaceName); + + for (TestRequest setupRequest : test.setup()) { + + var requestWithEnv = setupRequest.withEnvironment(env); + var requestSpec = setupRequest(requestWithEnv); + var response = executeRequest(setupRequest, requestSpec); + assertSetup(setupRequest, requestWithEnv, response); + } + + for (TestItem testItem : test.tests()) { + + var requestWithEnv = testItem.request().withEnvironment(env); + var requestSpec = setupRequest(requestWithEnv); + var response = executeRequest(testItem.request(), requestSpec); + + testAssertions(testItem, response); + } + } + + protected void createKeyspace(String keyspace) { + String json = + """ + { + "createKeyspace": { + "name": "%s" + } + } + """ + .formatted(keyspace); + + jsonRequest() + .body(json) + .when() + .post(GeneralResource.BASE_PATH) + .then() + .statusCode(200) + .body("$", responseIsDDLSuccess()) + .body("status.ok", is(1)); + } + + private RequestSpecification setupRequest(ObjectNode request) { + + String requestString; + try { + requestString = OBJECT_MAPPER.writeValueAsString(request); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + + return jsonRequest() + .body(requestString).when(); + } + + private ValidatableResponse executeRequest( + TestRequest testRequest, RequestSpecification requestSpec) { + + if (testRequest.commandName().getTargets().contains(CommandTarget.COLLECTION) || testRequest.commandName().getTargets().contains(CommandTarget.TABLE)){ + return requestSpec + .post(CollectionResource.BASE_PATH, keyspaceName, env.vars().get("COLLECTION_NAME")) + .then() + .log().all(); + } + + if (testRequest.commandName().getTargets().contains(CommandTarget.KEYSPACE) ){ + return requestSpec.post(KeyspaceResource.BASE_PATH, keyspaceName).then().log().all(); + } + throw new IllegalArgumentException("Do not know how to execute command: " + testRequest.commandName()); + + } + + private void assertSetup( + TestRequest testRequest, ObjectNode requestWithEnv, ValidatableResponse response) { + response.statusCode(200); + + switch (testRequest.commandName()) { + case INSERT_ONE, INSERT_MANY -> { + response + .body("$", responseIsWriteSuccess()) + .body("status.insertedIds[0]", not(emptyString())); + } + case DELETE_COLLECTION, CREATE_COLLECTION -> { + response + .body("$", responseIsDDLSuccess()) + .body("status.ok", is(1)); + } + } + } + + private void testAssertions (TestItem testItem, ValidatableResponse response) { + + for (Map.Entry entry : testItem.asserts().properties()){ + var args = entry.getValue(); // null / 3 / {...} + + var assertFactory = findAssertionFactory(entry.getKey()); + ITAssertion itAssertion; + try { + itAssertion = (ITAssertion)assertFactory.invoke(null, args); + } catch (IllegalAccessException | InvocationTargetException e) { + throw new RuntimeException(e); + } + + LOGGER.info("XXX Running test assertion key={}, value={}", entry.getKey(), entry.getValue().toString()); + response.body(itAssertion.bodyPath(), itAssertion.matcher()); + } + } + + private Method findAssertionFactory(String key){ + // "response.isFindSuccess" + + int dot = key.indexOf('.'); + String typeName = key.substring(0, dot); + String funcName = key.substring(dot + 1); + + String qualifiedTypeName = + "io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions." + + Character.toUpperCase(typeName.charAt(0)) + + typeName.substring(1).toLowerCase(); + + try { + Class cls = Class.forName(qualifiedTypeName); + + var factoryMethod = Arrays.stream(cls.getMethods()) + .filter(m -> m.getName().equalsIgnoreCase(funcName)) + .filter(m -> Modifier.isStatic(m.getModifiers())) + .findFirst() + .orElseThrow(); + + + return factoryMethod; + } catch (ReflectiveOperationException e) { + throw new RuntimeException("Invalid assertion: " + key, e); + } + } +} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/IntegrationWorkflow.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/IntegrationWorkflow.java new file mode 100644 index 0000000000..cc91079094 --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/IntegrationWorkflow.java @@ -0,0 +1,11 @@ +package io.stargate.sgv2.jsonapi.api.v1.vectorize; + +import java.util.List; + +public record IntegrationWorkflow(ITMetadata meta, List jobs) implements ITElement { + + @Override + public ITElementKind kind() { + return ITElementKind.WORKFLOW; + } +} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/RunnerBase.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/RunnerBase.java new file mode 100644 index 0000000000..9f1bb29dab --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/RunnerBase.java @@ -0,0 +1,34 @@ +package io.stargate.sgv2.jsonapi.api.v1.vectorize; + +import io.restassured.http.ContentType; +import io.restassured.specification.RequestSpecification; +import io.stargate.sgv2.jsonapi.config.constants.HttpConstants; + +import java.util.Map; + +import static io.restassured.RestAssured.given; + +public abstract class RunnerBase { + + protected abstract IntegrationEnv integrationEnv(); + + protected Map getHeaders() { + + var env = integrationEnv(); + return Map.of( + HttpConstants.AUTHENTICATION_TOKEN_HEADER_NAME, + env.requiredValue("Token"), + HttpConstants.EMBEDDING_AUTHENTICATION_TOKEN_HEADER_NAME, + env.requiredValue("x-embedding-api-key")); + } + + public RequestSpecification jsonRequest(){ + + return given() + .log().all() + .port(8181) + .headers(getHeaders()) + .contentType(ContentType.JSON); + + } +} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestAssertion.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestAssertion.java new file mode 100644 index 0000000000..ea5ad4793f --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestAssertion.java @@ -0,0 +1,5 @@ +package io.stargate.sgv2.jsonapi.api.v1.vectorize; + +import com.fasterxml.jackson.databind.JsonNode; + +public record TestAssertion(String name, JsonNode assertion) {} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestItem.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestItem.java new file mode 100644 index 0000000000..05c9745fe8 --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestItem.java @@ -0,0 +1,12 @@ +package io.stargate.sgv2.jsonapi.api.v1.vectorize; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.node.ObjectNode; + +public record TestItem( + + String name, + TestRequest request, + ObjectNode asserts, + @JsonProperty("$include") + String include) {} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestRequest.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestRequest.java new file mode 100644 index 0000000000..75d1acc258 --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestRequest.java @@ -0,0 +1,89 @@ +package io.stargate.sgv2.jsonapi.api.v1.vectorize; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.fasterxml.jackson.databind.node.TextNode; +import io.stargate.sgv2.jsonapi.api.model.command.CommandName; + +import java.util.List; +import java.util.UUID; +import org.apache.commons.text.StringSubstitutor; + +public record TestRequest(@JsonIgnore UUID requestId, ObjectNode request) { + + @JsonCreator(mode = JsonCreator.Mode.DELEGATING) + public TestRequest(ObjectNode request) { + this(UUID.randomUUID(), request); + } + + public CommandName commandName() { + var requestCommandName = commandNameString(); + for (CommandName name : CommandName.values()) { + if (name.getApiName().equals(requestCommandName)) { + return name; + } + } + throw new IllegalArgumentException("Unknown command name: " + requestCommandName); + } + + private String commandNameString() { + var it = request.fieldNames(); + if (!it.hasNext()) { + throw new IllegalStateException("Expected exactly one field, found none"); + } + String name = it.next(); + if (it.hasNext()) { + throw new IllegalStateException("Expected exactly one field, found multiple"); + } + return name; + } + + public ObjectNode withEnvironment(IntegrationEnv env) { + ObjectNode updated = request.deepCopy(); + var subs = new StringSubstitutor(env.vars()).setEnableUndefinedVariableException(true); + + walk(updated, subs); + return updated; + } + + private static void walk(ObjectNode obj, StringSubstitutor subs) { + obj.properties() + .forEach( + (entry) -> { + switch (entry.getValue()) { + case TextNode text -> { + obj.put(entry.getKey(), subs.replace(text.textValue())); + } + case ObjectNode nested -> { + walk(nested, subs); + } + case ArrayNode arr -> { + walk(arr, subs); + } + default -> {} + } + }); + } + + private static void walk(ArrayNode arr, StringSubstitutor subs) { + for (int i = 0; i < arr.size(); i++) { + var child = arr.get(i); + + switch (child) { + case TextNode text -> { + String value = subs.replace(text.textValue()); + arr.set(i, TextNode.valueOf(value)); + } + case ObjectNode nested -> { + walk(nested, subs); + } + case ArrayNode nestedArr -> { + walk(nestedArr, subs); + } + default -> {} + } + } + } +} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/VectorizeIT.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/VectorizeIT.java new file mode 100644 index 0000000000..552eafb5af --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/VectorizeIT.java @@ -0,0 +1,24 @@ +package io.stargate.sgv2.jsonapi.api.v1.vectorize; + +import io.quarkus.test.common.WithTestResource; +import io.quarkus.test.junit.QuarkusIntegrationTest; +import io.stargate.sgv2.jsonapi.api.v1.AbstractCollectionIntegrationTestBase; +import io.stargate.sgv2.jsonapi.testresource.DseTestResource; +import org.junit.jupiter.api.Test; + +@QuarkusIntegrationTest +@WithTestResource(value = DseTestResource.class) +public class VectorizeIT extends AbstractCollectionIntegrationTestBase { + + @Test + public void doTest() { + + var itCollection = ITCollection.loadAll("integration-tests/vectorize"); + + var workflow = itCollection.workflowFirstByName("all-vectorize-workflow"); + + var job = workflow.jobs().getFirst(); + + new IntegrationJobRunner(itCollection, job).run(); + } +} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/VectorizeUnit.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/VectorizeUnit.java new file mode 100644 index 0000000000..dc25915474 --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/VectorizeUnit.java @@ -0,0 +1,22 @@ +package io.stargate.sgv2.jsonapi.api.v1.vectorize; + +import io.quarkus.test.common.WithTestResource; +import io.quarkus.test.junit.QuarkusIntegrationTest; +import io.stargate.sgv2.jsonapi.api.v1.AbstractCollectionIntegrationTestBase; +import io.stargate.sgv2.jsonapi.testresource.DseTestResource; +import org.junit.jupiter.api.Test; + +public class VectorizeUnit { + + @Test + public void doTest() { + + var itCollection = ITCollection.loadAll("integration-tests/vectorize"); + + var workflow = itCollection.workflowFirstByName("all-vectorize-workflow"); + + var job = workflow.jobs().getLast(); + + new IntegrationJobRunner(itCollection, job).run(); + } +} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/AssertionFactory.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/AssertionFactory.java new file mode 100644 index 0000000000..78538e2ee4 --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/AssertionFactory.java @@ -0,0 +1,12 @@ +package io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions; + +import com.fasterxml.jackson.databind.JsonNode; +import org.hamcrest.TypeSafeMatcher; + +import java.util.Map; + +@FunctionalInterface +public interface AssertionFactory { + + ITAssertion create(JsonNode args); +} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Documents.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Documents.java new file mode 100644 index 0000000000..80d156e258 --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Documents.java @@ -0,0 +1,18 @@ +package io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions; + +import com.fasterxml.jackson.databind.JsonNode; +import org.hamcrest.Description; +import org.hamcrest.Matcher; +import org.hamcrest.TypeSafeMatcher; + +import java.util.Map; + +import static org.hamcrest.Matchers.hasSize; + +public class Documents { + + public static ITAssertion count(JsonNode args) { + var expectedCount = args.asInt(); + return new ITAssertion("data.documents", hasSize(expectedCount)); + } +} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/ITAssertion.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/ITAssertion.java new file mode 100644 index 0000000000..41615dd781 --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/ITAssertion.java @@ -0,0 +1,11 @@ +package io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions; + +import org.hamcrest.Matcher; +import org.hamcrest.TypeSafeMatcher; + +import java.util.Map; + +public record ITAssertion( + String bodyPath, + Matcher matcher +) {} \ No newline at end of file diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Response.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Response.java new file mode 100644 index 0000000000..385495f628 --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Response.java @@ -0,0 +1,13 @@ +package io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions; + +import com.fasterxml.jackson.databind.JsonNode; + +import static io.stargate.sgv2.jsonapi.api.v1.ResponseAssertions.responseIsFindSuccess; +import static org.hamcrest.Matchers.hasSize; + +public class Response { + + public static ITAssertion isFindSuccess(JsonNode args) { + return new ITAssertion("$", responseIsFindSuccess()); + } +} diff --git a/src/test/resources/integration-tests/vectorize/open-ai.json b/src/test/resources/integration-tests/vectorize/open-ai.json new file mode 100644 index 0000000000..5fcd7b6bf1 --- /dev/null +++ b/src/test/resources/integration-tests/vectorize/open-ai.json @@ -0,0 +1,30 @@ +{ + "meta": { + "name": "vectorize-header-auth", + "kind" : "test" + }, + "setup": [ + { + "createCollection": { + "name": "${COLLECTION_NAME}", + "options": { + "vector": { + "metric": "cosine", + "service": { + "provider": "${PROVIDER}", + "modelName": "${MODEL}" + } + } + } + } + }, + { + "$include": "vectorize-base" + } + ], + "tests": [ + { + "$include": "vectorize-base" + } + ] +} \ No newline at end of file diff --git a/src/test/resources/integration-tests/vectorize/vectorize-base.json b/src/test/resources/integration-tests/vectorize/vectorize-base.json new file mode 100644 index 0000000000..a712710122 --- /dev/null +++ b/src/test/resources/integration-tests/vectorize/vectorize-base.json @@ -0,0 +1,83 @@ +{ + "meta": { + "name": "vectorize-base", + "kind" : "test" + }, + "setup": [ + { + "insertOne": { + "document": { + "_id": "Inception", + "name": "Inception", + "genre": "Science Fiction", + "artist": [ + "Leonardo DiCaprio" + ], + "$vectorize": "Inception is a science fiction action film about a professional thief who steals information by infiltrating the subconscious, entering people's dreams. He is offered a chance to have his criminal history erased as payment for implanting another person's idea into a target's subconscious." + } + } + }, + { + "insertMany": { + "documents": [ + { + "_id": "The Shawshank Redemption", + "name": "The Shawshank Redemption", + "genre": "Drama", + "artist": [ + "Tim Robbins", + "Morgan Freeman" + ], + "$vectorize": "The Shawshank Redemption is a drama film about a banker who is sentenced to life in Shawshank State Penitentiary for the murder of his wife and her lover. He forms a bond with a fellow prisoner and manages to maintain his dignity and sense of hope in the face of unjust treatment." + }, + { + "_id": "The Godfather", + "name": "The Godfather", + "genre": "Crime", + "artist": [ + "Marlon Brando", + "Al Pacino" + ], + "$vectorize": "The Godfather is a crime film about the aging patriarch of an organized crime dynasty who transfers control of his clandestine empire to his reluctant son. The film follows the family under the patriarch's youngest son, Michael, as he becomes increasingly involved in the family business." + } + ] + } + } + ], + "tests": [ + { + "name": "findByVectorize", + "request": { + "find": { + "sort": { + "$vectorize": "I love movies!" + } + } + }, + "asserts": { + "response.isFindSuccess": null, + "documents.count": 3 + } + }, + { + "name": "findOneAndUpdate", + "request": { + "findOneAndUpdate": { + "sort": { + "$vectorize": "Inception is a science fiction action film" + }, + "update": { + "$set": { + "status": "active" + } + }, + "options": { + "returnDocument": "after" + } + } + }, + "asserts": { + } + } + ] +} \ No newline at end of file diff --git a/src/test/resources/integration-tests/vectorize/vectorize-workflow.json b/src/test/resources/integration-tests/vectorize/vectorize-workflow.json new file mode 100644 index 0000000000..ce364450ff --- /dev/null +++ b/src/test/resources/integration-tests/vectorize/vectorize-workflow.json @@ -0,0 +1,56 @@ +{ + "meta": { + "name": "all-vectorize-workflow", + "kind": "workflow" + }, + "jobs": [ + { + "meta": { + "name": "open-ai-vectorize" + }, + "fromEnvironment": { + "x-embedding-api-key": "x-embedding-api-key" + }, + "variables": { + "Token": "Cassandra:Y2Fzc2FuZHJh:Y2Fzc2FuZHJh", + "PROVIDER": "openai", + "COLLECTION_NAME": "${PROVIDER}-${MODEL}" + }, + "matrix": { + "MODEL": [ + "text-embedding-3-small", + "text-embedding-3-large", + "text-embedding-ada-002" + ] + }, + "tests": [ + "vectorize-header-auth" + ] + }, + { + "meta": { + "name": "voyageAI-vectorize" + }, + "fromEnvironment": { + "x-embedding-api-key": "voyageAI_KEY" + }, + "variables": { + "Token": "Cassandra:Y2Fzc2FuZHJh:Y2Fzc2FuZHJh", + "PROVIDER": "voyageAI", + "COLLECTION_NAME": "${PROVIDER}-${MODEL}" + }, + "matrix": { + "MODEL": [ + "voyage-large-2-instruct", + "voyage-law-2", + "voyage-code-2", + "voyage-large-2", + "voyage-2" + ] + }, + "tests": [ + "vectorize-header-auth" + ] + } + ] +} \ No newline at end of file From b89dd30cf1732b881b9e50c3ab7439b8c9dd008d Mon Sep 17 00:00:00 2001 From: Aaron Morton Date: Thu, 19 Feb 2026 12:56:55 +1300 Subject: [PATCH 02/89] WIP --- src/main/resources/application.yaml | 2 +- .../jsonapi/api/v1/vectorize/APIRequest.java | 126 ++++++++++++++++++ .../api/v1/vectorize/AstraBackend.java | 4 + .../jsonapi/api/v1/vectorize/Backend.java | 17 +++ .../api/v1/vectorize/CassandraBackend.java | 47 +++++++ .../jsonapi/api/v1/vectorize/Connection.java | 19 +++ .../api/v1/vectorize/ITCollection.java | 6 +- .../jsonapi/api/v1/vectorize/ITMetadata.java | 5 +- .../api/v1/vectorize/IntegrationEnv.java | 87 +++++++++--- .../api/v1/vectorize/IntegrationJob.java | 56 ++++---- .../v1/vectorize/IntegrationJobRunner.java | 29 +++- .../api/v1/vectorize/IntegrationTarget.java | 42 ++++++ .../api/v1/vectorize/IntegrationTest.java | 6 +- .../v1/vectorize/IntegrationTestRunner.java | 90 ++----------- .../jsonapi/api/v1/vectorize/RunnerBase.java | 19 --- .../sgv2/jsonapi/api/v1/vectorize/Target.java | 4 + .../jsonapi/api/v1/vectorize/Targets.java | 61 +++++++++ .../jsonapi/api/v1/vectorize/TestRequest.java | 17 ++- .../jsonapi/api/v1/vectorize/VectorizeIT.java | 14 +- .../api/v1/vectorize/VectorizeUnit.java | 51 ++++++- .../integration-tests/targets/targets.json | 22 +++ ...pen-ai.json => vectorize-header-auth.json} | 9 +- .../vectorize/vectorize-workflow.json | 34 ++++- 23 files changed, 594 insertions(+), 173 deletions(-) create mode 100644 src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/APIRequest.java create mode 100644 src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/AstraBackend.java create mode 100644 src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/Backend.java create mode 100644 src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/CassandraBackend.java create mode 100644 src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/Connection.java create mode 100644 src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/IntegrationTarget.java create mode 100644 src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/Target.java create mode 100644 src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/Targets.java create mode 100644 src/test/resources/integration-tests/targets/targets.json rename src/test/resources/integration-tests/vectorize/{open-ai.json => vectorize-header-auth.json} (79%) diff --git a/src/main/resources/application.yaml b/src/main/resources/application.yaml index 8435838930..6c421866a6 100644 --- a/src/main/resources/application.yaml +++ b/src/main/resources/application.yaml @@ -16,7 +16,7 @@ stargate: database: limits: - max-collections: 5 + max-collections: 15 debug: enabled: false diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/APIRequest.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/APIRequest.java new file mode 100644 index 0000000000..b24baa78aa --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/APIRequest.java @@ -0,0 +1,126 @@ +package io.stargate.sgv2.jsonapi.api.v1.vectorize; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import io.restassured.http.ContentType; +import io.restassured.response.Response; +import io.restassured.response.ValidatableResponse; +import io.restassured.specification.RequestSpecification; +import io.stargate.sgv2.jsonapi.api.model.command.CommandTarget; +import io.stargate.sgv2.jsonapi.api.v1.CollectionResource; +import io.stargate.sgv2.jsonapi.api.v1.KeyspaceResource; +import io.stargate.sgv2.jsonapi.config.constants.HttpConstants; + +import java.util.Map; + +import static io.restassured.RestAssured.given; +import static io.stargate.sgv2.jsonapi.api.v1.ResponseAssertions.responseIsDDLSuccess; +import static io.stargate.sgv2.jsonapi.api.v1.ResponseAssertions.responseIsWriteSuccess; +import static org.hamcrest.Matchers.*; + + +public class APIRequest { + + public static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + + private static String COLLECTION_PATH = "/{keyspace}/{collection}"; + private static String KEYSPACE_PATH = "/{keyspace}"; + private static String DB_PATH = "/"; + + private final Connection connection; + private final TestRequest testRequest; + private final IntegrationEnv env; + + public APIRequest( Connection connection, TestRequest testRequest, IntegrationEnv env ) { + this.connection = connection; + this.testRequest = testRequest; + this.env = env; + } + + public ValidatableResponse execute(){ + + var requestWithEnv = testRequest.withEnvironment(env); + var requestSpec = requestSpec(requestWithEnv); + return executeRequest(requestSpec); + } + + public ValidatableResponse executeWithSuccess(){ + var resp = execute(); + assertSuccess(resp); + return resp; + } + + private RequestSpecification requestSpec(ObjectNode request) { + + String requestString; + try { + requestString = OBJECT_MAPPER.writeValueAsString(request); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + + return jsonRequest() + .body(requestString).when(); + } + + private ValidatableResponse executeRequest(RequestSpecification requestSpec) { + + var commandName = testRequest.commandName(); + Response response; + if (commandName.getTargets().contains(CommandTarget.COLLECTION) || commandName.getTargets().contains(CommandTarget.TABLE)){ + response = requestSpec + .post(COLLECTION_PATH, env.requiredValue("KEYSPACE_NAME"), env.requiredValue("COLLECTION_NAME")); + } + else if (commandName.getTargets().contains(CommandTarget.KEYSPACE) ){ + response = requestSpec.post(KEYSPACE_PATH, env.requiredValue("KEYSPACE_NAME")); + } + else if(commandName.getTargets().contains(CommandTarget.DATABASE)){ + response = requestSpec.post(DB_PATH); + } + else { + throw new IllegalArgumentException("Do not know how to execute command: " + testRequest.commandName()); + } + + return response + .then().log().all(); + } + + protected Map getHeaders() { + + return Map.of( + HttpConstants.AUTHENTICATION_TOKEN_HEADER_NAME, + env.requiredValue("Token"), + HttpConstants.EMBEDDING_AUTHENTICATION_TOKEN_HEADER_NAME, + env.requiredValue("x-embedding-api-key")); + } + + public RequestSpecification jsonRequest(){ + + return given() + .log().all() + .baseUri(connection.domain()) + .port(connection.port()) + .basePath(connection.basePath()) + .headers(getHeaders()) + .contentType(ContentType.JSON); + + } + + private void assertSuccess(ValidatableResponse response) { + response.statusCode(200); + + switch (testRequest.commandName()) { + case INSERT_ONE, INSERT_MANY -> { + response + .body("$", responseIsWriteSuccess()) + .body("status.insertedIds[0]", not(emptyString())); + } + case DELETE_COLLECTION, CREATE_COLLECTION -> { + response + .body("$", responseIsDDLSuccess()) + .body("status.ok", is(1)); + } + } + } +} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/AstraBackend.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/AstraBackend.java new file mode 100644 index 0000000000..5169e824c1 --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/AstraBackend.java @@ -0,0 +1,4 @@ +package io.stargate.sgv2.jsonapi.api.v1.vectorize; + +public class AstraBackend extends Backend { +} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/Backend.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/Backend.java new file mode 100644 index 0000000000..7aa27d4fdf --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/Backend.java @@ -0,0 +1,17 @@ +package io.stargate.sgv2.jsonapi.api.v1.vectorize; + +public abstract class Backend { + + public void workflowStarting(IntegrationTarget integrationTarget, IntegrationWorkflow workflow) { } + + public void workflowFinished(IntegrationTarget integrationTarget, IntegrationWorkflow workflow) { } + + public void jobStarting(IntegrationTarget integrationTarget, IntegrationJob job) { } + + public void jobFinished(IntegrationTarget integrationTarget, IntegrationJob job) { } + + public void testStarting(IntegrationTarget integrationTarget, IntegrationTest test, IntegrationEnv env) { } + + public void testFinished(IntegrationTarget integrationTarget, IntegrationTest test, IntegrationEnv env) { } +} + diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/CassandraBackend.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/CassandraBackend.java new file mode 100644 index 0000000000..184d67f366 --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/CassandraBackend.java @@ -0,0 +1,47 @@ +package io.stargate.sgv2.jsonapi.api.v1.vectorize; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.commons.lang3.RandomStringUtils; + +import static io.stargate.sgv2.jsonapi.api.v1.vectorize.IntegrationEnv.toSafeSchemaIdentifier; + +public class CassandraBackend extends Backend { + + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + + + @Override + public void jobStarting(IntegrationTarget integrationTarget, IntegrationJob job) { + // max length for keyspace is 48 chars + + var keyspaceName = toSafeSchemaIdentifier( + "job_" + job.meta().name().substring(0, Math.min(job.meta().name().length(), 27)) + "_" + RandomStringUtils.insecure().nextAlphanumeric(16)); + + var request = TestRequest.fromJson( + """ + { + "createKeyspace": { + "name": "${KEYSPACE_NAME}" + } + } + """); + + job.variables().put("KEYSPACE_NAME", keyspaceName); + integrationTarget.apiRequest(request, job.withoutMatrix()).executeWithSuccess(); + + } + + @Override + public void jobFinished(IntegrationTarget integrationTarget, IntegrationJob job) { + var request = TestRequest.fromJson( + """ + { + "dropKeyspace": { + "name": "${KEYSPACE_NAME}" + } + } + """); + + integrationTarget.apiRequest(request, job.withoutMatrix()).executeWithSuccess(); + } +} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/Connection.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/Connection.java new file mode 100644 index 0000000000..f21099f4fa --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/Connection.java @@ -0,0 +1,19 @@ +package io.stargate.sgv2.jsonapi.api.v1.vectorize; + +public record Connection( + String domain, + Integer port, + String basePath +) { + public Connection { + if (domain == null) { + domain = "localhost"; + } + if (port == null) { + port = 8181; + } + if (basePath == null) { + basePath = "/v1"; + } + } +} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/ITCollection.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/ITCollection.java index 44268839cb..ba7c84a994 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/ITCollection.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/ITCollection.java @@ -53,11 +53,11 @@ public IntegrationWorkflow workflowFirstByName(String name) { return (IntegrationWorkflow) itFile.element(); } - public IntegrationTest testsFirstByName(String name) { - return testsByName(name).getFirst(); + public IntegrationTest testFirstByName(String name) { + return testByName(name).getFirst(); } - public List testsByName(String name) { + public List testByName(String name) { var path = "$.meta[?(@.name == '%s')]".formatted(name); return match(ITElement.ITElementKind.TEST, path).stream() diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/ITMetadata.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/ITMetadata.java index 828a6dfd77..b37690f5bb 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/ITMetadata.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/ITMetadata.java @@ -1,3 +1,6 @@ package io.stargate.sgv2.jsonapi.api.v1.vectorize; -public record ITMetadata(String name, String kind) {} +import java.util.List; + +public record ITMetadata(String name, String kind, + List tags) {} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/IntegrationEnv.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/IntegrationEnv.java index f8346c2c42..57a99c10b6 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/IntegrationEnv.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/IntegrationEnv.java @@ -1,35 +1,90 @@ package io.stargate.sgv2.jsonapi.api.v1.vectorize; +import org.apache.commons.text.StringSubstitutor; +import org.apache.commons.text.lookup.StringLookup; +import org.apache.commons.text.lookup.StringLookupFactory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; +import java.util.regex.Pattern; + +public class IntegrationEnv{ -public record IntegrationEnv(Map vars) { private static final Logger LOGGER = LoggerFactory.getLogger(IntegrationEnv.class); + private static final Pattern PATTERN_NOT_WORD_CHARS = Pattern.compile("\\W+"); - private static final List SCHEMA_IDENTIFIER = List.of("COLLECTION_NAME"); + private static final Set SCHEMA_IDENTIFIER = Set.of("KEYSPACE_NAME", "COLLECTION_NAME"); - public IntegrationEnv { + private final Map vars = new HashMap<>(); - for (var name : SCHEMA_IDENTIFIER) { - if (vars.containsKey(name)) { - var oldValue = vars.get(name); - var newValue = oldValue.replaceAll("[^A-Za-z0-9_]", "_"); + public IntegrationEnv(){ + this(new HashMap<>()); + } - if (newValue.length() > 48){ - throw new RuntimeException("Schema Identifier longer than 48 characters %s=%s".formatted(name,newValue)); - } - LOGGER.info("XXX Updated IntegrationEnv value because it is a Schema Identifier key={}, oldValue={}, newValue={}",name,oldValue,newValue); - vars.put(name, newValue); - } - } + public IntegrationEnv(Map vars) { + this.vars.putAll(vars); + } + + private IntegrationEnv(IntegrationEnv other){ + this.vars.putAll(other.vars); + } + + public IntegrationEnv clone(){ + return new IntegrationEnv(this); } + + public IntegrationEnv put(IntegrationEnv other){ + this.vars.putAll(other.vars); + return this; + } + + public void put(String key, String value){ + this.vars.put(key, value); + } + public String requiredValue(String name){ if (vars.containsKey(name)){ - return vars.get(name); + return get(name); } - throw new RuntimeException(String.format("Required parameter %s not found", name)); + throw new RuntimeException(String.format("Required env var %s not found", name)); + } + + public StringSubstitutor substitutor(){ + + return new StringSubstitutor(StringLookupFactory.INSTANCE.functionStringLookup(this::get)).setEnableUndefinedVariableException(true); + } + private String get(String name){ + + var value = vars.get(name); + if (value == null){ + return ""; + } + + var substituted = substitutor().replace(value); + var cleaned = SCHEMA_IDENTIFIER.contains(name) ? + toSafeSchemaIdentifier(substituted) + : + substituted; + + return cleaned; + } + + + public static String toSafeSchemaIdentifier(String name){ + + var newValue = PATTERN_NOT_WORD_CHARS.matcher(name).replaceAll("_"); + if (newValue.length() > 48){ + throw new RuntimeException("Schema Identifier longer than 48 characters %s=%s".formatted(name,newValue)); + } + return newValue; + } + + @Override + public String toString() { + return vars.toString(); } } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/IntegrationJob.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/IntegrationJob.java index cc34b490cb..c161cd3f88 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/IntegrationJob.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/IntegrationJob.java @@ -19,9 +19,23 @@ public record IntegrationJob( private static final Logger LOGGER = LoggerFactory.getLogger(IntegrationJob.class); + public IntegrationEnv withoutMatrix(){ + var fromEnv = new IntegrationEnv(); + for (Map.Entry entry : fromEnvironment.entrySet()) { + var value = System.getenv(entry.getValue()); + if (value== null) { + throw new RuntimeException("Environment variable " + entry.getValue() + " is undefined"); + } + fromEnv.put(entry.getKey(), value); + } + + var fromVariables = new IntegrationEnv(variables); + + return fromEnv.clone().put(fromVariables); + } public List allEnvironments() { - Map fromEnv = new HashMap<>(); + var fromEnv = new IntegrationEnv(); for (Map.Entry entry : fromEnvironment.entrySet()) { var value = System.getenv(entry.getValue()); if (value== null) { @@ -30,33 +44,27 @@ public List allEnvironments() { fromEnv.put(entry.getKey(), value); } - // TODO: handle more matrix - var modelList = matrix.get("MODEL"); + var fromVariables = new IntegrationEnv(variables); - List> allMatrix = new ArrayList<>(); - modelList.forEach( + // TODO: handle more matrix + List fromMatrix = new ArrayList<>(); + matrix.get("MODEL").forEach( model -> { - var env = new HashMap(); + var env = new IntegrationEnv(); env.put("MODEL", model); - allMatrix.add(env); + fromMatrix.add(env); }); - List envs = new ArrayList<>(); - allMatrix.forEach( - matrix -> { - matrix.putAll(variables); - matrix.putAll(fromEnv); - - LOGGER.info("XXX ENV BEFORE SUBS {}", matrix); - var subs = new StringSubstitutor(matrix).setEnableUndefinedVariableException(true); - matrix.forEach((key, value) -> { - matrix.put(key, subs.replace(value)); - }); - - LOGGER.info("XXX ENV AFTER SUBS {}", matrix); - var env = new IntegrationEnv(matrix); - envs.add(env); - }); - return envs; + List allEnvs = new ArrayList<>(); + for (var matrixEnv : fromMatrix) { + + var completeEnv = fromEnv + .clone() + .put(fromVariables) + .put(matrixEnv); + allEnvs.add(completeEnv); + } + + return allEnvs; } } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/IntegrationJobRunner.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/IntegrationJobRunner.java index c93ad332d5..ed2c03fd2d 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/IntegrationJobRunner.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/IntegrationJobRunner.java @@ -5,30 +5,47 @@ public class IntegrationJobRunner { + private final IntegrationTarget target; private final ITCollection itCollection; private final IntegrationJob job; - public IntegrationJobRunner(ITCollection itCollection, IntegrationJob job) { + public IntegrationJobRunner( IntegrationTarget target, ITCollection itCollection, IntegrationJob job) { + this.target = target; this.itCollection = itCollection; this.job = job; } public void run() { + target.jobStarting(job); var allEnvs = job.allEnvironments(); List allTests = new ArrayList<>(); job.tests() .forEach( testName -> { - allTests.addAll(itCollection.testsByName(testName)); + allTests.addAll(itCollection.testByName(testName)); }); - for (IntegrationTest test : allTests) { - for (IntegrationEnv env : allEnvs) { - var testRunner = new IntegrationTestRunner(itCollection, test, env); - testRunner.run(); + try{ + + for (IntegrationTest test : allTests) { + for (IntegrationEnv env : allEnvs) { + + try{ + target.testStarting(test, env); + var testRunner = new IntegrationTestRunner( itCollection, target, test, env); + testRunner.run(); + } + finally{ + target.testFinished(test, env); + } + + } } } + finally { + target.jobFinished(job); + } } } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/IntegrationTarget.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/IntegrationTarget.java new file mode 100644 index 0000000000..32cbe8ded2 --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/IntegrationTarget.java @@ -0,0 +1,42 @@ +package io.stargate.sgv2.jsonapi.api.v1.vectorize; + +import java.util.HashMap; + +public class IntegrationTarget { + + private final Target target; + private final Backend backend; + private final IntegrationEnv env; + + public IntegrationTarget(Target target) { + this.target = target; + this.env = new IntegrationEnv(new HashMap<>()); + + this.backend = switch (target.backend()) { + case "cassandra" -> new CassandraBackend(); + case "astra" -> new AstraBackend(); + default -> throw new IllegalArgumentException("Unknown backend: " + target.backend()); + }; + } + + public Connection connection(){ + return target.connection(); + } + + public APIRequest apiRequest(TestRequest testRequest, IntegrationEnv env){ + return new APIRequest(target.connection(), testRequest, env); + } + + public void workflowStarting(IntegrationWorkflow workflow){ } + public void workflowFinished(IntegrationWorkflow workflow){ } + + public void jobStarting(IntegrationJob job){ + backend.jobStarting(this, job); + } + public void jobFinished(IntegrationJob job){ + backend.jobFinished(this, job); + } + + public void testStarting(IntegrationTest test, IntegrationEnv env){ } + public void testFinished(IntegrationTest test, IntegrationEnv env){ } +} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/IntegrationTest.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/IntegrationTest.java index b994f46bb2..b05758be57 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/IntegrationTest.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/IntegrationTest.java @@ -3,7 +3,7 @@ import java.util.ArrayList; import java.util.List; -public record IntegrationTest(ITMetadata meta, List setup, List tests) +public record IntegrationTest(ITMetadata meta, List setup, List tests, List cleanup) implements ITElement { @Override @@ -16,7 +16,7 @@ public void expand(ITCollection itCollection) { List expandedSetup = new ArrayList<>(); for (TestRequest request : setup) { if (request.request().has("$include")){ - var includedTest = itCollection.testsFirstByName(request.request().get("$include").textValue()); + var includedTest = itCollection.testFirstByName(request.request().get("$include").textValue()); expandedSetup.addAll(includedTest.setup()); } else { @@ -29,7 +29,7 @@ public void expand(ITCollection itCollection) { List expandedTests = new ArrayList<>(); for (TestItem item : tests) { if (item.include() != null ){ - var includedTest = itCollection.testsFirstByName(item.include()); + var includedTest = itCollection.testFirstByName(item.include()); expandedTests.addAll(includedTest.tests()); } else { diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/IntegrationTestRunner.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/IntegrationTestRunner.java index 5ac9b240b1..7f82340540 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/IntegrationTestRunner.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/IntegrationTestRunner.java @@ -34,7 +34,7 @@ import static org.hamcrest.Matchers.*; public class IntegrationTestRunner extends RunnerBase { - private static final Logger LOGGER = LoggerFactory.getLogger(IntegrationEnv.class); + private static final Logger LOGGER = LoggerFactory.getLogger(IntegrationTestRunner.class); // keyspace automatically created in this test @@ -43,12 +43,14 @@ public class IntegrationTestRunner extends RunnerBase { private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); private final ITCollection itCollection; + private final IntegrationTarget target; private final IntegrationTest test; private final IntegrationEnv env; public IntegrationTestRunner( - ITCollection itCollection, IntegrationTest test, IntegrationEnv env) { + ITCollection itCollection, IntegrationTarget target, IntegrationTest test, IntegrationEnv env) { this.itCollection = itCollection; + this.target = target; this.test = test; this.env = env; } @@ -60,94 +62,22 @@ protected IntegrationEnv integrationEnv() { public void run() { - createKeyspace(keyspaceName); - + LOGGER.info("Starting Integration Test with env={}", env); for (TestRequest setupRequest : test.setup()) { - - var requestWithEnv = setupRequest.withEnvironment(env); - var requestSpec = setupRequest(requestWithEnv); - var response = executeRequest(setupRequest, requestSpec); - assertSetup(setupRequest, requestWithEnv, response); + target.apiRequest(setupRequest, env).executeWithSuccess(); } for (TestItem testItem : test.tests()) { - var requestWithEnv = testItem.request().withEnvironment(env); - var requestSpec = setupRequest(requestWithEnv); - var response = executeRequest(testItem.request(), requestSpec); - - testAssertions(testItem, response); + var resp = target.apiRequest(testItem.request(), env).execute(); + testAssertions(testItem, resp); } - } - - protected void createKeyspace(String keyspace) { - String json = - """ - { - "createKeyspace": { - "name": "%s" - } - } - """ - .formatted(keyspace); - - jsonRequest() - .body(json) - .when() - .post(GeneralResource.BASE_PATH) - .then() - .statusCode(200) - .body("$", responseIsDDLSuccess()) - .body("status.ok", is(1)); - } - private RequestSpecification setupRequest(ObjectNode request) { - - String requestString; - try { - requestString = OBJECT_MAPPER.writeValueAsString(request); - } catch (JsonProcessingException e) { - throw new RuntimeException(e); + for (TestRequest setupRequest : test.cleanup()) { + target.apiRequest(setupRequest, env).executeWithSuccess(); } - - return jsonRequest() - .body(requestString).when(); } - private ValidatableResponse executeRequest( - TestRequest testRequest, RequestSpecification requestSpec) { - - if (testRequest.commandName().getTargets().contains(CommandTarget.COLLECTION) || testRequest.commandName().getTargets().contains(CommandTarget.TABLE)){ - return requestSpec - .post(CollectionResource.BASE_PATH, keyspaceName, env.vars().get("COLLECTION_NAME")) - .then() - .log().all(); - } - - if (testRequest.commandName().getTargets().contains(CommandTarget.KEYSPACE) ){ - return requestSpec.post(KeyspaceResource.BASE_PATH, keyspaceName).then().log().all(); - } - throw new IllegalArgumentException("Do not know how to execute command: " + testRequest.commandName()); - - } - - private void assertSetup( - TestRequest testRequest, ObjectNode requestWithEnv, ValidatableResponse response) { - response.statusCode(200); - - switch (testRequest.commandName()) { - case INSERT_ONE, INSERT_MANY -> { - response - .body("$", responseIsWriteSuccess()) - .body("status.insertedIds[0]", not(emptyString())); - } - case DELETE_COLLECTION, CREATE_COLLECTION -> { - response - .body("$", responseIsDDLSuccess()) - .body("status.ok", is(1)); - } - } - } private void testAssertions (TestItem testItem, ValidatableResponse response) { diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/RunnerBase.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/RunnerBase.java index 9f1bb29dab..c39a23a459 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/RunnerBase.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/RunnerBase.java @@ -12,23 +12,4 @@ public abstract class RunnerBase { protected abstract IntegrationEnv integrationEnv(); - protected Map getHeaders() { - - var env = integrationEnv(); - return Map.of( - HttpConstants.AUTHENTICATION_TOKEN_HEADER_NAME, - env.requiredValue("Token"), - HttpConstants.EMBEDDING_AUTHENTICATION_TOKEN_HEADER_NAME, - env.requiredValue("x-embedding-api-key")); - } - - public RequestSpecification jsonRequest(){ - - return given() - .log().all() - .port(8181) - .headers(getHeaders()) - .contentType(ContentType.JSON); - - } } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/Target.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/Target.java new file mode 100644 index 0000000000..6b458d9ea5 --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/Target.java @@ -0,0 +1,4 @@ +package io.stargate.sgv2.jsonapi.api.v1.vectorize; + +public record Target (String name, String backend, Connection connection) { +} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/Targets.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/Targets.java new file mode 100644 index 0000000000..bad582652f --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/Targets.java @@ -0,0 +1,61 @@ +package io.stargate.sgv2.jsonapi.api.v1.vectorize; + +import com.fasterxml.jackson.databind.ObjectMapper; + +import java.io.IOException; +import java.net.URISyntaxException; +import java.net.URL; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +public record Targets(List targets) { + + private static final ObjectMapper MAPPER = new ObjectMapper(); + + + public Targets{ + Set seen = new HashSet(); + for (Target target : targets) { + if (seen.contains(target.name())) { + throw new IllegalArgumentException("target name already exists: " + target.name() ); + } + seen.add(target.name()); + } + } + + public Target target(String name) { + return targets.stream().filter(target -> target.name().equals(name)).findFirst().orElseThrow(() -> new IllegalArgumentException("target name not found: " + name)); + } + + static Targets loadAll(String path) { + final Path dir = resourceDir(path); + + try { + return MAPPER.readValue(dir.toFile(), Targets.class); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + private static Path resourceDir(String path) { + String normalized = path.startsWith("/") ? path.substring(1) : path; + + ClassLoader cl = Thread.currentThread().getContextClassLoader(); + URL url = cl.getResource(normalized); + if (url == null) { + throw new IllegalArgumentException("Test resource folder not found: " + path); + } + + try { + // Works for file: URLs; if you run tests from a jar, switch to getResourceAsStream-based + // walking. + return Paths.get(url.toURI()); + } catch (URISyntaxException e) { + throw new IllegalArgumentException("Bad resource URI for: " + path + " -> " + url, e); + } + } + +} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestRequest.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestRequest.java index 75d1acc258..f3d939dc20 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestRequest.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestRequest.java @@ -2,6 +2,8 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.fasterxml.jackson.databind.node.TextNode; @@ -13,6 +15,8 @@ public record TestRequest(@JsonIgnore UUID requestId, ObjectNode request) { + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + @JsonCreator(mode = JsonCreator.Mode.DELEGATING) public TestRequest(ObjectNode request) { this(UUID.randomUUID(), request); @@ -42,12 +46,19 @@ private String commandNameString() { public ObjectNode withEnvironment(IntegrationEnv env) { ObjectNode updated = request.deepCopy(); - var subs = new StringSubstitutor(env.vars()).setEnableUndefinedVariableException(true); - - walk(updated, subs); + walk(updated, env.substitutor()); return updated; } + public static TestRequest fromJson(String json) { + + try { + return new TestRequest((ObjectNode) OBJECT_MAPPER.readTree(json)); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + + } private static void walk(ObjectNode obj, StringSubstitutor subs) { obj.properties() .forEach( diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/VectorizeIT.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/VectorizeIT.java index 552eafb5af..c8fb36e65c 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/VectorizeIT.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/VectorizeIT.java @@ -13,12 +13,12 @@ public class VectorizeIT extends AbstractCollectionIntegrationTestBase { @Test public void doTest() { - var itCollection = ITCollection.loadAll("integration-tests/vectorize"); - - var workflow = itCollection.workflowFirstByName("all-vectorize-workflow"); - - var job = workflow.jobs().getFirst(); - - new IntegrationJobRunner(itCollection, job).run(); +// var itCollection = ITCollection.loadAll("integration-tests/vectorize"); +// +// var workflow = itCollection.workflowFirstByName("all-vectorize-workflow"); +// +// var job = workflow.jobs().getFirst(); +// +// new IntegrationJobRunner(itCollection, job).run(); } } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/VectorizeUnit.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/VectorizeUnit.java index dc25915474..41e6856cbd 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/VectorizeUnit.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/VectorizeUnit.java @@ -1,22 +1,59 @@ package io.stargate.sgv2.jsonapi.api.v1.vectorize; -import io.quarkus.test.common.WithTestResource; -import io.quarkus.test.junit.QuarkusIntegrationTest; -import io.stargate.sgv2.jsonapi.api.v1.AbstractCollectionIntegrationTestBase; -import io.stargate.sgv2.jsonapi.testresource.DseTestResource; import org.junit.jupiter.api.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -public class VectorizeUnit { +public class VectorizeUnit { + private static final Logger LOGGER = LoggerFactory.getLogger(VectorizeUnit.class); @Test public void doTest() { + var targets = Targets.loadAll("integration-tests/targets/targets.json"); + var integrationTarget = new IntegrationTarget(targets.target("local")); + var itCollection = ITCollection.loadAll("integration-tests/vectorize"); var workflow = itCollection.workflowFirstByName("all-vectorize-workflow"); - var job = workflow.jobs().getLast(); + var jobs = workflow.jobs().stream() + .filter(job -> !job.meta().tags().contains("disabled")) + .toList(); + + + // SEQUENTIAL +// integrationTarget.workflowStarting(workflow); +// try { +// for (var job : jobs) { +// LOGGER.info("Starting job {}", job.meta()); +// new IntegrationJobRunner(integrationTarget, itCollection, job).run(); +// } +// } finally { +// integrationTarget.workflowFinished(workflow); +// } + + // Parallel + int maxConcurrentJobs = 2; + integrationTarget.workflowStarting(workflow); + try { + var executor = java.util.concurrent.Executors.newFixedThreadPool(maxConcurrentJobs); + try { + var futures = jobs.stream() + .map(job -> java.util.concurrent.CompletableFuture.runAsync(() -> { + LOGGER.info("Starting job {}", job.meta()); + new IntegrationJobRunner(integrationTarget, itCollection, job).run(); + }, executor)) + .toList(); + + // wait for all, fail if any failed + futures.forEach(java.util.concurrent.CompletableFuture::join); + } finally { + executor.shutdown(); + } + } finally { + integrationTarget.workflowFinished(workflow); + } - new IntegrationJobRunner(itCollection, job).run(); } } diff --git a/src/test/resources/integration-tests/targets/targets.json b/src/test/resources/integration-tests/targets/targets.json new file mode 100644 index 0000000000..97449b6732 --- /dev/null +++ b/src/test/resources/integration-tests/targets/targets.json @@ -0,0 +1,22 @@ +{ + "targets": [ + { + "name": "local", + "backend": "cassandra", + "connection": { + "domain": "http://localhost", + "port": 8181, + "basePath": "/v1" + } + }, + { + "name": "astra-dev", + "backend": "astra", + "connection": { + "domain": "http://localhost", + "port": 8181, + "basePath": "/v1/json" + } + } + ] +} \ No newline at end of file diff --git a/src/test/resources/integration-tests/vectorize/open-ai.json b/src/test/resources/integration-tests/vectorize/vectorize-header-auth.json similarity index 79% rename from src/test/resources/integration-tests/vectorize/open-ai.json rename to src/test/resources/integration-tests/vectorize/vectorize-header-auth.json index 5fcd7b6bf1..11d61833f1 100644 --- a/src/test/resources/integration-tests/vectorize/open-ai.json +++ b/src/test/resources/integration-tests/vectorize/vectorize-header-auth.json @@ -1,7 +1,7 @@ { "meta": { "name": "vectorize-header-auth", - "kind" : "test" + "kind": "test" }, "setup": [ { @@ -26,5 +26,12 @@ { "$include": "vectorize-base" } + ], + "cleanup": [ + { + "deleteCollection": { + "name": "${COLLECTION_NAME}" + } + } ] } \ No newline at end of file diff --git a/src/test/resources/integration-tests/vectorize/vectorize-workflow.json b/src/test/resources/integration-tests/vectorize/vectorize-workflow.json index ce364450ff..5164ed3804 100644 --- a/src/test/resources/integration-tests/vectorize/vectorize-workflow.json +++ b/src/test/resources/integration-tests/vectorize/vectorize-workflow.json @@ -6,7 +6,8 @@ "jobs": [ { "meta": { - "name": "open-ai-vectorize" + "name": "open-ai-vectorize", + "tags": [] }, "fromEnvironment": { "x-embedding-api-key": "x-embedding-api-key" @@ -29,7 +30,10 @@ }, { "meta": { - "name": "voyageAI-vectorize" + "name": "voyageAI-vectorize", + "tags": [ + "disabled" + ] }, "fromEnvironment": { "x-embedding-api-key": "voyageAI_KEY" @@ -51,6 +55,32 @@ "tests": [ "vectorize-header-auth" ] + }, + { + "meta": { + "name": "jinaAI-vectorize", + "tags": [] + }, + "fromEnvironment": { + "x-embedding-api-key": "jinaAI_KEY" + }, + "variables": { + "Token": "Cassandra:Y2Fzc2FuZHJh:Y2Fzc2FuZHJh", + "PROVIDER": "jinaAI", + "COLLECTION_NAME": "${PROVIDER}-${MODEL}" + }, + "matrix": { + "MODEL": [ + "jina-embeddings-v2-base-en", + "jina-embeddings-v2-base-de", + "jina-embeddings-v2-base-es", + "jina-embeddings-v2-base-code", + "jina-embeddings-v2-base-zh" + ] + }, + "tests": [ + "vectorize-header-auth" + ] } ] } \ No newline at end of file From f1307d55399ba7f65579c375181cd943d105f5d7 Mon Sep 17 00:00:00 2001 From: Aaron Morton Date: Fri, 20 Feb 2026 05:55:48 +1300 Subject: [PATCH 03/89] WIP --- .../jsonapi/api/v1/vectorize/APIResponse.java | 9 ++++ .../api/v1/vectorize/IntegrationTest.java | 6 +-- .../v1/vectorize/IntegrationTestRunner.java | 36 ++++++------- .../{TestItem.java => TestCase.java} | 2 +- .../jsonapi/api/v1/vectorize/TestResult.java | 16 ++++++ .../api/v1/vectorize/VectorizeUnit.java | 50 +++++++++---------- .../vectorize/vectorize-base.json | 2 +- .../vectorize/vectorize-workflow.json | 2 +- 8 files changed, 71 insertions(+), 52 deletions(-) create mode 100644 src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/APIResponse.java rename src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/{TestItem.java => TestCase.java} (92%) create mode 100644 src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestResult.java diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/APIResponse.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/APIResponse.java new file mode 100644 index 0000000000..d1b2346ea6 --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/APIResponse.java @@ -0,0 +1,9 @@ +package io.stargate.sgv2.jsonapi.api.v1.vectorize; + +import com.fasterxml.jackson.databind.node.ObjectNode; +import io.restassured.response.ValidatableResponse; + +public record APIResponse (APIRequest apiRequest, + ObjectNode actualRequest, + ValidatableResponse response) { +} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/IntegrationTest.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/IntegrationTest.java index b05758be57..844c220297 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/IntegrationTest.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/IntegrationTest.java @@ -3,7 +3,7 @@ import java.util.ArrayList; import java.util.List; -public record IntegrationTest(ITMetadata meta, List setup, List tests, List cleanup) +public record IntegrationTest(ITMetadata meta, List setup, List tests, List cleanup) implements ITElement { @Override @@ -26,8 +26,8 @@ public void expand(ITCollection itCollection) { setup.clear(); setup.addAll(expandedSetup); - List expandedTests = new ArrayList<>(); - for (TestItem item : tests) { + List expandedTests = new ArrayList<>(); + for (TestCase item : tests) { if (item.include() != null ){ var includedTest = itCollection.testFirstByName(item.include()); expandedTests.addAll(includedTest.tests()); diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/IntegrationTestRunner.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/IntegrationTestRunner.java index 7f82340540..ae91556fe6 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/IntegrationTestRunner.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/IntegrationTestRunner.java @@ -1,20 +1,9 @@ package io.stargate.sgv2.jsonapi.api.v1.vectorize; -import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.node.ObjectNode; -import io.restassured.http.ContentType; import io.restassured.response.ValidatableResponse; -import io.restassured.specification.RequestSpecification; -import io.stargate.sgv2.jsonapi.api.model.command.CommandTarget; -import io.stargate.sgv2.jsonapi.api.v1.CollectionResource; -import io.stargate.sgv2.jsonapi.api.v1.GeneralResource; -import io.stargate.sgv2.jsonapi.api.v1.KeyspaceResource; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions.AssertionFactory; import io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions.ITAssertion; -import io.stargate.sgv2.jsonapi.config.constants.HttpConstants; -import io.stargate.sgv2.jsonapi.service.embedding.operation.test.CustomITEmbeddingProvider; import org.apache.commons.lang3.RandomStringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -23,15 +12,9 @@ import java.lang.reflect.Method; import java.lang.reflect.Modifier; import java.util.Arrays; -import java.util.Base64; import java.util.Map; import static io.restassured.RestAssured.given; -import static io.stargate.sgv2.jsonapi.api.v1.ResponseAssertions.responseIsDDLSuccess; -import static io.stargate.sgv2.jsonapi.api.v1.ResponseAssertions.responseIsWriteSuccess; -import static io.stargate.sgv2.jsonapi.api.v1.util.IntegrationTestUtils.getCassandraPassword; -import static io.stargate.sgv2.jsonapi.api.v1.util.IntegrationTestUtils.getCassandraUsername; -import static org.hamcrest.Matchers.*; public class IntegrationTestRunner extends RunnerBase { private static final Logger LOGGER = LoggerFactory.getLogger(IntegrationTestRunner.class); @@ -67,10 +50,8 @@ public void run() { target.apiRequest(setupRequest, env).executeWithSuccess(); } - for (TestItem testItem : test.tests()) { + for (TestCase testItem : test.tests()) { - var resp = target.apiRequest(testItem.request(), env).execute(); - testAssertions(testItem, resp); } for (TestRequest setupRequest : test.cleanup()) { @@ -78,8 +59,21 @@ public void run() { } } + private TestResult runTest(TestCase testItem, IntegrationEnv env) { - private void testAssertions (TestItem testItem, ValidatableResponse response) { + AssertionError textException; + ValidatableResponse response; + try { + response = target.apiRequest(testItem.request(), env).execute(); + testAssertions(testItem, response); + } + catch (AssertionError ae){ + textException = ae; + } + + response.extract(). + } + private void testAssertions (TestCase testItem, ValidatableResponse response) { for (Map.Entry entry : testItem.asserts().properties()){ var args = entry.getValue(); // null / 3 / {...} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestItem.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestCase.java similarity index 92% rename from src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestItem.java rename to src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestCase.java index 05c9745fe8..87b6f75a7c 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestItem.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestCase.java @@ -3,7 +3,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.databind.node.ObjectNode; -public record TestItem( +public record TestCase( String name, TestRequest request, diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestResult.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestResult.java new file mode 100644 index 0000000000..3e7eabee6e --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestResult.java @@ -0,0 +1,16 @@ +package io.stargate.sgv2.jsonapi.api.v1.vectorize; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; + +public record TestResult( + TestCase testItem, + ObjectNode actualRequest, + JsonNode response, + AssertionError error +) { + + public boolean failed(){ + return error == null; + } +} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/VectorizeUnit.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/VectorizeUnit.java index 41e6856cbd..4a8f88e1e3 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/VectorizeUnit.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/VectorizeUnit.java @@ -23,37 +23,37 @@ public void doTest() { // SEQUENTIAL -// integrationTarget.workflowStarting(workflow); -// try { -// for (var job : jobs) { -// LOGGER.info("Starting job {}", job.meta()); -// new IntegrationJobRunner(integrationTarget, itCollection, job).run(); -// } -// } finally { -// integrationTarget.workflowFinished(workflow); -// } - - // Parallel - int maxConcurrentJobs = 2; integrationTarget.workflowStarting(workflow); try { - var executor = java.util.concurrent.Executors.newFixedThreadPool(maxConcurrentJobs); - try { - var futures = jobs.stream() - .map(job -> java.util.concurrent.CompletableFuture.runAsync(() -> { - LOGGER.info("Starting job {}", job.meta()); - new IntegrationJobRunner(integrationTarget, itCollection, job).run(); - }, executor)) - .toList(); - - // wait for all, fail if any failed - futures.forEach(java.util.concurrent.CompletableFuture::join); - } finally { - executor.shutdown(); + for (var job : jobs) { + LOGGER.info("Starting job {}", job.meta()); + new IntegrationJobRunner(integrationTarget, itCollection, job).run(); } } finally { integrationTarget.workflowFinished(workflow); } + // Parallel +// int maxConcurrentJobs = 2; +// integrationTarget.workflowStarting(workflow); +// try { +// var executor = java.util.concurrent.Executors.newFixedThreadPool(maxConcurrentJobs); +// try { +// var futures = jobs.stream() +// .map(job -> java.util.concurrent.CompletableFuture.runAsync(() -> { +// LOGGER.info("Starting job {}", job.meta()); +// new IntegrationJobRunner(integrationTarget, itCollection, job).run(); +// }, executor)) +// .toList(); +// +// // wait for all, fail if any failed +// futures.forEach(java.util.concurrent.CompletableFuture::join); +// } finally { +// executor.shutdown(); +// } +// } finally { +// integrationTarget.workflowFinished(workflow); +// } + } } diff --git a/src/test/resources/integration-tests/vectorize/vectorize-base.json b/src/test/resources/integration-tests/vectorize/vectorize-base.json index a712710122..0a545927c6 100644 --- a/src/test/resources/integration-tests/vectorize/vectorize-base.json +++ b/src/test/resources/integration-tests/vectorize/vectorize-base.json @@ -56,7 +56,7 @@ }, "asserts": { "response.isFindSuccess": null, - "documents.count": 3 + "documents.count": 4 } }, { diff --git a/src/test/resources/integration-tests/vectorize/vectorize-workflow.json b/src/test/resources/integration-tests/vectorize/vectorize-workflow.json index 5164ed3804..851f9dc40b 100644 --- a/src/test/resources/integration-tests/vectorize/vectorize-workflow.json +++ b/src/test/resources/integration-tests/vectorize/vectorize-workflow.json @@ -59,7 +59,7 @@ { "meta": { "name": "jinaAI-vectorize", - "tags": [] + "tags": ["disabled"] }, "fromEnvironment": { "x-embedding-api-key": "jinaAI_KEY" From 716d86e7693b7517dc2f3e3484c8876403725016 Mon Sep 17 00:00:00 2001 From: Aaron Morton Date: Mon, 23 Feb 2026 07:47:24 +1300 Subject: [PATCH 04/89] WIP --- .../jsonapi/api/v1/vectorize/APIRequest.java | 56 +++----- .../jsonapi/api/v1/vectorize/APIResponse.java | 3 +- .../api/v1/vectorize/CassandraBackend.java | 15 +- .../api/v1/vectorize/IntegrationTarget.java | 4 +- .../api/v1/vectorize/IntegrationTest.java | 12 +- .../v1/vectorize/IntegrationTestRunner.java | 119 ++++++---------- .../api/v1/vectorize/TestAssertion.java | 5 - .../jsonapi/api/v1/vectorize/TestCase.java | 2 +- .../api/v1/vectorize/TestCaseResult.java | 17 +++ .../jsonapi/api/v1/vectorize/TestCommand.java | 134 ++++++++++++++++++ .../jsonapi/api/v1/vectorize/TestRequest.java | 100 ++----------- .../api/v1/vectorize/TestResponse.java | 36 +++++ .../jsonapi/api/v1/vectorize/TestResult.java | 16 --- .../assertions/AssertionFactory.java | 5 +- .../vectorize/assertions/BodyAssertion.java | 30 ++++ .../v1/vectorize/assertions/Documents.java | 14 +- .../v1/vectorize/assertions/ITAssertion.java | 11 -- .../api/v1/vectorize/assertions/Response.java | 20 ++- .../api/v1/vectorize/assertions/Status.java | 11 ++ .../v1/vectorize/assertions/StatusCode.java | 12 ++ .../vectorize/assertions/TestAssertion.java | 81 +++++++++++ .../vectorize/vectorize-base.json | 18 ++- 22 files changed, 454 insertions(+), 267 deletions(-) delete mode 100644 src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestAssertion.java create mode 100644 src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestCaseResult.java create mode 100644 src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestCommand.java create mode 100644 src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestResponse.java delete mode 100644 src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestResult.java create mode 100644 src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/BodyAssertion.java delete mode 100644 src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/ITAssertion.java create mode 100644 src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Status.java create mode 100644 src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/StatusCode.java create mode 100644 src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/TestAssertion.java diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/APIRequest.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/APIRequest.java index b24baa78aa..b0d537c307 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/APIRequest.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/APIRequest.java @@ -7,9 +7,8 @@ import io.restassured.response.Response; import io.restassured.response.ValidatableResponse; import io.restassured.specification.RequestSpecification; +import io.stargate.sgv2.jsonapi.api.model.command.CommandName; import io.stargate.sgv2.jsonapi.api.model.command.CommandTarget; -import io.stargate.sgv2.jsonapi.api.v1.CollectionResource; -import io.stargate.sgv2.jsonapi.api.v1.KeyspaceResource; import io.stargate.sgv2.jsonapi.config.constants.HttpConstants; import java.util.Map; @@ -29,29 +28,24 @@ public class APIRequest { private static String DB_PATH = "/"; private final Connection connection; - private final TestRequest testRequest; - private final IntegrationEnv env; + private final IntegrationEnv integrationEnv; + private final ObjectNode request; + + public APIRequest(Connection connection, IntegrationEnv integrationEnv, ObjectNode request ) { - public APIRequest( Connection connection, TestRequest testRequest, IntegrationEnv env ) { this.connection = connection; - this.testRequest = testRequest; - this.env = env; + this.integrationEnv = integrationEnv; + this.request = request; } - public ValidatableResponse execute(){ + public APIResponse execute(){ - var requestWithEnv = testRequest.withEnvironment(env); - var requestSpec = requestSpec(requestWithEnv); - return executeRequest(requestSpec); + var requestSpec = requestSpec(); + return new APIResponse(this, executeRequest(requestSpec)); } - public ValidatableResponse executeWithSuccess(){ - var resp = execute(); - assertSuccess(resp); - return resp; - } - private RequestSpecification requestSpec(ObjectNode request) { + private RequestSpecification requestSpec() { String requestString; try { @@ -66,20 +60,20 @@ private RequestSpecification requestSpec(ObjectNode request) { private ValidatableResponse executeRequest(RequestSpecification requestSpec) { - var commandName = testRequest.commandName(); + var commandName = TestCommand.commandName(request); Response response; if (commandName.getTargets().contains(CommandTarget.COLLECTION) || commandName.getTargets().contains(CommandTarget.TABLE)){ response = requestSpec - .post(COLLECTION_PATH, env.requiredValue("KEYSPACE_NAME"), env.requiredValue("COLLECTION_NAME")); + .post(COLLECTION_PATH, integrationEnv.requiredValue("KEYSPACE_NAME"), integrationEnv.requiredValue("COLLECTION_NAME")); } else if (commandName.getTargets().contains(CommandTarget.KEYSPACE) ){ - response = requestSpec.post(KEYSPACE_PATH, env.requiredValue("KEYSPACE_NAME")); + response = requestSpec.post(KEYSPACE_PATH, integrationEnv.requiredValue("KEYSPACE_NAME")); } else if(commandName.getTargets().contains(CommandTarget.DATABASE)){ response = requestSpec.post(DB_PATH); } else { - throw new IllegalArgumentException("Do not know how to execute command: " + testRequest.commandName()); + throw new IllegalArgumentException("Do not know how to execute command: " + commandName); } return response @@ -90,9 +84,9 @@ protected Map getHeaders() { return Map.of( HttpConstants.AUTHENTICATION_TOKEN_HEADER_NAME, - env.requiredValue("Token"), + integrationEnv.requiredValue("Token"), HttpConstants.EMBEDDING_AUTHENTICATION_TOKEN_HEADER_NAME, - env.requiredValue("x-embedding-api-key")); + integrationEnv.requiredValue("x-embedding-api-key")); } public RequestSpecification jsonRequest(){ @@ -107,20 +101,4 @@ public RequestSpecification jsonRequest(){ } - private void assertSuccess(ValidatableResponse response) { - response.statusCode(200); - - switch (testRequest.commandName()) { - case INSERT_ONE, INSERT_MANY -> { - response - .body("$", responseIsWriteSuccess()) - .body("status.insertedIds[0]", not(emptyString())); - } - case DELETE_COLLECTION, CREATE_COLLECTION -> { - response - .body("$", responseIsDDLSuccess()) - .body("status.ok", is(1)); - } - } - } } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/APIResponse.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/APIResponse.java index d1b2346ea6..5f14d2b5c1 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/APIResponse.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/APIResponse.java @@ -4,6 +4,5 @@ import io.restassured.response.ValidatableResponse; public record APIResponse (APIRequest apiRequest, - ObjectNode actualRequest, - ValidatableResponse response) { + ValidatableResponse validatableResponse) { } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/CassandraBackend.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/CassandraBackend.java index 184d67f366..a3bf06cd5a 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/CassandraBackend.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/CassandraBackend.java @@ -1,6 +1,7 @@ package io.stargate.sgv2.jsonapi.api.v1.vectorize; import com.fasterxml.jackson.databind.ObjectMapper; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions.TestAssertion; import org.apache.commons.lang3.RandomStringUtils; import static io.stargate.sgv2.jsonapi.api.v1.vectorize.IntegrationEnv.toSafeSchemaIdentifier; @@ -17,7 +18,7 @@ public void jobStarting(IntegrationTarget integrationTarget, IntegrationJob job) var keyspaceName = toSafeSchemaIdentifier( "job_" + job.meta().name().substring(0, Math.min(job.meta().name().length(), 27)) + "_" + RandomStringUtils.insecure().nextAlphanumeric(16)); - var request = TestRequest.fromJson( + var command = TestCommand.fromJson( """ { "createKeyspace": { @@ -27,13 +28,16 @@ public void jobStarting(IntegrationTarget integrationTarget, IntegrationJob job) """); job.variables().put("KEYSPACE_NAME", keyspaceName); - integrationTarget.apiRequest(request, job.withoutMatrix()).executeWithSuccess(); + var setupRequest = new TestRequest(command, integrationTarget, job.withoutMatrix(), TestAssertion.forSuccess(command.commandName())); + + var setupResponse = setupRequest.execute(); + setupResponse.validate(null, null, true); } @Override public void jobFinished(IntegrationTarget integrationTarget, IntegrationJob job) { - var request = TestRequest.fromJson( + var command = TestCommand.fromJson( """ { "dropKeyspace": { @@ -42,6 +46,9 @@ public void jobFinished(IntegrationTarget integrationTarget, IntegrationJob job) } """); - integrationTarget.apiRequest(request, job.withoutMatrix()).executeWithSuccess(); + var setupRequest = new TestRequest(command, integrationTarget, job.withoutMatrix(), TestAssertion.forSuccess(command.commandName())); + + var setupResponse = setupRequest.execute(); + setupResponse.validate(null, null, true); } } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/IntegrationTarget.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/IntegrationTarget.java index 32cbe8ded2..f2f55c6663 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/IntegrationTarget.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/IntegrationTarget.java @@ -23,8 +23,8 @@ public Connection connection(){ return target.connection(); } - public APIRequest apiRequest(TestRequest testRequest, IntegrationEnv env){ - return new APIRequest(target.connection(), testRequest, env); + public APIRequest apiRequest(TestCommand testCommand, IntegrationEnv env){ + return new APIRequest(target.connection(), env, testCommand.withEnvironment(env)); } public void workflowStarting(IntegrationWorkflow workflow){ } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/IntegrationTest.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/IntegrationTest.java index 844c220297..0c869816b5 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/IntegrationTest.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/IntegrationTest.java @@ -3,7 +3,7 @@ import java.util.ArrayList; import java.util.List; -public record IntegrationTest(ITMetadata meta, List setup, List tests, List cleanup) +public record IntegrationTest(ITMetadata meta, List setup, List tests, List cleanup) implements ITElement { @Override @@ -13,14 +13,14 @@ public ITElementKind kind() { public void expand(ITCollection itCollection) { - List expandedSetup = new ArrayList<>(); - for (TestRequest request : setup) { - if (request.request().has("$include")){ - var includedTest = itCollection.testFirstByName(request.request().get("$include").textValue()); + List expandedSetup = new ArrayList<>(); + for (TestCommand command : setup) { + if (command.includeFrom() != null){ + var includedTest = itCollection.testFirstByName(command.includeFrom()); expandedSetup.addAll(includedTest.setup()); } else { - expandedSetup.add(request); + expandedSetup.add(command); } } setup.clear(); diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/IntegrationTestRunner.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/IntegrationTestRunner.java index ae91556fe6..9819fcc691 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/IntegrationTestRunner.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/IntegrationTestRunner.java @@ -1,19 +1,12 @@ package io.stargate.sgv2.jsonapi.api.v1.vectorize; -import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; -import io.restassured.response.ValidatableResponse; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions.ITAssertion; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions.BodyAssertion; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions.TestAssertion; import org.apache.commons.lang3.RandomStringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; -import java.lang.reflect.Modifier; -import java.util.Arrays; -import java.util.Map; - import static io.restassured.RestAssured.given; public class IntegrationTestRunner extends RunnerBase { @@ -27,95 +20,73 @@ public class IntegrationTestRunner extends RunnerBase { private final ITCollection itCollection; private final IntegrationTarget target; - private final IntegrationTest test; - private final IntegrationEnv env; + private final IntegrationTest integrationTest; + private final IntegrationEnv integrationEnv; public IntegrationTestRunner( - ITCollection itCollection, IntegrationTarget target, IntegrationTest test, IntegrationEnv env) { + ITCollection itCollection, IntegrationTarget target, IntegrationTest integrationTest, IntegrationEnv integrationEnv) { this.itCollection = itCollection; this.target = target; - this.test = test; - this.env = env; + this.integrationTest = integrationTest; + this.integrationEnv = integrationEnv; } @Override protected IntegrationEnv integrationEnv() { - return env; + return integrationEnv; } public void run() { - LOGGER.info("Starting Integration Test with env={}", env); - for (TestRequest setupRequest : test.setup()) { - target.apiRequest(setupRequest, env).executeWithSuccess(); - } - - for (TestCase testItem : test.tests()) { + LOGGER.info("Starting Integration Test with env={}", integrationEnv); + for (TestCommand setupCommand : integrationTest.setup()) { + var setupRequest = new TestRequest(setupCommand, target, integrationEnv, TestAssertion.forSuccess(setupCommand.commandName())); - } + var setupResponse = setupRequest.execute(); + var testCaseResult = setupResponse.validate(integrationTest, null); - for (TestRequest setupRequest : test.cleanup()) { - target.apiRequest(setupRequest, env).executeWithSuccess(); - } - } - - private TestResult runTest(TestCase testItem, IntegrationEnv env) { + if (testCaseResult.failed()){ + LOGGER.warn("TestSetup FAILED: test.name={}, testCase.name={}, failedAssertion={}, error={}", + testCaseResult.integrationTest().meta().name(), "NULL", testCaseResult.failedAssertion(), String.valueOf(testCaseResult.error())); + } + else{ + LOGGER.info("TestSetup PASSED: test.name={}, testCase.name={}", + testCaseResult.integrationTest().meta().name(), "NULL"); + } - AssertionError textException; - ValidatableResponse response; - try { - response = target.apiRequest(testItem.request(), env).execute(); - testAssertions(testItem, response); - } - catch (AssertionError ae){ - textException = ae; } - response.extract(). - } - private void testAssertions (TestCase testItem, ValidatableResponse response) { + for (TestCase testCase : integrationTest.tests()) { + var testRequest = new TestRequest(testCase.command(), target, integrationEnv, TestAssertion.buildAssertions(testCase)); + var testResponse = testRequest.execute(); + var testCaseResult = testResponse.validate(integrationTest, testCase); - for (Map.Entry entry : testItem.asserts().properties()){ - var args = entry.getValue(); // null / 3 / {...} - - var assertFactory = findAssertionFactory(entry.getKey()); - ITAssertion itAssertion; - try { - itAssertion = (ITAssertion)assertFactory.invoke(null, args); - } catch (IllegalAccessException | InvocationTargetException e) { - throw new RuntimeException(e); + if (testCaseResult.failed()){ + LOGGER.warn("TestCase FAILED: test.name={}, testCase.name={}, failedAssertion={}, error={}", + testCaseResult.integrationTest().meta().name(),testCaseResult.testCase().name(), testCaseResult.failedAssertion(), String.valueOf(testCaseResult.error())); + } + else{ + LOGGER.info("TestCase PASSED: test.name={}, testCase.name={}", + testCaseResult.integrationTest().meta().name(),testCaseResult.testCase().name()); } - - LOGGER.info("XXX Running test assertion key={}, value={}", entry.getKey(), entry.getValue().toString()); - response.body(itAssertion.bodyPath(), itAssertion.matcher()); } - } - - private Method findAssertionFactory(String key){ - // "response.isFindSuccess" - int dot = key.indexOf('.'); - String typeName = key.substring(0, dot); - String funcName = key.substring(dot + 1); + for (TestCommand cleanupCommand : integrationTest.cleanup()) { - String qualifiedTypeName = - "io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions." - + Character.toUpperCase(typeName.charAt(0)) - + typeName.substring(1).toLowerCase(); + var cleanupRequest = new TestRequest(cleanupCommand, target, integrationEnv, TestAssertion.forSuccess(cleanupCommand.commandName())); - try { - Class cls = Class.forName(qualifiedTypeName); + var cleanupResponse = cleanupRequest.execute(); + var testCaseResult = cleanupResponse.validate(integrationTest, null); - var factoryMethod = Arrays.stream(cls.getMethods()) - .filter(m -> m.getName().equalsIgnoreCase(funcName)) - .filter(m -> Modifier.isStatic(m.getModifiers())) - .findFirst() - .orElseThrow(); - - - return factoryMethod; - } catch (ReflectiveOperationException e) { - throw new RuntimeException("Invalid assertion: " + key, e); + if (testCaseResult.failed()){ + LOGGER.warn("TestCleanup FAILED: test.name={}, testCase.name={}, failedAssertion={}, error={}", + testCaseResult.integrationTest().meta().name(), "NULL", testCaseResult.failedAssertion().toString(), testCaseResult.error().toString()); + } + else{ + LOGGER.info("TestCleanup PASSED: test.name={}, testCase.name={}", + testCaseResult.integrationTest().meta().name(), "NULL"); + } } } + } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestAssertion.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestAssertion.java deleted file mode 100644 index ea5ad4793f..0000000000 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestAssertion.java +++ /dev/null @@ -1,5 +0,0 @@ -package io.stargate.sgv2.jsonapi.api.v1.vectorize; - -import com.fasterxml.jackson.databind.JsonNode; - -public record TestAssertion(String name, JsonNode assertion) {} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestCase.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestCase.java index 87b6f75a7c..f94dd93752 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestCase.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestCase.java @@ -6,7 +6,7 @@ public record TestCase( String name, - TestRequest request, + TestCommand command, ObjectNode asserts, @JsonProperty("$include") String include) {} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestCaseResult.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestCaseResult.java new file mode 100644 index 0000000000..1ff331cf06 --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestCaseResult.java @@ -0,0 +1,17 @@ +package io.stargate.sgv2.jsonapi.api.v1.vectorize; + +import io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions.BodyAssertion; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions.TestAssertion; + +public record TestCaseResult( + IntegrationTest integrationTest, + TestCase testCase, // Nullable + TestResponse testResponse, + AssertionError error, + TestAssertion failedAssertion +) { + + public boolean failed(){ + return error != null; + } +} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestCommand.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestCommand.java new file mode 100644 index 0000000000..ffccc858d8 --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestCommand.java @@ -0,0 +1,134 @@ +package io.stargate.sgv2.jsonapi.api.v1.vectorize; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.fasterxml.jackson.databind.node.TextNode; +import io.stargate.sgv2.jsonapi.api.model.command.CommandName; + +import java.util.UUID; +import org.apache.commons.text.StringSubstitutor; + +public class TestCommand { + + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + + private final ObjectNode request; + private final CommandName commandName; + private final String includeFrom; + + @JsonCreator(mode = JsonCreator.Mode.DELEGATING) + public TestCommand(ObjectNode request) { + + // if non null, that this is a point to find commands in the named test. + var includeField =request.get("$include"); + this.includeFrom = includeField == null ? null : includeField.asText(); + + if (includeField != null) { + this.request = null; + this.commandName = null; + } + else { + this.request = request; + this.commandName = commandName(request); + } + } + + private void checkIsInclude(){ + if (includeFrom != null) { + throw new IllegalStateException("TestCommand is defined to $include from: " + includeFrom); + } + } + public ObjectNode request() { + checkIsInclude(); + return request; + } + public CommandName commandName() { + checkIsInclude(); + return commandName; + } + + public String includeFrom() { + return includeFrom; + } + + public static TestCommand fromJson(String json) { + + try { + return new TestCommand((ObjectNode) OBJECT_MAPPER.readTree(json)); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + } + + public static CommandName commandName(ObjectNode request) { + var requestCommandName = commandNameString(request); + for (CommandName name : CommandName.values()) { + if (name.getApiName().equals(requestCommandName)) { + return name; + } + } + throw new IllegalArgumentException("Unknown command name: " + requestCommandName); + } + + private static String commandNameString(ObjectNode request) { + var it = request.fieldNames(); + if (!it.hasNext()) { + throw new IllegalStateException("Expected exactly one field, found none"); + } + String name = it.next(); + if (it.hasNext()) { + throw new IllegalStateException("Expected exactly one field, found multiple"); + } + return name; + } + + public ObjectNode withEnvironment(IntegrationEnv env) { + ObjectNode updated = request.deepCopy(); + walk(updated, env.substitutor()); + return updated; + } + + + private static void walk(ObjectNode obj, StringSubstitutor subs) { + obj.properties() + .forEach( + (entry) -> { + switch (entry.getValue()) { + case TextNode text -> { + obj.put(entry.getKey(), subs.replace(text.textValue())); + } + case ObjectNode nested -> { + walk(nested, subs); + } + case ArrayNode arr -> { + walk(arr, subs); + } + default -> {} + } + }); + } + + private static void walk(ArrayNode arr, StringSubstitutor subs) { + for (int i = 0; i < arr.size(); i++) { + var child = arr.get(i); + + switch (child) { + case TextNode text -> { + String value = subs.replace(text.textValue()); + arr.set(i, TextNode.valueOf(value)); + } + case ObjectNode nested -> { + walk(nested, subs); + } + case ArrayNode nestedArr -> { + walk(nestedArr, subs); + } + default -> {} + } + } + } +} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestRequest.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestRequest.java index f3d939dc20..5d9145c9a1 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestRequest.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestRequest.java @@ -1,100 +1,18 @@ package io.stargate.sgv2.jsonapi.api.v1.vectorize; -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonIgnore; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.node.ArrayNode; -import com.fasterxml.jackson.databind.node.ObjectNode; -import com.fasterxml.jackson.databind.node.TextNode; -import io.stargate.sgv2.jsonapi.api.model.command.CommandName; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions.BodyAssertion; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions.TestAssertion; import java.util.List; -import java.util.UUID; -import org.apache.commons.text.StringSubstitutor; -public record TestRequest(@JsonIgnore UUID requestId, ObjectNode request) { +public record TestRequest(TestCommand testCommand, + IntegrationTarget integrationTarget, + IntegrationEnv integrationEnv, + List testAssertions) { - private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + public TestResponse execute(){ - @JsonCreator(mode = JsonCreator.Mode.DELEGATING) - public TestRequest(ObjectNode request) { - this(UUID.randomUUID(), request); - } - - public CommandName commandName() { - var requestCommandName = commandNameString(); - for (CommandName name : CommandName.values()) { - if (name.getApiName().equals(requestCommandName)) { - return name; - } - } - throw new IllegalArgumentException("Unknown command name: " + requestCommandName); - } - - private String commandNameString() { - var it = request.fieldNames(); - if (!it.hasNext()) { - throw new IllegalStateException("Expected exactly one field, found none"); - } - String name = it.next(); - if (it.hasNext()) { - throw new IllegalStateException("Expected exactly one field, found multiple"); - } - return name; - } - - public ObjectNode withEnvironment(IntegrationEnv env) { - ObjectNode updated = request.deepCopy(); - walk(updated, env.substitutor()); - return updated; - } - - public static TestRequest fromJson(String json) { - - try { - return new TestRequest((ObjectNode) OBJECT_MAPPER.readTree(json)); - } catch (JsonProcessingException e) { - throw new RuntimeException(e); - } - - } - private static void walk(ObjectNode obj, StringSubstitutor subs) { - obj.properties() - .forEach( - (entry) -> { - switch (entry.getValue()) { - case TextNode text -> { - obj.put(entry.getKey(), subs.replace(text.textValue())); - } - case ObjectNode nested -> { - walk(nested, subs); - } - case ArrayNode arr -> { - walk(arr, subs); - } - default -> {} - } - }); - } - - private static void walk(ArrayNode arr, StringSubstitutor subs) { - for (int i = 0; i < arr.size(); i++) { - var child = arr.get(i); - - switch (child) { - case TextNode text -> { - String value = subs.replace(text.textValue()); - arr.set(i, TextNode.valueOf(value)); - } - case ObjectNode nested -> { - walk(nested, subs); - } - case ArrayNode nestedArr -> { - walk(nestedArr, subs); - } - default -> {} - } - } + var apiRequest = integrationTarget.apiRequest(testCommand, integrationEnv); + return new TestResponse(this, apiRequest, apiRequest.execute()); } } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestResponse.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestResponse.java new file mode 100644 index 0000000000..b3607c192e --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestResponse.java @@ -0,0 +1,36 @@ +package io.stargate.sgv2.jsonapi.api.v1.vectorize; + +import io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions.BodyAssertion; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions.TestAssertion; + +public record TestResponse( + TestRequest testRequest, + APIRequest apiRequest, + APIResponse apiResponse +) { + + public TestCaseResult validate(IntegrationTest integrationTest, TestCase testCase){ + return validate(integrationTest, testCase, false); + } + + public TestCaseResult validate(IntegrationTest integrationTest, TestCase testCase, boolean throwOnError) { + + AssertionError assertionError = null; + TestAssertion failedAssertion = null; + for (var testAssertion : testRequest.testAssertions()){ + try{ + testAssertion.run(apiResponse); + } + catch(AssertionError e){ + if (throwOnError){ + throw e; + } + + failedAssertion = testAssertion; + assertionError = e; + break; + } + } + return new TestCaseResult(integrationTest, testCase, this , assertionError, failedAssertion); + } +} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestResult.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestResult.java deleted file mode 100644 index 3e7eabee6e..0000000000 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestResult.java +++ /dev/null @@ -1,16 +0,0 @@ -package io.stargate.sgv2.jsonapi.api.v1.vectorize; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.ObjectNode; - -public record TestResult( - TestCase testItem, - ObjectNode actualRequest, - JsonNode response, - AssertionError error -) { - - public boolean failed(){ - return error == null; - } -} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/AssertionFactory.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/AssertionFactory.java index 78538e2ee4..3c9f6198da 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/AssertionFactory.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/AssertionFactory.java @@ -1,12 +1,9 @@ package io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions; import com.fasterxml.jackson.databind.JsonNode; -import org.hamcrest.TypeSafeMatcher; - -import java.util.Map; @FunctionalInterface public interface AssertionFactory { - ITAssertion create(JsonNode args); + TestAssertion create(JsonNode args); } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/BodyAssertion.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/BodyAssertion.java new file mode 100644 index 0000000000..ca995742db --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/BodyAssertion.java @@ -0,0 +1,30 @@ +package io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions; + +import com.fasterxml.jackson.databind.JsonNode; +import io.stargate.sgv2.jsonapi.api.model.command.CommandName; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.APIResponse; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.TestCase; +import org.hamcrest.Matcher; + +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.lang.reflect.Modifier; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +import static io.stargate.sgv2.jsonapi.api.v1.ResponseAssertions.responseIsDDLSuccess; +import static io.stargate.sgv2.jsonapi.api.v1.ResponseAssertions.responseIsWriteSuccess; +import static org.hamcrest.Matchers.*; + +public record BodyAssertion( + String bodyPath, + Matcher matcher +) implements TestAssertion{ + + public void run(APIResponse apiResponse) { + + apiResponse.validatableResponse().body(bodyPath(), matcher()); + } +} \ No newline at end of file diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Documents.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Documents.java index 80d156e258..57b346eddc 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Documents.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Documents.java @@ -1,18 +1,18 @@ package io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions; import com.fasterxml.jackson.databind.JsonNode; -import org.hamcrest.Description; -import org.hamcrest.Matcher; -import org.hamcrest.TypeSafeMatcher; - -import java.util.Map; +import static net.javacrumbs.jsonunit.JsonMatchers.jsonEquals; import static org.hamcrest.Matchers.hasSize; public class Documents { - public static ITAssertion count(JsonNode args) { + public static TestAssertion count(JsonNode args) { var expectedCount = args.asInt(); - return new ITAssertion("data.documents", hasSize(expectedCount)); + return new BodyAssertion("data.documents", hasSize(expectedCount)); + } + + public static TestAssertion isExactly(JsonNode args) { + return new BodyAssertion("data.document", jsonEquals(args)); } } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/ITAssertion.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/ITAssertion.java deleted file mode 100644 index 41615dd781..0000000000 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/ITAssertion.java +++ /dev/null @@ -1,11 +0,0 @@ -package io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions; - -import org.hamcrest.Matcher; -import org.hamcrest.TypeSafeMatcher; - -import java.util.Map; - -public record ITAssertion( - String bodyPath, - Matcher matcher -) {} \ No newline at end of file diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Response.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Response.java index 385495f628..1aabd520ca 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Response.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Response.java @@ -2,12 +2,26 @@ import com.fasterxml.jackson.databind.JsonNode; -import static io.stargate.sgv2.jsonapi.api.v1.ResponseAssertions.responseIsFindSuccess; +import static io.stargate.sgv2.jsonapi.api.v1.ResponseAssertions.*; import static org.hamcrest.Matchers.hasSize; public class Response { - public static ITAssertion isFindSuccess(JsonNode args) { - return new ITAssertion("$", responseIsFindSuccess()); + public static TestAssertion isFindSuccess(JsonNode args) { + return new BodyAssertion("$", responseIsFindSuccess()); } + + public static TestAssertion isFindAndSuccess(JsonNode args) { + return new BodyAssertion("$", responseIsFindAndSuccess()); + } + + + public static TestAssertion isWriteSuccess(JsonNode args) { + return new BodyAssertion("$", responseIsWriteSuccess()); + } + + public static TestAssertion isDDLSuccess(JsonNode args) { + return new BodyAssertion("$", responseIsDDLSuccess()); + } + } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Status.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Status.java new file mode 100644 index 0000000000..d724ddfdd5 --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Status.java @@ -0,0 +1,11 @@ +package io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions; + +import com.fasterxml.jackson.databind.JsonNode; + +import static net.javacrumbs.jsonunit.JsonMatchers.jsonEquals; + +public class Status { + public static TestAssertion isExactly(JsonNode args) { + return new BodyAssertion("status", jsonEquals(args)); + } +} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/StatusCode.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/StatusCode.java new file mode 100644 index 0000000000..56d0bd8b07 --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/StatusCode.java @@ -0,0 +1,12 @@ +package io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions; + +import io.stargate.sgv2.jsonapi.api.v1.vectorize.APIResponse; +import org.apache.http.HttpStatus; +import org.testcontainers.shaded.com.fasterxml.jackson.databind.JsonNode; + +public class StatusCode { + + public static TestAssertion success(JsonNode args){ + return apiResponse -> apiResponse.validatableResponse().statusCode(HttpStatus.SC_OK); + } +} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/TestAssertion.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/TestAssertion.java new file mode 100644 index 0000000000..ca0073c79c --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/TestAssertion.java @@ -0,0 +1,81 @@ +package io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions; + +import com.fasterxml.jackson.databind.JsonNode; +import io.stargate.sgv2.jsonapi.api.model.command.CommandName; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.APIResponse; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.TestCase; + +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.lang.reflect.Modifier; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +public interface TestAssertion { + + void run(APIResponse apiResponse); + + static List forSuccess(CommandName commandName) { + + List assertions = new ArrayList<>(); + assertions.add(StatusCode.success(null)); + + switch (commandName) { + case INSERT_ONE, INSERT_MANY -> { + assertions.add(Response.isWriteSuccess(null)); + } + case DELETE_COLLECTION, CREATE_COLLECTION -> { + assertions.add(Response.isDDLSuccess(null)); + } + } + return assertions; + } + + + public static List buildAssertions (TestCase testCase) { + + List testAssertions = new ArrayList<>(); + for (Map.Entry entry : testCase.asserts().properties()){ + var args = entry.getValue(); + + var assertFactory = findAssertionFactory(entry.getKey()); + try { + testAssertions.add((BodyAssertion)assertFactory.invoke(null, args)); + } catch (IllegalAccessException | InvocationTargetException e) { + throw new RuntimeException(e); + } + } + return testAssertions; + } + + private static Method findAssertionFactory(String key){ + // "validatableResponse.isFindSuccess" + + int dot = key.indexOf('.'); + String typeName = key.substring(0, dot); + String funcName = key.substring(dot + 1); + + String qualifiedTypeName = + "io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions." + + Character.toUpperCase(typeName.charAt(0)) + + typeName.substring(1).toLowerCase(); + + try { + Class cls = Class.forName(qualifiedTypeName); + + var factoryMethod = Arrays.stream(cls.getMethods()) + .filter(m -> m.getName().equalsIgnoreCase(funcName)) + .filter(m -> Modifier.isStatic(m.getModifiers())) + .findFirst() + .orElseThrow(); + + + return factoryMethod; + } catch (ReflectiveOperationException e) { + throw new RuntimeException("Invalid assertion: " + key, e); + } + } + +} diff --git a/src/test/resources/integration-tests/vectorize/vectorize-base.json b/src/test/resources/integration-tests/vectorize/vectorize-base.json index 0a545927c6..a7d5969dd8 100644 --- a/src/test/resources/integration-tests/vectorize/vectorize-base.json +++ b/src/test/resources/integration-tests/vectorize/vectorize-base.json @@ -47,7 +47,7 @@ "tests": [ { "name": "findByVectorize", - "request": { + "command": { "find": { "sort": { "$vectorize": "I love movies!" @@ -61,7 +61,7 @@ }, { "name": "findOneAndUpdate", - "request": { + "command": { "findOneAndUpdate": { "sort": { "$vectorize": "Inception is a science fiction action film" @@ -77,6 +77,20 @@ } }, "asserts": { + "response.isFindAndSuccess": null, + "documents.isExactly": { + "_id": "Inception", + "name": "Inception", + "genre": "Science Fiction", + "artist": [ + "Leonardo DiCaprio" + ], + "status": "active" + }, + "status.isExactly" : { + "matchedCount": 1, + "modifiedCount": 1 + } } } ] From 1a14b93f57093fda3952f008817dbdde84fdabf7 Mon Sep 17 00:00:00 2001 From: Aaron Morton Date: Mon, 23 Feb 2026 15:01:58 +1300 Subject: [PATCH 05/89] WIP dynamic --- .../jsonapi/api/v1/vectorize/APIRequest.java | 8 +- .../jsonapi/api/v1/vectorize/Backend.java | 15 ++- .../api/v1/vectorize/CassandraBackend.java | 19 ++-- .../sgv2/jsonapi/api/v1/vectorize/ITFile.java | 22 ---- .../jsonapi/api/v1/vectorize/ITMetadata.java | 6 - .../api/v1/vectorize/IntegrationJob.java | 70 ------------ .../v1/vectorize/IntegrationJobRunner.java | 74 ++++++------- .../api/v1/vectorize/IntegrationTarget.java | 42 ------- .../v1/vectorize/IntegrationTestRunner.java | 45 ++++---- .../api/v1/vectorize/IntegrationWorkflow.java | 11 -- .../sgv2/jsonapi/api/v1/vectorize/Job.java | 103 ++++++++++++++++++ .../jsonapi/api/v1/vectorize/RunnerBase.java | 8 +- .../jsonapi/api/v1/vectorize/SpecFile.java | 22 ++++ .../{ITCollection.java => SpecFiles.java} | 58 +++++----- .../sgv2/jsonapi/api/v1/vectorize/Target.java | 48 +++++++- .../api/v1/vectorize/TargetConfiguration.java | 4 + ...argets.java => TargetConfigurationss.java} | 12 +- .../api/v1/vectorize/TestCaseResult.java | 3 +- .../jsonapi/api/v1/vectorize/TestCommand.java | 4 +- ...tegrationEnv.java => TestEnvironment.java} | 37 +++++-- .../jsonapi/api/v1/vectorize/TestPlan.java | 87 +++++++++++++++ .../jsonapi/api/v1/vectorize/TestRequest.java | 5 +- .../api/v1/vectorize/TestResponse.java | 7 +- .../{ITElement.java => TestSpec.java} | 8 +- .../api/v1/vectorize/TestSpecMeta.java | 6 + .../{IntegrationTest.java => TestSuite.java} | 27 ++++- .../v1/vectorize/VectorizeTestFactory.java | 50 +++++++++ .../api/v1/vectorize/VectorizeUnit.java | 43 ++++---- .../jsonapi/api/v1/vectorize/Workflow.java | 36 ++++++ .../vectorize/assertions/TestAssertion.java | 2 +- .../vectorize/vectorize-base.json | 2 +- 31 files changed, 554 insertions(+), 330 deletions(-) delete mode 100644 src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/ITFile.java delete mode 100644 src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/ITMetadata.java delete mode 100644 src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/IntegrationJob.java delete mode 100644 src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/IntegrationTarget.java delete mode 100644 src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/IntegrationWorkflow.java create mode 100644 src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/Job.java create mode 100644 src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/SpecFile.java rename src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/{ITCollection.java => SpecFiles.java} (70%) create mode 100644 src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TargetConfiguration.java rename src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/{Targets.java => TargetConfigurationss.java} (82%) rename src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/{IntegrationEnv.java => TestEnvironment.java} (62%) create mode 100644 src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestPlan.java rename src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/{ITElement.java => TestSpec.java} (55%) create mode 100644 src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestSpecMeta.java rename src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/{IntegrationTest.java => TestSuite.java} (55%) create mode 100644 src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/VectorizeTestFactory.java create mode 100644 src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/Workflow.java diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/APIRequest.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/APIRequest.java index b0d537c307..e57f2768d1 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/APIRequest.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/APIRequest.java @@ -7,16 +7,12 @@ import io.restassured.response.Response; import io.restassured.response.ValidatableResponse; import io.restassured.specification.RequestSpecification; -import io.stargate.sgv2.jsonapi.api.model.command.CommandName; import io.stargate.sgv2.jsonapi.api.model.command.CommandTarget; import io.stargate.sgv2.jsonapi.config.constants.HttpConstants; import java.util.Map; import static io.restassured.RestAssured.given; -import static io.stargate.sgv2.jsonapi.api.v1.ResponseAssertions.responseIsDDLSuccess; -import static io.stargate.sgv2.jsonapi.api.v1.ResponseAssertions.responseIsWriteSuccess; -import static org.hamcrest.Matchers.*; public class APIRequest { @@ -28,10 +24,10 @@ public class APIRequest { private static String DB_PATH = "/"; private final Connection connection; - private final IntegrationEnv integrationEnv; + private final TestEnvironment integrationEnv; private final ObjectNode request; - public APIRequest(Connection connection, IntegrationEnv integrationEnv, ObjectNode request ) { + public APIRequest(Connection connection, TestEnvironment integrationEnv, ObjectNode request ) { this.connection = connection; this.integrationEnv = integrationEnv; diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/Backend.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/Backend.java index 7aa27d4fdf..1d5c94f8f0 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/Backend.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/Backend.java @@ -2,16 +2,19 @@ public abstract class Backend { - public void workflowStarting(IntegrationTarget integrationTarget, IntegrationWorkflow workflow) { } + public void updateJobForTarget(Job job){ + } - public void workflowFinished(IntegrationTarget integrationTarget, IntegrationWorkflow workflow) { } + public void workflowStarting(TestPlan testPlan, Workflow workflow) { } - public void jobStarting(IntegrationTarget integrationTarget, IntegrationJob job) { } + public void workflowFinished(TestPlan testPlan, Workflow workflow) { } - public void jobFinished(IntegrationTarget integrationTarget, IntegrationJob job) { } + public void jobStarting(TestPlan testPlan, Job job) { } - public void testStarting(IntegrationTarget integrationTarget, IntegrationTest test, IntegrationEnv env) { } + public void jobFinished(TestPlan testPlan, Job job) { } - public void testFinished(IntegrationTarget integrationTarget, IntegrationTest test, IntegrationEnv env) { } + public void testStarting(TestPlan testPlan, TestSuite test, TestEnvironment env) { } + + public void testFinished(TestPlan testPlan, TestSuite test, TestEnvironment env) { } } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/CassandraBackend.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/CassandraBackend.java index a3bf06cd5a..8d6dca61ef 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/CassandraBackend.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/CassandraBackend.java @@ -4,20 +4,22 @@ import io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions.TestAssertion; import org.apache.commons.lang3.RandomStringUtils; -import static io.stargate.sgv2.jsonapi.api.v1.vectorize.IntegrationEnv.toSafeSchemaIdentifier; +import static io.stargate.sgv2.jsonapi.api.v1.vectorize.TestEnvironment.toSafeSchemaIdentifier; public class CassandraBackend extends Backend { private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); - @Override - public void jobStarting(IntegrationTarget integrationTarget, IntegrationJob job) { - // max length for keyspace is 48 chars - + public void updateJobForTarget(Job job) { var keyspaceName = toSafeSchemaIdentifier( "job_" + job.meta().name().substring(0, Math.min(job.meta().name().length(), 27)) + "_" + RandomStringUtils.insecure().nextAlphanumeric(16)); + job.variables().put("KEYSPACE_NAME", keyspaceName); + } + @Override + public void jobStarting(TestPlan testPlan, Job job) { + // max length for keyspace is 48 chars var command = TestCommand.fromJson( """ { @@ -27,8 +29,7 @@ public void jobStarting(IntegrationTarget integrationTarget, IntegrationJob job) } """); - job.variables().put("KEYSPACE_NAME", keyspaceName); - var setupRequest = new TestRequest(command, integrationTarget, job.withoutMatrix(), TestAssertion.forSuccess(command.commandName())); + var setupRequest = new TestRequest(command, testPlan.target(), job.withoutMatrix(testPlan), TestAssertion.forSuccess(command.commandName())); var setupResponse = setupRequest.execute(); setupResponse.validate(null, null, true); @@ -36,7 +37,7 @@ public void jobStarting(IntegrationTarget integrationTarget, IntegrationJob job) } @Override - public void jobFinished(IntegrationTarget integrationTarget, IntegrationJob job) { + public void jobFinished(TestPlan testPlan, Job job) { var command = TestCommand.fromJson( """ { @@ -46,7 +47,7 @@ public void jobFinished(IntegrationTarget integrationTarget, IntegrationJob job) } """); - var setupRequest = new TestRequest(command, integrationTarget, job.withoutMatrix(), TestAssertion.forSuccess(command.commandName())); + var setupRequest = new TestRequest(command, testPlan.target(), job.withoutMatrix(testPlan), TestAssertion.forSuccess(command.commandName())); var setupResponse = setupRequest.execute(); setupResponse.validate(null, null, true); diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/ITFile.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/ITFile.java deleted file mode 100644 index 85f9087916..0000000000 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/ITFile.java +++ /dev/null @@ -1,22 +0,0 @@ -package io.stargate.sgv2.jsonapi.api.v1.vectorize; - -import com.fasterxml.jackson.databind.JsonNode; - -public class ITFile { - - private final ITElement element; - private final JsonNode root; - - ITFile(ITElement element, JsonNode root) { - this.element = element; - this.root = root; - } - - public ITElement element() { - return element; - } - - public JsonNode root() { - return root; - } -} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/ITMetadata.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/ITMetadata.java deleted file mode 100644 index b37690f5bb..0000000000 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/ITMetadata.java +++ /dev/null @@ -1,6 +0,0 @@ -package io.stargate.sgv2.jsonapi.api.v1.vectorize; - -import java.util.List; - -public record ITMetadata(String name, String kind, - List tags) {} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/IntegrationJob.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/IntegrationJob.java deleted file mode 100644 index c161cd3f88..0000000000 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/IntegrationJob.java +++ /dev/null @@ -1,70 +0,0 @@ -package io.stargate.sgv2.jsonapi.api.v1.vectorize; - -import io.stargate.sgv2.jsonapi.api.model.command.CommandErrorFactory; -import org.apache.commons.text.StringSubstitutor; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -public record IntegrationJob( - ITMetadata meta, - Map fromEnvironment, - Map variables, - Map> matrix, - List tests) { - private static final Logger LOGGER = LoggerFactory.getLogger(IntegrationJob.class); - - - public IntegrationEnv withoutMatrix(){ - var fromEnv = new IntegrationEnv(); - for (Map.Entry entry : fromEnvironment.entrySet()) { - var value = System.getenv(entry.getValue()); - if (value== null) { - throw new RuntimeException("Environment variable " + entry.getValue() + " is undefined"); - } - fromEnv.put(entry.getKey(), value); - } - - var fromVariables = new IntegrationEnv(variables); - - return fromEnv.clone().put(fromVariables); - } - public List allEnvironments() { - - var fromEnv = new IntegrationEnv(); - for (Map.Entry entry : fromEnvironment.entrySet()) { - var value = System.getenv(entry.getValue()); - if (value== null) { - throw new RuntimeException("Environment variable " + entry.getValue() + " is undefined"); - } - fromEnv.put(entry.getKey(), value); - } - - var fromVariables = new IntegrationEnv(variables); - - // TODO: handle more matrix - List fromMatrix = new ArrayList<>(); - matrix.get("MODEL").forEach( - model -> { - var env = new IntegrationEnv(); - env.put("MODEL", model); - fromMatrix.add(env); - }); - - List allEnvs = new ArrayList<>(); - for (var matrixEnv : fromMatrix) { - - var completeEnv = fromEnv - .clone() - .put(fromVariables) - .put(matrixEnv); - allEnvs.add(completeEnv); - } - - return allEnvs; - } -} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/IntegrationJobRunner.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/IntegrationJobRunner.java index ed2c03fd2d..1aeec9bcf3 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/IntegrationJobRunner.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/IntegrationJobRunner.java @@ -5,47 +5,47 @@ public class IntegrationJobRunner { - private final IntegrationTarget target; - private final ITCollection itCollection; - private final IntegrationJob job; + private final Target target; + private final SpecFiles itCollection; + private final Job job; - public IntegrationJobRunner( IntegrationTarget target, ITCollection itCollection, IntegrationJob job) { + public IntegrationJobRunner(Target target, SpecFiles itCollection, Job job) { this.target = target; this.itCollection = itCollection; this.job = job; } - public void run() { - - target.jobStarting(job); - var allEnvs = job.allEnvironments(); - - List allTests = new ArrayList<>(); - job.tests() - .forEach( - testName -> { - allTests.addAll(itCollection.testByName(testName)); - }); - - try{ - - for (IntegrationTest test : allTests) { - for (IntegrationEnv env : allEnvs) { - - try{ - target.testStarting(test, env); - var testRunner = new IntegrationTestRunner( itCollection, target, test, env); - testRunner.run(); - } - finally{ - target.testFinished(test, env); - } - - } - } - } - finally { - target.jobFinished(job); - } - } +// public void run() { +// +// target.jobStarting(job); +// var allEnvs = job.allEnvironments(); +// +// List allTests = new ArrayList<>(); +// job.tests() +// .forEach( +// testName -> { +// allTests.addAll(itCollection.testByName(testName)); +// }); +// +// try{ +// +// for (TestSuite test : allTests) { +// for (TestEnvironment env : allEnvs) { +// +// try{ +// target.testStarting(test, env); +// var testRunner = new IntegrationTestRunner( itCollection, target, test, env); +// testRunner.run(); +// } +// finally{ +// target.testFinished(test, env); +// } +// +// } +// } +// } +// finally { +// target.jobFinished(job); +// } +// } } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/IntegrationTarget.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/IntegrationTarget.java deleted file mode 100644 index f2f55c6663..0000000000 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/IntegrationTarget.java +++ /dev/null @@ -1,42 +0,0 @@ -package io.stargate.sgv2.jsonapi.api.v1.vectorize; - -import java.util.HashMap; - -public class IntegrationTarget { - - private final Target target; - private final Backend backend; - private final IntegrationEnv env; - - public IntegrationTarget(Target target) { - this.target = target; - this.env = new IntegrationEnv(new HashMap<>()); - - this.backend = switch (target.backend()) { - case "cassandra" -> new CassandraBackend(); - case "astra" -> new AstraBackend(); - default -> throw new IllegalArgumentException("Unknown backend: " + target.backend()); - }; - } - - public Connection connection(){ - return target.connection(); - } - - public APIRequest apiRequest(TestCommand testCommand, IntegrationEnv env){ - return new APIRequest(target.connection(), env, testCommand.withEnvironment(env)); - } - - public void workflowStarting(IntegrationWorkflow workflow){ } - public void workflowFinished(IntegrationWorkflow workflow){ } - - public void jobStarting(IntegrationJob job){ - backend.jobStarting(this, job); - } - public void jobFinished(IntegrationJob job){ - backend.jobFinished(this, job); - } - - public void testStarting(IntegrationTest test, IntegrationEnv env){ } - public void testFinished(IntegrationTest test, IntegrationEnv env){ } -} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/IntegrationTestRunner.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/IntegrationTestRunner.java index 9819fcc691..59cc5015ce 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/IntegrationTestRunner.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/IntegrationTestRunner.java @@ -1,9 +1,7 @@ package io.stargate.sgv2.jsonapi.api.v1.vectorize; import com.fasterxml.jackson.databind.ObjectMapper; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions.BodyAssertion; import io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions.TestAssertion; -import org.apache.commons.lang3.RandomStringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -13,37 +11,32 @@ public class IntegrationTestRunner extends RunnerBase { private static final Logger LOGGER = LoggerFactory.getLogger(IntegrationTestRunner.class); - // keyspace automatically created in this test - protected static final String keyspaceName = - "ks" + RandomStringUtils.insecure().nextAlphanumeric(16); private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); - private final ITCollection itCollection; - private final IntegrationTarget target; - private final IntegrationTest integrationTest; - private final IntegrationEnv integrationEnv; + private final TestPlan testPlan; + private final TestSuite testSuite; + private final TestEnvironment testEnvironment; public IntegrationTestRunner( - ITCollection itCollection, IntegrationTarget target, IntegrationTest integrationTest, IntegrationEnv integrationEnv) { - this.itCollection = itCollection; - this.target = target; - this.integrationTest = integrationTest; - this.integrationEnv = integrationEnv; + TestPlan testPlan, TestSuite testSuite, TestEnvironment testEnvironment) { + this.testPlan = testPlan; + this.testSuite = testSuite; + this.testEnvironment = testEnvironment; } @Override - protected IntegrationEnv integrationEnv() { - return integrationEnv; + protected TestEnvironment integrationEnv() { + return testEnvironment; } public void run() { - LOGGER.info("Starting Integration Test with env={}", integrationEnv); - for (TestCommand setupCommand : integrationTest.setup()) { - var setupRequest = new TestRequest(setupCommand, target, integrationEnv, TestAssertion.forSuccess(setupCommand.commandName())); + LOGGER.info("Starting Integration Test with env={}", testEnvironment); + for (TestCommand setupCommand : testSuite.setup()) { + var setupRequest = new TestRequest(setupCommand, testPlan.target(), testEnvironment, TestAssertion.forSuccess(setupCommand.commandName())); var setupResponse = setupRequest.execute(); - var testCaseResult = setupResponse.validate(integrationTest, null); + var testCaseResult = setupResponse.validate(testSuite, null); if (testCaseResult.failed()){ LOGGER.warn("TestSetup FAILED: test.name={}, testCase.name={}, failedAssertion={}, error={}", @@ -56,10 +49,10 @@ public void run() { } - for (TestCase testCase : integrationTest.tests()) { - var testRequest = new TestRequest(testCase.command(), target, integrationEnv, TestAssertion.buildAssertions(testCase)); + for (TestCase testCase : testSuite.tests()) { + var testRequest = new TestRequest(testCase.command(), testPlan.target(), testEnvironment, TestAssertion.buildAssertions(testCase)); var testResponse = testRequest.execute(); - var testCaseResult = testResponse.validate(integrationTest, testCase); + var testCaseResult = testResponse.validate(testSuite, testCase); if (testCaseResult.failed()){ LOGGER.warn("TestCase FAILED: test.name={}, testCase.name={}, failedAssertion={}, error={}", @@ -71,12 +64,12 @@ public void run() { } } - for (TestCommand cleanupCommand : integrationTest.cleanup()) { + for (TestCommand cleanupCommand : testSuite.cleanup()) { - var cleanupRequest = new TestRequest(cleanupCommand, target, integrationEnv, TestAssertion.forSuccess(cleanupCommand.commandName())); + var cleanupRequest = new TestRequest(cleanupCommand, testPlan.target(), testEnvironment, TestAssertion.forSuccess(cleanupCommand.commandName())); var cleanupResponse = cleanupRequest.execute(); - var testCaseResult = cleanupResponse.validate(integrationTest, null); + var testCaseResult = cleanupResponse.validate(testSuite, null); if (testCaseResult.failed()){ LOGGER.warn("TestCleanup FAILED: test.name={}, testCase.name={}, failedAssertion={}, error={}", diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/IntegrationWorkflow.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/IntegrationWorkflow.java deleted file mode 100644 index cc91079094..0000000000 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/IntegrationWorkflow.java +++ /dev/null @@ -1,11 +0,0 @@ -package io.stargate.sgv2.jsonapi.api.v1.vectorize; - -import java.util.List; - -public record IntegrationWorkflow(ITMetadata meta, List jobs) implements ITElement { - - @Override - public ITElementKind kind() { - return ITElementKind.WORKFLOW; - } -} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/Job.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/Job.java new file mode 100644 index 0000000000..14d851175d --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/Job.java @@ -0,0 +1,103 @@ +package io.stargate.sgv2.jsonapi.api.v1.vectorize; + +import org.junit.jupiter.api.DynamicContainer; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Stream; + +import static org.junit.jupiter.api.DynamicContainer.dynamicContainer; + +public record Job( + + TestSpecMeta meta, + Map fromEnvironment, + Map variables, + Map> matrix, + List tests) { + + private static final Logger LOGGER = LoggerFactory.getLogger(Job.class); + + + public DynamicContainer testNode(TestPlan testPlan) { + + var desc = "Job: %s ".formatted( + meta.name()); + + testPlan.updateJobForTarget(this); + var allEnvs = allEnvironments(testPlan); + var testSuiteNodes = testSuites(testPlan) + .map(testSuite -> testSuite.testNode(testPlan, allEnvs)); + + return dynamicContainer( + desc, + testPlan.addLifecycle(this, testSuiteNodes)); + } + + public Stream testSuites(TestPlan testPlan) { + List allTests = new ArrayList<>(); + tests() + .forEach( + testName -> { + allTests.addAll(testPlan.specFiles().testByName(testName)); + }); + + return allTests.stream(); + } + public TestEnvironment withoutMatrix(TestPlan testPlan) { + + var fromEnv = new TestEnvironment(); + + for (Map.Entry entry : fromEnvironment.entrySet()) { + var value = System.getenv(entry.getValue()); + if (value== null) { + throw new RuntimeException("Environment variable " + entry.getValue() + " is undefined"); + } + fromEnv.put(entry.getKey(), value); + } + + var fromVariables = new TestEnvironment(variables); + + return fromEnv.clone().put(fromVariables); + } + public List allEnvironments(TestPlan testPlan) { + + var fromEnv = new TestEnvironment(); + + for (Map.Entry entry : fromEnvironment.entrySet()) { + var value = System.getenv(entry.getValue()); + if (value== null) { + throw new RuntimeException("Environment variable " + entry.getValue() + " is undefined"); + } + fromEnv.put(entry.getKey(), value); + } + + var fromVariables = new TestEnvironment(variables); + + // TODO: handle more matrix + List fromMatrix = new ArrayList<>(); + matrix.get("MODEL").forEach( + model -> { + var env = new TestEnvironment(); + env.put("MODEL", model); + fromMatrix.add(env); + }); + + List allEnvs = new ArrayList<>(); + for (var matrixEnv : fromMatrix) { + + var completeEnv = fromEnv + .clone() + .put(fromVariables) + .put(matrixEnv); + + allEnvs.add(completeEnv); + } + + return allEnvs; + } +} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/RunnerBase.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/RunnerBase.java index c39a23a459..017c747297 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/RunnerBase.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/RunnerBase.java @@ -1,15 +1,9 @@ package io.stargate.sgv2.jsonapi.api.v1.vectorize; -import io.restassured.http.ContentType; -import io.restassured.specification.RequestSpecification; -import io.stargate.sgv2.jsonapi.config.constants.HttpConstants; - -import java.util.Map; - import static io.restassured.RestAssured.given; public abstract class RunnerBase { - protected abstract IntegrationEnv integrationEnv(); + protected abstract TestEnvironment integrationEnv(); } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/SpecFile.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/SpecFile.java new file mode 100644 index 0000000000..5bcfc17bb4 --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/SpecFile.java @@ -0,0 +1,22 @@ +package io.stargate.sgv2.jsonapi.api.v1.vectorize; + +import com.fasterxml.jackson.databind.JsonNode; + +public class SpecFile { + + private final TestSpec spec; + private final JsonNode rootNode; + + SpecFile(TestSpec spec, JsonNode rootNode) { + this.spec = spec; + this.rootNode = rootNode; + } + + public TestSpec spec() { + return spec; + } + + public JsonNode root() { + return rootNode; + } +} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/ITCollection.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/SpecFiles.java similarity index 70% rename from src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/ITCollection.java rename to src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/SpecFiles.java index ba7c84a994..1f363a2817 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/ITCollection.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/SpecFiles.java @@ -19,64 +19,68 @@ import java.util.List; import java.util.stream.Stream; -public class ITCollection { +/** + * Collection of all the test spec files read for this execution. + */ +public class SpecFiles { private static final ObjectMapper MAPPER = new ObjectMapper(); - private final Configuration config = + private static final Configuration JSON_PATH_CONFIG = Configuration.builder() .jsonProvider(new JacksonJsonNodeJsonProvider()) .mappingProvider(new JacksonMappingProvider()) .build(); - List itFiles; + List specFiles; - private ITCollection(List itFiles) { - this.itFiles = itFiles; - for (ITFile file : itFiles) { - if (file.element() instanceof IntegrationTest it){ + private SpecFiles(List specFiles) { + this.specFiles = specFiles; + + for (SpecFile file : specFiles) { + if (file.spec() instanceof TestSuite it){ it.expand(this); } } } - public IntegrationWorkflow workflowFirstByName(String name) { + public Workflow workflowFirstByName(String name) { var path = "$.meta[?(@.name == '%s')]".formatted(name); var itFile = - match(ITElement.ITElementKind.WORKFLOW, path).stream() + match(TestSpec.TestSpecKind.WORKFLOW, path).stream() .findFirst() .orElseThrow( () -> new IllegalArgumentException( "No IT file found with meta.name == '%s'".formatted(name))); - return (IntegrationWorkflow) itFile.element(); + return (Workflow) itFile.spec(); } - public IntegrationTest testFirstByName(String name) { + public TestSuite testFirstByName(String name) { return testByName(name).getFirst(); } - public List testByName(String name) { + public List testByName(String name) { var path = "$.meta[?(@.name == '%s')]".formatted(name); - return match(ITElement.ITElementKind.TEST, path).stream() - .map(itFile -> (IntegrationTest) itFile.element()) + return match(TestSpec.TestSpecKind.TEST, path).stream() + .map(itFile -> (TestSuite) itFile.spec()) .toList(); } - private List byKind(ITElement.ITElementKind kind) { - return itFiles.stream().filter(itFile -> itFile.element().kind() == kind).toList(); + public Stream byKind(TestSpec.TestSpecKind kind) { + return specFiles.stream().filter(itFile -> itFile.spec().kind() == kind); } - private List match(ITElement.ITElementKind kind, String jsonPath) { + private List match(TestSpec.TestSpecKind kind, String jsonPath) { var compiled = JsonPath.compile(jsonPath); - return byKind(kind).stream().filter(itFile -> hasMatch(itFile.root(), compiled)).toList(); + return byKind(kind).filter(itFile -> hasMatch(itFile.root(), compiled)).toList(); } private boolean hasMatch(JsonNode root, JsonPath compiled) { - var pathResult = JsonPath.using(config).parse(root).read(compiled); + var pathResult = JsonPath.using(JSON_PATH_CONFIG).parse(root).read(compiled); return switch (pathResult) { case null -> false; @@ -88,24 +92,24 @@ private boolean hasMatch(JsonNode root, JsonPath compiled) { }; } - static ITCollection loadAll(String path) { + static SpecFiles loadAll(String path) { final Path dir = resourceDir(path); - List itFiles = new ArrayList<>(); + List itFiles = new ArrayList<>(); try (Stream s = Files.walk(dir)) { itFiles = s.filter(Files::isRegularFile) .filter(p -> p.getFileName().toString().endsWith(".json")) - .map(ITCollection::loadOne) + .map(SpecFiles::loadOne) .toList(); } catch (IOException e) { throw new UncheckedIOException("Failed reading test resources under: " + dir, e); } - return new ITCollection(itFiles); + return new SpecFiles(itFiles); } - private static ITFile loadOne(Path file) { + private static SpecFile loadOne(Path file) { try { var root = MAPPER.readTree(file.toFile()); @@ -117,12 +121,12 @@ private static ITFile loadOne(Path file) { var kind = kindNode.asText(); var element = switch (kind.toUpperCase()) { - case "TEST" -> MAPPER.treeToValue(root, IntegrationTest.class); - case "WORKFLOW" -> MAPPER.treeToValue(root, IntegrationWorkflow.class); + case "TEST" -> MAPPER.treeToValue(root, TestSuite.class); + case "WORKFLOW" -> MAPPER.treeToValue(root, Workflow.class); default -> throw new IllegalArgumentException("Unknown meta.kind '" + kind + "' in " + file); }; - return new ITFile(element, root); + return new SpecFile(element, root); } catch (IOException e) { throw new UncheckedIOException("Failed parsing JSON file: " + file, e); } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/Target.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/Target.java index 6b458d9ea5..aaad219945 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/Target.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/Target.java @@ -1,4 +1,50 @@ package io.stargate.sgv2.jsonapi.api.v1.vectorize; -public record Target (String name, String backend, Connection connection) { +import java.util.HashMap; + +public class Target { + + private final TargetConfiguration targetConfiguration; + private final Backend backend; + private final TestEnvironment env; + + public Target(TargetConfiguration targetConfiguration) { + this.targetConfiguration = targetConfiguration; + this.env = new TestEnvironment(new HashMap<>()); + + this.backend = switch (targetConfiguration.backend()) { + case "cassandra" -> new CassandraBackend(); + case "astra" -> new AstraBackend(); + default -> throw new IllegalArgumentException("Unknown backend: " + targetConfiguration.backend()); + }; + } + + public TargetConfiguration configuration(){ + return targetConfiguration; + } + public Connection connection(){ + return targetConfiguration.connection(); + } + + public void updateJobForTarget(Job job){ + backend.updateJobForTarget(job); + } + + + public APIRequest apiRequest(TestCommand testCommand, TestEnvironment env){ + return new APIRequest(targetConfiguration.connection(), env, testCommand.withEnvironment(env)); + } + + public void workflowStarting(TestPlan testPlan, Workflow workflow){ } + public void workflowFinished(TestPlan testPlan,Workflow workflow){ } + + public void jobStarting(TestPlan testPlan,Job job){ + backend.jobStarting(testPlan, job); + } + public void jobFinished(TestPlan testPlan,Job job){ + backend.jobFinished(testPlan, job); + } + + public void testRunStarting(TestPlan testPlan,TestSuite test, TestEnvironment env){ } + public void testRunFinished(TestPlan testPlan,TestSuite test, TestEnvironment env){ } } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TargetConfiguration.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TargetConfiguration.java new file mode 100644 index 0000000000..58874c620d --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TargetConfiguration.java @@ -0,0 +1,4 @@ +package io.stargate.sgv2.jsonapi.api.v1.vectorize; + +public record TargetConfiguration(String name, String backend, Connection connection) { +} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/Targets.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TargetConfigurationss.java similarity index 82% rename from src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/Targets.java rename to src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TargetConfigurationss.java index bad582652f..2fe7182b16 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/Targets.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TargetConfigurationss.java @@ -11,14 +11,14 @@ import java.util.List; import java.util.Set; -public record Targets(List targets) { +public record TargetConfigurationss(List targets) { private static final ObjectMapper MAPPER = new ObjectMapper(); - public Targets{ + public TargetConfigurationss { Set seen = new HashSet(); - for (Target target : targets) { + for (TargetConfiguration target : targets) { if (seen.contains(target.name())) { throw new IllegalArgumentException("target name already exists: " + target.name() ); } @@ -26,15 +26,15 @@ public record Targets(List targets) { } } - public Target target(String name) { + public TargetConfiguration configuration(String name) { return targets.stream().filter(target -> target.name().equals(name)).findFirst().orElseThrow(() -> new IllegalArgumentException("target name not found: " + name)); } - static Targets loadAll(String path) { + static TargetConfigurationss loadAll(String path) { final Path dir = resourceDir(path); try { - return MAPPER.readValue(dir.toFile(), Targets.class); + return MAPPER.readValue(dir.toFile(), TargetConfigurationss.class); } catch (IOException e) { throw new RuntimeException(e); } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestCaseResult.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestCaseResult.java index 1ff331cf06..62b8e5d4f3 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestCaseResult.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestCaseResult.java @@ -1,10 +1,9 @@ package io.stargate.sgv2.jsonapi.api.v1.vectorize; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions.BodyAssertion; import io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions.TestAssertion; public record TestCaseResult( - IntegrationTest integrationTest, + TestSuite integrationTest, TestCase testCase, // Nullable TestResponse testResponse, AssertionError error, diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestCommand.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestCommand.java index ffccc858d8..255994aed0 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestCommand.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestCommand.java @@ -1,7 +1,6 @@ package io.stargate.sgv2.jsonapi.api.v1.vectorize; import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ArrayNode; @@ -9,7 +8,6 @@ import com.fasterxml.jackson.databind.node.TextNode; import io.stargate.sgv2.jsonapi.api.model.command.CommandName; -import java.util.UUID; import org.apache.commons.text.StringSubstitutor; public class TestCommand { @@ -86,7 +84,7 @@ private static String commandNameString(ObjectNode request) { return name; } - public ObjectNode withEnvironment(IntegrationEnv env) { + public ObjectNode withEnvironment(TestEnvironment env) { ObjectNode updated = request.deepCopy(); walk(updated, env.substitutor()); return updated; diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/IntegrationEnv.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestEnvironment.java similarity index 62% rename from src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/IntegrationEnv.java rename to src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestEnvironment.java index 57a99c10b6..920f3bad0e 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/IntegrationEnv.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestEnvironment.java @@ -1,8 +1,9 @@ package io.stargate.sgv2.jsonapi.api.v1.vectorize; import org.apache.commons.text.StringSubstitutor; -import org.apache.commons.text.lookup.StringLookup; import org.apache.commons.text.lookup.StringLookupFactory; +import org.junit.jupiter.api.DynamicContainer; +import org.junit.jupiter.api.DynamicTest; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -11,33 +12,49 @@ import java.util.Map; import java.util.Set; import java.util.regex.Pattern; +import java.util.stream.Stream; -public class IntegrationEnv{ +import static org.junit.jupiter.api.DynamicTest.dynamicTest; - private static final Logger LOGGER = LoggerFactory.getLogger(IntegrationEnv.class); +public class TestEnvironment { + + private static final Logger LOGGER = LoggerFactory.getLogger(TestEnvironment.class); private static final Pattern PATTERN_NOT_WORD_CHARS = Pattern.compile("\\W+"); private static final Set SCHEMA_IDENTIFIER = Set.of("KEYSPACE_NAME", "COLLECTION_NAME"); private final Map vars = new HashMap<>(); - public IntegrationEnv(){ + public TestEnvironment(){ this(new HashMap<>()); } - public IntegrationEnv(Map vars) { + public TestEnvironment(Map vars) { this.vars.putAll(vars); } - private IntegrationEnv(IntegrationEnv other){ + public DynamicContainer testNode(TestPlan testPlan, TestSuite testSuite) { + + var desc = "TestEnv: %s ".formatted(this); + + var envNodes = Stream.of( + dynamicTest("Running TestSuite", () -> new IntegrationTestRunner( testPlan, testSuite, this).run()) + ); + + return DynamicContainer.dynamicContainer( + desc, + testPlan.addLifecycle(testSuite, this, envNodes)); + } + + private TestEnvironment(TestEnvironment other){ this.vars.putAll(other.vars); } - public IntegrationEnv clone(){ - return new IntegrationEnv(this); + public TestEnvironment clone(){ + return new TestEnvironment(this); } - public IntegrationEnv put(IntegrationEnv other){ + public TestEnvironment put(TestEnvironment other){ this.vars.putAll(other.vars); return this; } @@ -50,7 +67,7 @@ public String requiredValue(String name){ if (vars.containsKey(name)){ return get(name); } - throw new RuntimeException(String.format("Required env var %s not found", name)); + throw new RuntimeException(String.format("Required env var not found name:%s, defined: %s", name, String.join(", ", vars.keySet()))); } public StringSubstitutor substitutor(){ diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestPlan.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestPlan.java new file mode 100644 index 0000000000..550aa2b5a0 --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestPlan.java @@ -0,0 +1,87 @@ +package io.stargate.sgv2.jsonapi.api.v1.vectorize; + +import org.junit.jupiter.api.DynamicContainer; +import org.junit.jupiter.api.DynamicNode; + +import java.util.List; +import java.util.Set; +import java.util.stream.Stream; + +import static org.junit.jupiter.api.DynamicContainer.dynamicContainer; +import static org.junit.jupiter.api.DynamicTest.dynamicTest; + +public record TestPlan(Target target, SpecFiles specFiles, Set workflows){ + + + public static TestPlan create(String targetName, List workflows){ + var targetConfigs = TargetConfigurationss.loadAll("integration-tests/targets/targets.json"); + var target = new Target(targetConfigs.configuration(targetName)); + + var specFiles = SpecFiles.loadAll("integration-tests/vectorize"); + + return new TestPlan(target, specFiles, Set.copyOf(workflows)); + } + + public Stream selectedWorkflows(){ + + return specFiles.byKind(TestSpec.TestSpecKind.WORKFLOW) + .filter(specFile -> + workflows.isEmpty() || workflows.contains(specFile.spec().meta().name()) + ) + .map(testSpec -> (Workflow) testSpec.spec()); + } + + public Stream testNode() { + + var desc = "TestPlan: %s on %s workflows %s".formatted( + target.configuration().name(), + target.configuration().backend(), + workflows.isEmpty() ? "" : String.join(", ", workflows) + ); + + return Stream.of( + dynamicContainer( + desc, + selectedWorkflows() + .map(workflow -> workflow.testNode(this)) + ) + ); + } + + public void updateJobForTarget(Job job){ + target.updateJobForTarget(job); + } + + public Stream addLifecycle(Workflow workflow, Stream dynamicNodes){ + + var starting = dynamicTest("Workflow Starting: %s".formatted(workflow.meta().name()), () -> target.workflowStarting(this,workflow)); + var finished = dynamicTest("Workflow Finished: %s".formatted(workflow.meta().name()), () -> target.workflowFinished(this,workflow)); + + return Stream.concat( + Stream.of(starting), + Stream.concat(dynamicNodes, Stream.of(finished)) + ); + } + + public Stream addLifecycle(Job job, Stream dynamicNodes){ + + var starting = dynamicTest("Job Starting: %s".formatted(job.meta().name()), () -> target.jobStarting(this,job)); + var finished = dynamicTest("Job Finished: %s".formatted(job.meta().name()), () -> target.jobFinished(this,job)); + + return Stream.concat( + Stream.of(starting), + Stream.concat(dynamicNodes, Stream.of(finished)) + ); + } + + public Stream addLifecycle(TestSuite testSuite, TestEnvironment environment, Stream dynamicNodes){ + + var starting = dynamicTest("TestRun Starting: %s".formatted(testSuite.meta().name()), () -> target.testRunStarting(this,testSuite, environment)); + var finished = dynamicTest("TestRun Finished: %s".formatted(testSuite.meta().name()), () -> target.testRunFinished(this,testSuite, environment)); + + return Stream.concat( + Stream.of(starting), + Stream.concat(dynamicNodes, Stream.of(finished)) + ); + } +} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestRequest.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestRequest.java index 5d9145c9a1..a762ccf082 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestRequest.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestRequest.java @@ -1,13 +1,12 @@ package io.stargate.sgv2.jsonapi.api.v1.vectorize; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions.BodyAssertion; import io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions.TestAssertion; import java.util.List; public record TestRequest(TestCommand testCommand, - IntegrationTarget integrationTarget, - IntegrationEnv integrationEnv, + Target integrationTarget, + TestEnvironment integrationEnv, List testAssertions) { public TestResponse execute(){ diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestResponse.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestResponse.java index b3607c192e..6bffe85324 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestResponse.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestResponse.java @@ -1,6 +1,5 @@ package io.stargate.sgv2.jsonapi.api.v1.vectorize; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions.BodyAssertion; import io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions.TestAssertion; public record TestResponse( @@ -9,11 +8,11 @@ public record TestResponse( APIResponse apiResponse ) { - public TestCaseResult validate(IntegrationTest integrationTest, TestCase testCase){ - return validate(integrationTest, testCase, false); + public TestCaseResult validate(TestSuite integrationTest, TestCase testCase){ + return validate(integrationTest, testCase, true); } - public TestCaseResult validate(IntegrationTest integrationTest, TestCase testCase, boolean throwOnError) { + public TestCaseResult validate(TestSuite integrationTest, TestCase testCase, boolean throwOnError) { AssertionError assertionError = null; TestAssertion failedAssertion = null; diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/ITElement.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestSpec.java similarity index 55% rename from src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/ITElement.java rename to src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestSpec.java index e2314448be..01f0a1ce73 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/ITElement.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestSpec.java @@ -1,14 +1,14 @@ package io.stargate.sgv2.jsonapi.api.v1.vectorize; -public interface ITElement { +public interface TestSpec { - ITElementKind kind(); + TestSpecKind kind(); - ITMetadata meta(); + TestSpecMeta meta(); // void setJson(JsonNode json); - enum ITElementKind { + enum TestSpecKind { TEST, WORKFLOW } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestSpecMeta.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestSpecMeta.java new file mode 100644 index 0000000000..fcf919b4a2 --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestSpecMeta.java @@ -0,0 +1,6 @@ +package io.stargate.sgv2.jsonapi.api.v1.vectorize; + +import java.util.List; + +public record TestSpecMeta(String name, String kind, + List tags) {} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/IntegrationTest.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestSuite.java similarity index 55% rename from src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/IntegrationTest.java rename to src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestSuite.java index 0c869816b5..cd1a2051f7 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/IntegrationTest.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestSuite.java @@ -1,17 +1,34 @@ package io.stargate.sgv2.jsonapi.api.v1.vectorize; +import org.junit.jupiter.api.DynamicContainer; + import java.util.ArrayList; import java.util.List; +import java.util.stream.Stream; + +import static org.junit.jupiter.api.DynamicContainer.dynamicContainer; -public record IntegrationTest(ITMetadata meta, List setup, List tests, List cleanup) - implements ITElement { +public record TestSuite(TestSpecMeta meta, List setup, List tests, List cleanup) + implements TestSpec { @Override - public ITElementKind kind() { - return ITElementKind.TEST; + public TestSpecKind kind() { + return TestSpecKind.TEST; + } + + public DynamicContainer testNode(TestPlan testPlan, List allEnvs) { + + var desc = "TestSuite: %s ".formatted( + meta.name()); + + return dynamicContainer( + desc, + allEnvs.stream() + .map(testEnv -> testEnv.testNode(testPlan, this)) + ); } - public void expand(ITCollection itCollection) { + public void expand(SpecFiles itCollection) { List expandedSetup = new ArrayList<>(); for (TestCommand command : setup) { diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/VectorizeTestFactory.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/VectorizeTestFactory.java new file mode 100644 index 0000000000..daf2137038 --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/VectorizeTestFactory.java @@ -0,0 +1,50 @@ +package io.stargate.sgv2.jsonapi.api.v1.vectorize; + +import org.junit.jupiter.api.DynamicContainer; +import org.junit.jupiter.api.TestFactory; +import org.junit.jupiter.api.parallel.Execution; +import org.junit.jupiter.api.parallel.ExecutionMode; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Collection; +import java.util.List; +import java.util.stream.Stream; + +public class VectorizeTestFactory { + private static final Logger LOGGER = LoggerFactory.getLogger(VectorizeTestFactory.class); + + + private static final int TEST_PARALLELISM = 8; // ← set this + + @TestFactory + Stream jobs() { + + var testPlan = TestPlan.create("local", List.of("all-vectorize-workflow")); + + return testPlan.testNode(); + + // Parallel +// int maxConcurrentJobs = 2; +// integrationTarget.workflowStarting(workflow); +// try { +// var executor = java.util.concurrent.Executors.newFixedThreadPool(maxConcurrentJobs); +// try { +// var futures = jobs.stream() +// .map(job -> java.util.concurrent.CompletableFuture.runAsync(() -> { +// LOGGER.info("Starting job {}", job.meta()); +// new IntegrationJobRunner(integrationTarget, itCollection, job).run(); +// }, executor)) +// .toList(); +// +// // wait for all, fail if any failed +// futures.forEach(java.util.concurrent.CompletableFuture::join); +// } finally { +// executor.shutdown(); +// } +// } finally { +// integrationTarget.workflowFinished(workflow); +// } + + } +} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/VectorizeUnit.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/VectorizeUnit.java index 4a8f88e1e3..782e37bc7b 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/VectorizeUnit.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/VectorizeUnit.java @@ -1,37 +1,38 @@ package io.stargate.sgv2.jsonapi.api.v1.vectorize; +import org.eclipse.aether.util.artifact.OverlayArtifactTypeRegistry; import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.List; + public class VectorizeUnit { private static final Logger LOGGER = LoggerFactory.getLogger(VectorizeUnit.class); @Test public void doTest() { - var targets = Targets.loadAll("integration-tests/targets/targets.json"); - var integrationTarget = new IntegrationTarget(targets.target("local")); - - var itCollection = ITCollection.loadAll("integration-tests/vectorize"); - - var workflow = itCollection.workflowFirstByName("all-vectorize-workflow"); - - var jobs = workflow.jobs().stream() - .filter(job -> !job.meta().tags().contains("disabled")) - .toList(); - +// var targets = TargetConfigurationss.loadAll("integration-tests/targets/targets.json"); +// var integrationTarget = new Target(targets.configuration("local")); +// +// var itCollection = ITCollection.loadAll("integration-tests/vectorize"); +// +// var workflow = itCollection.workflowFirstByName("all-vectorize-workflow"); - // SEQUENTIAL - integrationTarget.workflowStarting(workflow); - try { - for (var job : jobs) { - LOGGER.info("Starting job {}", job.meta()); - new IntegrationJobRunner(integrationTarget, itCollection, job).run(); - } - } finally { - integrationTarget.workflowFinished(workflow); - } +// var testPlan = TestPlan.create("local", List.of("all-vectorize-workflow")); +// +// +// // SEQUENTIAL +// integrationTarget.workflowStarting(workflow); +// try { +// for (var job : jobs) { +// LOGGER.info("Starting job {}", job.meta()); +// new IntegrationJobRunner(integrationTarget, itCollection, job).run(); +// } +// } finally { +// integrationTarget.workflowFinished(workflow); +// } // Parallel // int maxConcurrentJobs = 2; diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/Workflow.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/Workflow.java new file mode 100644 index 0000000000..ed3666de1e --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/Workflow.java @@ -0,0 +1,36 @@ +package io.stargate.sgv2.jsonapi.api.v1.vectorize; + +import org.junit.jupiter.api.DynamicContainer; + +import java.util.List; +import java.util.stream.Stream; + +import static org.junit.jupiter.api.DynamicContainer.dynamicContainer; + +public record Workflow(TestSpecMeta meta, List jobs) implements TestSpec { + + @Override + public TestSpecKind kind() { + return TestSpecKind.WORKFLOW; + } + + public Stream activeJobs(){ + return jobs().stream() + .filter(job -> !job.meta().tags().contains("disabled")); + } + + + public DynamicContainer testNode(TestPlan testPlan) { + + var desc = "Workflow: %s ".formatted( + meta.name()); + + var jobNodes = activeJobs() + .map(job -> job.testNode(testPlan)); + + return dynamicContainer( + desc, + testPlan.addLifecycle(this, jobNodes)); + } + +} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/TestAssertion.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/TestAssertion.java index ca0073c79c..7bf168ccf3 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/TestAssertion.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/TestAssertion.java @@ -26,7 +26,7 @@ static List forSuccess(CommandName commandName) { case INSERT_ONE, INSERT_MANY -> { assertions.add(Response.isWriteSuccess(null)); } - case DELETE_COLLECTION, CREATE_COLLECTION -> { + case CREATE_KEYSPACE, DROP_KEYSPACE, CREATE_NAMESPACE, DROP_NAMESPACE, DELETE_COLLECTION, CREATE_COLLECTION -> { assertions.add(Response.isDDLSuccess(null)); } } diff --git a/src/test/resources/integration-tests/vectorize/vectorize-base.json b/src/test/resources/integration-tests/vectorize/vectorize-base.json index a7d5969dd8..a27f4a9e33 100644 --- a/src/test/resources/integration-tests/vectorize/vectorize-base.json +++ b/src/test/resources/integration-tests/vectorize/vectorize-base.json @@ -56,7 +56,7 @@ }, "asserts": { "response.isFindSuccess": null, - "documents.count": 4 + "documents.count": 3 } }, { From 1d07e97244979f3b22e74bf29128ae173b80d1d2 Mon Sep 17 00:00:00 2001 From: Aaron Morton Date: Tue, 24 Feb 2026 06:39:37 +1300 Subject: [PATCH 06/89] WIP --- .../api/v1/vectorize/AstraBackend.java | 10 ++++ .../api/v1/vectorize/CassandraBackend.java | 3 ++ .../api/v1/vectorize/VectorizeAstra.java | 47 +++++++++++++++++++ .../jsonapi/api/v1/vectorize/VectorizeIT.java | 19 ++++---- .../integration-tests/targets/targets.json | 15 ++++-- .../vectorize/vectorize-workflow.json | 14 +++--- 6 files changed, 90 insertions(+), 18 deletions(-) create mode 100644 src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/VectorizeAstra.java diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/AstraBackend.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/AstraBackend.java index 5169e824c1..e7296f0e4d 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/AstraBackend.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/AstraBackend.java @@ -1,4 +1,14 @@ package io.stargate.sgv2.jsonapi.api.v1.vectorize; +import org.apache.commons.lang3.RandomStringUtils; + +import static io.stargate.sgv2.jsonapi.api.v1.vectorize.TestEnvironment.toSafeSchemaIdentifier; + public class AstraBackend extends Backend { + + @Override + public void updateJobForTarget(Job job) { + + job.variables().put("KEYSPACE_NAME", "default_keyspace"); + } } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/CassandraBackend.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/CassandraBackend.java index 8d6dca61ef..f8080b9146 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/CassandraBackend.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/CassandraBackend.java @@ -6,6 +6,9 @@ import static io.stargate.sgv2.jsonapi.api.v1.vectorize.TestEnvironment.toSafeSchemaIdentifier; +/** + * Cassandra:Y2Fzc2FuZHJh:Y2Fzc2FuZHJh + */ public class CassandraBackend extends Backend { private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/VectorizeAstra.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/VectorizeAstra.java new file mode 100644 index 0000000000..effb9182aa --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/VectorizeAstra.java @@ -0,0 +1,47 @@ +package io.stargate.sgv2.jsonapi.api.v1.vectorize; + +import org.junit.jupiter.api.DynamicContainer; +import org.junit.jupiter.api.TestFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.List; +import java.util.stream.Stream; + +public class VectorizeAstra { + private static final Logger LOGGER = LoggerFactory.getLogger(VectorizeAstra.class); + + + private static final int TEST_PARALLELISM = 8; // ← set this + + @TestFactory + Stream jobs() { + + var testPlan = TestPlan.create("astra-dev", List.of("all-vectorize-workflow")); + + return testPlan.testNode(); + + // Parallel +// int maxConcurrentJobs = 2; +// integrationTarget.workflowStarting(workflow); +// try { +// var executor = java.util.concurrent.Executors.newFixedThreadPool(maxConcurrentJobs); +// try { +// var futures = jobs.stream() +// .map(job -> java.util.concurrent.CompletableFuture.runAsync(() -> { +// LOGGER.info("Starting job {}", job.meta()); +// new IntegrationJobRunner(integrationTarget, itCollection, job).run(); +// }, executor)) +// .toList(); +// +// // wait for all, fail if any failed +// futures.forEach(java.util.concurrent.CompletableFuture::join); +// } finally { +// executor.shutdown(); +// } +// } finally { +// integrationTarget.workflowFinished(workflow); +// } + + } +} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/VectorizeIT.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/VectorizeIT.java index c8fb36e65c..5ae06c4758 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/VectorizeIT.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/VectorizeIT.java @@ -4,21 +4,22 @@ import io.quarkus.test.junit.QuarkusIntegrationTest; import io.stargate.sgv2.jsonapi.api.v1.AbstractCollectionIntegrationTestBase; import io.stargate.sgv2.jsonapi.testresource.DseTestResource; +import org.junit.jupiter.api.DynamicContainer; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestFactory; + +import java.util.List; +import java.util.stream.Stream; @QuarkusIntegrationTest @WithTestResource(value = DseTestResource.class) public class VectorizeIT extends AbstractCollectionIntegrationTestBase { - @Test - public void doTest() { + @TestFactory + Stream jobs() { + + var testPlan = TestPlan.create("integration", List.of("all-vectorize-workflow")); -// var itCollection = ITCollection.loadAll("integration-tests/vectorize"); -// -// var workflow = itCollection.workflowFirstByName("all-vectorize-workflow"); -// -// var job = workflow.jobs().getFirst(); -// -// new IntegrationJobRunner(itCollection, job).run(); + return testPlan.testNode(); } } diff --git a/src/test/resources/integration-tests/targets/targets.json b/src/test/resources/integration-tests/targets/targets.json index 97449b6732..fc65150a81 100644 --- a/src/test/resources/integration-tests/targets/targets.json +++ b/src/test/resources/integration-tests/targets/targets.json @@ -9,13 +9,22 @@ "basePath": "/v1" } }, + { + "name": "integration", + "backend": "cassandra", + "connection": { + "domain": "http://localhost", + "port": 9080, + "basePath": "/v1" + } + }, { "name": "astra-dev", "backend": "astra", "connection": { - "domain": "http://localhost", - "port": 8181, - "basePath": "/v1/json" + "domain": "https://830c0231-8c79-4212-b148-8c4bd467c917-us-west-2.apps.astra-dev.datastax.com", + "port": 443, + "basePath": "/api/json/v1" } } ] diff --git a/src/test/resources/integration-tests/vectorize/vectorize-workflow.json b/src/test/resources/integration-tests/vectorize/vectorize-workflow.json index 851f9dc40b..6045d6a7dc 100644 --- a/src/test/resources/integration-tests/vectorize/vectorize-workflow.json +++ b/src/test/resources/integration-tests/vectorize/vectorize-workflow.json @@ -10,10 +10,11 @@ "tags": [] }, "fromEnvironment": { - "x-embedding-api-key": "x-embedding-api-key" + "x-embedding-api-key": "x-embedding-api-key", + "Token": "Token" }, "variables": { - "Token": "Cassandra:Y2Fzc2FuZHJh:Y2Fzc2FuZHJh", + "PROVIDER": "openai", "COLLECTION_NAME": "${PROVIDER}-${MODEL}" }, @@ -36,10 +37,11 @@ ] }, "fromEnvironment": { - "x-embedding-api-key": "voyageAI_KEY" + "x-embedding-api-key": "voyageAI_KEY", + "Token" : "Token" + }, "variables": { - "Token": "Cassandra:Y2Fzc2FuZHJh:Y2Fzc2FuZHJh", "PROVIDER": "voyageAI", "COLLECTION_NAME": "${PROVIDER}-${MODEL}" }, @@ -62,10 +64,10 @@ "tags": ["disabled"] }, "fromEnvironment": { - "x-embedding-api-key": "jinaAI_KEY" + "x-embedding-api-key": "jinaAI_KEY", + "Token" : "Token" }, "variables": { - "Token": "Cassandra:Y2Fzc2FuZHJh:Y2Fzc2FuZHJh", "PROVIDER": "jinaAI", "COLLECTION_NAME": "${PROVIDER}-${MODEL}" }, From 5b0a8dc5874b193fa78b26f3bda7027689d514cd Mon Sep 17 00:00:00 2001 From: Aaron Morton Date: Tue, 24 Feb 2026 13:49:56 +1300 Subject: [PATCH 07/89] WIP --- .../jsonapi/api/v1/vectorize/Backend.java | 28 ++++-- .../api/v1/vectorize/CassandraBackend.java | 28 +++--- .../v1/vectorize/IntegrationTestRunner.java | 85 ------------------- .../jsonapi/api/v1/vectorize/RunnerBase.java | 12 ++- .../sgv2/jsonapi/api/v1/vectorize/Target.java | 27 ++++-- .../jsonapi/api/v1/vectorize/TestCase.java | 17 +++- .../api/v1/vectorize/TestCaseResult.java | 4 +- .../api/v1/vectorize/TestEnvironment.java | 27 ++++-- .../jsonapi/api/v1/vectorize/TestPlan.java | 42 +++++---- .../jsonapi/api/v1/vectorize/TestRequest.java | 50 +++++++++-- .../api/v1/vectorize/TestRequestRunner.java | 29 +++++++ .../api/v1/vectorize/TestResponse.java | 50 +++++------ .../jsonapi/api/v1/vectorize/TestSuite.java | 32 ++++++- .../api/v1/vectorize/TestSuiteRunner.java | 50 +++++++++++ .../assertions/AssertionFactory.java | 2 +- .../assertions/AssertionMatcher.java | 8 ++ .../vectorize/assertions/BodyAssertion.java | 19 +---- .../v1/vectorize/assertions/Documents.java | 4 +- .../api/v1/vectorize/assertions/Response.java | 8 +- .../api/v1/vectorize/assertions/Status.java | 2 +- .../{StatusCode.java => Statuscode.java} | 5 +- .../vectorize/assertions/TestAssertion.java | 79 +++++++++++------ .../vectorize/vectorize-base.json | 4 +- 23 files changed, 390 insertions(+), 222 deletions(-) delete mode 100644 src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/IntegrationTestRunner.java create mode 100644 src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestRequestRunner.java create mode 100644 src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestSuiteRunner.java create mode 100644 src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/AssertionMatcher.java rename src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/{StatusCode.java => Statuscode.java} (65%) diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/Backend.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/Backend.java index 1d5c94f8f0..eff80f178d 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/Backend.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/Backend.java @@ -1,20 +1,36 @@ package io.stargate.sgv2.jsonapi.api.v1.vectorize; +import org.junit.jupiter.api.DynamicNode; + +import java.util.Optional; + public abstract class Backend { public void updateJobForTarget(Job job){ } - public void workflowStarting(TestPlan testPlan, Workflow workflow) { } + public Optional beforeWorkflow(TestPlan testPlan, Workflow workflow) { + return Optional.empty(); + } - public void workflowFinished(TestPlan testPlan, Workflow workflow) { } + public Optional afterWorkflow(TestPlan testPlan, Workflow workflow) { + return Optional.empty(); + } - public void jobStarting(TestPlan testPlan, Job job) { } + public Optional beforeJob(TestPlan testPlan, Job job) { + return Optional.empty(); + } - public void jobFinished(TestPlan testPlan, Job job) { } + public Optional afterJob(TestPlan testPlan, Job job) { + return Optional.empty(); + } - public void testStarting(TestPlan testPlan, TestSuite test, TestEnvironment env) { } + public Optional beforeTestSuite(TestPlan testPlan, TestSuite test, TestEnvironment env) { + return Optional.empty(); + } - public void testFinished(TestPlan testPlan, TestSuite test, TestEnvironment env) { } + public Optional afterTestSuite(TestPlan testPlan, TestSuite test, TestEnvironment env) { + return Optional.empty(); + } } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/CassandraBackend.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/CassandraBackend.java index f8080b9146..274550ecfd 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/CassandraBackend.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/CassandraBackend.java @@ -1,8 +1,12 @@ package io.stargate.sgv2.jsonapi.api.v1.vectorize; import com.fasterxml.jackson.databind.ObjectMapper; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions.AssertionMatcher; import io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions.TestAssertion; import org.apache.commons.lang3.RandomStringUtils; +import org.junit.jupiter.api.DynamicNode; + +import java.util.Optional; import static io.stargate.sgv2.jsonapi.api.v1.vectorize.TestEnvironment.toSafeSchemaIdentifier; @@ -15,14 +19,15 @@ public class CassandraBackend extends Backend { @Override public void updateJobForTarget(Job job) { + // max length for keyspace is 48 chars var keyspaceName = toSafeSchemaIdentifier( "job_" + job.meta().name().substring(0, Math.min(job.meta().name().length(), 27)) + "_" + RandomStringUtils.insecure().nextAlphanumeric(16)); job.variables().put("KEYSPACE_NAME", keyspaceName); } @Override - public void jobStarting(TestPlan testPlan, Job job) { - // max length for keyspace is 48 chars + public Optional beforeJob(TestPlan testPlan, Job job) { + var command = TestCommand.fromJson( """ { @@ -32,15 +37,16 @@ public void jobStarting(TestPlan testPlan, Job job) { } """); - var setupRequest = new TestRequest(command, testPlan.target(), job.withoutMatrix(testPlan), TestAssertion.forSuccess(command.commandName())); - - var setupResponse = setupRequest.execute(); - setupResponse.validate(null, null, true); + var env = job.withoutMatrix(testPlan); + var setupRequest = new TestRequest( + env.substitutor().replace("createKeyspace: ${KEYSPACE_NAME}"), + command, testPlan.target(), env, TestAssertion.forSuccess(command.commandName())); + return Optional.of(setupRequest.testNodes()); } @Override - public void jobFinished(TestPlan testPlan, Job job) { + public Optional afterJob(TestPlan testPlan, Job job) { var command = TestCommand.fromJson( """ { @@ -50,9 +56,11 @@ public void jobFinished(TestPlan testPlan, Job job) { } """); - var setupRequest = new TestRequest(command, testPlan.target(), job.withoutMatrix(testPlan), TestAssertion.forSuccess(command.commandName())); + var env = job.withoutMatrix(testPlan); + var setupRequest = new TestRequest( + env.substitutor().replace("dropKeyspace: ${KEYSPACE_NAME}"), + command, testPlan.target(), job.withoutMatrix(testPlan), TestAssertion.forSuccess(command.commandName())); - var setupResponse = setupRequest.execute(); - setupResponse.validate(null, null, true); + return Optional.of(setupRequest.testNodes()); } } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/IntegrationTestRunner.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/IntegrationTestRunner.java deleted file mode 100644 index 59cc5015ce..0000000000 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/IntegrationTestRunner.java +++ /dev/null @@ -1,85 +0,0 @@ -package io.stargate.sgv2.jsonapi.api.v1.vectorize; - -import com.fasterxml.jackson.databind.ObjectMapper; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions.TestAssertion; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import static io.restassured.RestAssured.given; - -public class IntegrationTestRunner extends RunnerBase { - private static final Logger LOGGER = LoggerFactory.getLogger(IntegrationTestRunner.class); - - - private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); - - private final TestPlan testPlan; - private final TestSuite testSuite; - private final TestEnvironment testEnvironment; - - public IntegrationTestRunner( - TestPlan testPlan, TestSuite testSuite, TestEnvironment testEnvironment) { - this.testPlan = testPlan; - this.testSuite = testSuite; - this.testEnvironment = testEnvironment; - } - - @Override - protected TestEnvironment integrationEnv() { - return testEnvironment; - } - - public void run() { - - LOGGER.info("Starting Integration Test with env={}", testEnvironment); - for (TestCommand setupCommand : testSuite.setup()) { - var setupRequest = new TestRequest(setupCommand, testPlan.target(), testEnvironment, TestAssertion.forSuccess(setupCommand.commandName())); - - var setupResponse = setupRequest.execute(); - var testCaseResult = setupResponse.validate(testSuite, null); - - if (testCaseResult.failed()){ - LOGGER.warn("TestSetup FAILED: test.name={}, testCase.name={}, failedAssertion={}, error={}", - testCaseResult.integrationTest().meta().name(), "NULL", testCaseResult.failedAssertion(), String.valueOf(testCaseResult.error())); - } - else{ - LOGGER.info("TestSetup PASSED: test.name={}, testCase.name={}", - testCaseResult.integrationTest().meta().name(), "NULL"); - } - - } - - for (TestCase testCase : testSuite.tests()) { - var testRequest = new TestRequest(testCase.command(), testPlan.target(), testEnvironment, TestAssertion.buildAssertions(testCase)); - var testResponse = testRequest.execute(); - var testCaseResult = testResponse.validate(testSuite, testCase); - - if (testCaseResult.failed()){ - LOGGER.warn("TestCase FAILED: test.name={}, testCase.name={}, failedAssertion={}, error={}", - testCaseResult.integrationTest().meta().name(),testCaseResult.testCase().name(), testCaseResult.failedAssertion(), String.valueOf(testCaseResult.error())); - } - else{ - LOGGER.info("TestCase PASSED: test.name={}, testCase.name={}", - testCaseResult.integrationTest().meta().name(),testCaseResult.testCase().name()); - } - } - - for (TestCommand cleanupCommand : testSuite.cleanup()) { - - var cleanupRequest = new TestRequest(cleanupCommand, testPlan.target(), testEnvironment, TestAssertion.forSuccess(cleanupCommand.commandName())); - - var cleanupResponse = cleanupRequest.execute(); - var testCaseResult = cleanupResponse.validate(testSuite, null); - - if (testCaseResult.failed()){ - LOGGER.warn("TestCleanup FAILED: test.name={}, testCase.name={}, failedAssertion={}, error={}", - testCaseResult.integrationTest().meta().name(), "NULL", testCaseResult.failedAssertion().toString(), testCaseResult.error().toString()); - } - else{ - LOGGER.info("TestCleanup PASSED: test.name={}, testCase.name={}", - testCaseResult.integrationTest().meta().name(), "NULL"); - } - } - } - -} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/RunnerBase.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/RunnerBase.java index 017c747297..ceb8576e45 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/RunnerBase.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/RunnerBase.java @@ -1,9 +1,15 @@ package io.stargate.sgv2.jsonapi.api.v1.vectorize; -import static io.restassured.RestAssured.given; +import org.junit.jupiter.api.DynamicNode; +import org.junit.jupiter.api.DynamicTest; +import org.junit.jupiter.api.function.Executable; -public abstract class RunnerBase { +import java.util.Collection; +import java.util.List; + +import static org.junit.jupiter.api.DynamicTest.dynamicTest; + +public abstract class RunnerBase implements Executable { - protected abstract TestEnvironment integrationEnv(); } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/Target.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/Target.java index aaad219945..805732df42 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/Target.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/Target.java @@ -1,6 +1,9 @@ package io.stargate.sgv2.jsonapi.api.v1.vectorize; +import org.junit.jupiter.api.DynamicNode; + import java.util.HashMap; +import java.util.Optional; public class Target { @@ -35,16 +38,24 @@ public APIRequest apiRequest(TestCommand testCommand, TestEnvironment env){ return new APIRequest(targetConfiguration.connection(), env, testCommand.withEnvironment(env)); } - public void workflowStarting(TestPlan testPlan, Workflow workflow){ } - public void workflowFinished(TestPlan testPlan,Workflow workflow){ } + public Optional beforeWorkflow(TestPlan testPlan, Workflow workflow){ + return backend.beforeWorkflow(testPlan, workflow); + } + public Optional afterWorkflow(TestPlan testPlan,Workflow workflow){ + return backend.afterWorkflow(testPlan, workflow); + } - public void jobStarting(TestPlan testPlan,Job job){ - backend.jobStarting(testPlan, job); + public Optional beforeJob(TestPlan testPlan,Job job){ + return backend.beforeJob(testPlan, job); } - public void jobFinished(TestPlan testPlan,Job job){ - backend.jobFinished(testPlan, job); + public Optional afterJob(TestPlan testPlan,Job job){ + return backend.afterJob(testPlan, job); } - public void testRunStarting(TestPlan testPlan,TestSuite test, TestEnvironment env){ } - public void testRunFinished(TestPlan testPlan,TestSuite test, TestEnvironment env){ } + public Optional beforeTestSuite(TestPlan testPlan,TestSuite test, TestEnvironment env){ + return backend.beforeTestSuite(testPlan, test, env); + } + public Optional afterTestSuite(TestPlan testPlan,TestSuite test, TestEnvironment env){ + return backend.afterTestSuite(testPlan, test, env); + } } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestCase.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestCase.java index f94dd93752..2508eb985e 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestCase.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestCase.java @@ -2,6 +2,9 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.databind.node.ObjectNode; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions.AssertionMatcher; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions.TestAssertion; +import org.junit.jupiter.api.DynamicContainer; public record TestCase( @@ -9,4 +12,16 @@ public record TestCase( TestCommand command, ObjectNode asserts, @JsonProperty("$include") - String include) {} + String include) { + + + DynamicContainer testNodesForEnvironment(TestPlan testPlan, TestEnvironment testEnvironment) { + + var testRequest = new TestRequest( + "TestCase: name=%s".formatted(name, command.commandName()), + command(), testPlan.target(), testEnvironment, TestAssertion.buildAssertions(this)); + + return testRequest.testNodes(); + } + +} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestCaseResult.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestCaseResult.java index 62b8e5d4f3..297adb22ca 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestCaseResult.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestCaseResult.java @@ -1,13 +1,13 @@ package io.stargate.sgv2.jsonapi.api.v1.vectorize; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions.TestAssertion; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions.AssertionMatcher; public record TestCaseResult( TestSuite integrationTest, TestCase testCase, // Nullable TestResponse testResponse, AssertionError error, - TestAssertion failedAssertion + AssertionMatcher failedAssertion ) { public boolean failed(){ diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestEnvironment.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestEnvironment.java index 920f3bad0e..85de1077af 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestEnvironment.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestEnvironment.java @@ -3,12 +3,10 @@ import org.apache.commons.text.StringSubstitutor; import org.apache.commons.text.lookup.StringLookupFactory; import org.junit.jupiter.api.DynamicContainer; -import org.junit.jupiter.api.DynamicTest; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.HashMap; -import java.util.List; import java.util.Map; import java.util.Set; import java.util.regex.Pattern; @@ -35,17 +33,34 @@ public TestEnvironment(Map vars) { public DynamicContainer testNode(TestPlan testPlan, TestSuite testSuite) { - var desc = "TestEnv: %s ".formatted(this); + var desc = "TestEnv: %s ".formatted(description()); - var envNodes = Stream.of( - dynamicTest("Running TestSuite", () -> new IntegrationTestRunner( testPlan, testSuite, this).run()) - ); + var envNodes = testSuite.testNodesForEnvironment(testPlan, this).stream(); return DynamicContainer.dynamicContainer( desc, testPlan.addLifecycle(testSuite, this, envNodes)); } + private String description(){ + return vars.entrySet().stream() + .filter( + entry -> { + var name = entry.getKey().toUpperCase(); + if (SCHEMA_IDENTIFIER.contains(name)){ + return false; // schema names not usually interesting + } + if (name.contains("KEY") || name.contains("API") || name.contains("TOKEN")){ + return false; // assume security + } + return true; + } + ) + .sorted(Map.Entry.comparingByKey()) + .toList() + .toString(); + } + private TestEnvironment(TestEnvironment other){ this.vars.putAll(other.vars); } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestPlan.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestPlan.java index 550aa2b5a0..11d60aefa7 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestPlan.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestPlan.java @@ -4,7 +4,9 @@ import org.junit.jupiter.api.DynamicNode; import java.util.List; +import java.util.Optional; import java.util.Set; +import java.util.function.Supplier; import java.util.stream.Stream; import static org.junit.jupiter.api.DynamicContainer.dynamicContainer; @@ -52,36 +54,46 @@ public void updateJobForTarget(Job job){ target.updateJobForTarget(job); } - public Stream addLifecycle(Workflow workflow, Stream dynamicNodes){ + private static Optional containerIfPresent(String namePrefix, TestSpecMeta meta, Optional dynamicNode){ + return dynamicNode + .map( + node -> dynamicContainer(namePrefix + ": " + meta.name(), Stream.of(node)) + ); + } - var starting = dynamicTest("Workflow Starting: %s".formatted(workflow.meta().name()), () -> target.workflowStarting(this,workflow)); - var finished = dynamicTest("Workflow Finished: %s".formatted(workflow.meta().name()), () -> target.workflowFinished(this,workflow)); + private static Stream streamIfPresent(Optional container){ + return container.stream().flatMap(Stream::of); + } + + private static Stream lifecycleNodes(String namePrefix, TestSpecMeta meta, Supplier> nodeSupplier){ + var targetDynamicNode = nodeSupplier.get(); + return streamIfPresent(containerIfPresent(namePrefix, meta, targetDynamicNode)); + } + + public Stream addLifecycle(Workflow workflow, Stream dynamicNodes){ return Stream.concat( - Stream.of(starting), - Stream.concat(dynamicNodes, Stream.of(finished)) + lifecycleNodes("Before Workflow", workflow.meta(), () -> target.beforeWorkflow(this, workflow)), + Stream.concat(dynamicNodes, + lifecycleNodes("After Workflow", workflow.meta(), () -> target.afterWorkflow(this, workflow))) ); } public Stream addLifecycle(Job job, Stream dynamicNodes){ - var starting = dynamicTest("Job Starting: %s".formatted(job.meta().name()), () -> target.jobStarting(this,job)); - var finished = dynamicTest("Job Finished: %s".formatted(job.meta().name()), () -> target.jobFinished(this,job)); - return Stream.concat( - Stream.of(starting), - Stream.concat(dynamicNodes, Stream.of(finished)) + lifecycleNodes("Before Job", job.meta(), () -> target.beforeJob(this, job)), + Stream.concat(dynamicNodes, + lifecycleNodes("After Job", job.meta(), () -> target.afterJob(this, job))) ); } public Stream addLifecycle(TestSuite testSuite, TestEnvironment environment, Stream dynamicNodes){ - var starting = dynamicTest("TestRun Starting: %s".formatted(testSuite.meta().name()), () -> target.testRunStarting(this,testSuite, environment)); - var finished = dynamicTest("TestRun Finished: %s".formatted(testSuite.meta().name()), () -> target.testRunFinished(this,testSuite, environment)); - return Stream.concat( - Stream.of(starting), - Stream.concat(dynamicNodes, Stream.of(finished)) + lifecycleNodes("Before TestSuite", testSuite.meta(), () -> target.beforeTestSuite(this, testSuite, environment)), + Stream.concat(dynamicNodes, + lifecycleNodes("After TestSuite", testSuite.meta(), () -> target.afterTestSuite(this, testSuite, environment))) ); } } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestRequest.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestRequest.java index a762ccf082..6c95b2f5fa 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestRequest.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestRequest.java @@ -1,17 +1,55 @@ package io.stargate.sgv2.jsonapi.api.v1.vectorize; import io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions.TestAssertion; +import org.junit.jupiter.api.DynamicContainer; +import org.junit.jupiter.api.DynamicNode; +import java.util.ArrayList; import java.util.List; +import java.util.concurrent.atomic.AtomicReference; -public record TestRequest(TestCommand testCommand, - Target integrationTarget, - TestEnvironment integrationEnv, - List testAssertions) { +import static org.junit.jupiter.api.DynamicContainer.dynamicContainer; +import static org.junit.jupiter.api.DynamicTest.dynamicTest; - public TestResponse execute(){ +public record TestRequest( + String name, + TestCommand testCommand, + Target target, + TestEnvironment testEnvironment, + List testAssertions) { - var apiRequest = integrationTarget.apiRequest(testCommand, integrationEnv); + public TestResponse execute() { + + var apiRequest = target.apiRequest(testCommand, testEnvironment); return new TestResponse(this, apiRequest, apiRequest.execute()); } + + public DynamicContainer testNodes() { + + List nodes = new ArrayList<>(); + AtomicReference atomicResponse = new AtomicReference<>(); + + // Execute the request, and set so the assertions can pull the response after. + nodes.add(dynamicTest("Command: " + testCommand.commandName().getApiName(), () -> atomicResponse.set(execute()))); + + // tests for each assertion + var assertionTests = testAssertions().stream().map( + testAssertion -> + dynamicTest(testAssertion.name(), () -> { + var resp = atomicResponse.get(); + if (resp == null) { + throw new IllegalStateException("Response is null"); + } + testAssertion.run(resp); + } + )) + .toList(); + + // if we have assertion tests, put them in a container + if (!assertionTests.isEmpty()) { + nodes.add(dynamicContainer("Assertions", assertionTests)); + } + + return dynamicContainer("Request: " + name, nodes); + } } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestRequestRunner.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestRequestRunner.java new file mode 100644 index 0000000000..06388569cc --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestRequestRunner.java @@ -0,0 +1,29 @@ +package io.stargate.sgv2.jsonapi.api.v1.vectorize; + +import org.junit.jupiter.api.DynamicTest; + +import java.util.Collection; +import java.util.Objects; + +import static org.junit.jupiter.api.DynamicTest.dynamicTest; + +public class TestRequestRunner extends RunnerBase{ + + private final TestRequest testRequest; + private final TestSuite testSuite; + private final TestCase testCase; + + public TestRequestRunner(TestRequest testRequest, TestSuite testSuite, TestCase testCase ) { + this.testRequest = Objects.requireNonNull(testRequest, "testRequest must not be null"); + + this.testSuite = testSuite; + this.testCase = testCase; + } + + @Override + public void execute() throws Throwable { + +// testRequest.execute().validate(testSuite, testCase); + } + +} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestResponse.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestResponse.java index 6bffe85324..a80e796e89 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestResponse.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestResponse.java @@ -1,6 +1,6 @@ package io.stargate.sgv2.jsonapi.api.v1.vectorize; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions.TestAssertion; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions.AssertionMatcher; public record TestResponse( TestRequest testRequest, @@ -8,28 +8,28 @@ public record TestResponse( APIResponse apiResponse ) { - public TestCaseResult validate(TestSuite integrationTest, TestCase testCase){ - return validate(integrationTest, testCase, true); - } - - public TestCaseResult validate(TestSuite integrationTest, TestCase testCase, boolean throwOnError) { - - AssertionError assertionError = null; - TestAssertion failedAssertion = null; - for (var testAssertion : testRequest.testAssertions()){ - try{ - testAssertion.run(apiResponse); - } - catch(AssertionError e){ - if (throwOnError){ - throw e; - } - - failedAssertion = testAssertion; - assertionError = e; - break; - } - } - return new TestCaseResult(integrationTest, testCase, this , assertionError, failedAssertion); - } +// public TestCaseResult validate(TestSuite integrationTest, TestCase testCase){ +// return validate(integrationTest, testCase, true); +// } +// +// public TestCaseResult validate(TestSuite testSuite, TestCase testCase, boolean throwOnError) { +// +// AssertionError assertionError = null; +// AssertionMatcher failedAssertion = null; +// for (var testAssertion : testRequest.testAssertions()){ +// try{ +// testAssertion.run(apiResponse); +// } +// catch(AssertionError e){ +// if (throwOnError){ +// throw e; +// } +// +// failedAssertion = testAssertion; +// assertionError = e; +// break; +// } +// } +// return new TestCaseResult(testSuite, testCase, this , assertionError, failedAssertion); +// } } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestSuite.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestSuite.java index cd1a2051f7..e7c159aced 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestSuite.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestSuite.java @@ -1,10 +1,13 @@ package io.stargate.sgv2.jsonapi.api.v1.vectorize; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions.AssertionMatcher; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions.TestAssertion; import org.junit.jupiter.api.DynamicContainer; +import org.junit.jupiter.api.DynamicNode; import java.util.ArrayList; +import java.util.Collection; import java.util.List; -import java.util.stream.Stream; import static org.junit.jupiter.api.DynamicContainer.dynamicContainer; @@ -28,6 +31,33 @@ public DynamicContainer testNode(TestPlan testPlan, List allEnv ); } + Collection testNodesForEnvironment(TestPlan testPlan, TestEnvironment testEnvironment) { + + List nodes = new ArrayList<>(); + + int i = 1; + for (TestCommand setupCommand : setup()) { + var setupRequest = new TestRequest( + "SetupRequest[%s]: %s".formatted(i++, setupCommand.commandName()), + setupCommand, testPlan.target(), testEnvironment, TestAssertion.forSuccess(setupCommand.commandName())); + + nodes.add(setupRequest.testNodes()); + } + + for (var testCase : tests()) { + nodes.add(testCase.testNodesForEnvironment(testPlan, testEnvironment)); + } + + for (TestCommand cleanupCommand : cleanup()) { + var cleanupRequest = new TestRequest( + "CleanupRequest[%s]: %s".formatted(i++, cleanupCommand.commandName()), + cleanupCommand, testPlan.target(), testEnvironment, TestAssertion.forSuccess(cleanupCommand.commandName())); + nodes.add(cleanupRequest.testNodes()); + } + + return nodes; + + } public void expand(SpecFiles itCollection) { List expandedSetup = new ArrayList<>(); diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestSuiteRunner.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestSuiteRunner.java new file mode 100644 index 0000000000..3aaabe045a --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestSuiteRunner.java @@ -0,0 +1,50 @@ +package io.stargate.sgv2.jsonapi.api.v1.vectorize; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static io.restassured.RestAssured.given; + +public class TestSuiteRunner extends RunnerBase { + private static final Logger LOGGER = LoggerFactory.getLogger(TestSuiteRunner.class); + + + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + + private final TestPlan testPlan; + private final TestSuite testSuite; + private final TestEnvironment testEnvironment; + + public TestSuiteRunner( + TestPlan testPlan, TestSuite testSuite, TestEnvironment testEnvironment) { + this.testPlan = testPlan; + this.testSuite = testSuite; + this.testEnvironment = testEnvironment; + } + + + + @Override + public void execute() throws Throwable { + + + +// for (TestCase testCase : testSuite.tests()) { +// var testRequest = new TestRequest(testCase.command(), testPlan.target(), testEnvironment, TestAssertion.buildAssertions(testCase)); +// var testResponse = testRequest.execute(); +// var testCaseResult = testResponse.validate(testSuite, testCase); +// +// if (testCaseResult.failed()){ +// LOGGER.warn("TestCase FAILED: test.name={}, testCase.name={}, failedAssertion={}, error={}", +// testCaseResult.integrationTest().meta().name(),testCaseResult.testCase().name(), testCaseResult.failedAssertion(), String.valueOf(testCaseResult.error())); +// } +// else{ +// LOGGER.info("TestCase PASSED: test.name={}, testCase.name={}", +// testCaseResult.integrationTest().meta().name(),testCaseResult.testCase().name()); +// } +// } + + } + +} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/AssertionFactory.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/AssertionFactory.java index 3c9f6198da..0694a73528 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/AssertionFactory.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/AssertionFactory.java @@ -5,5 +5,5 @@ @FunctionalInterface public interface AssertionFactory { - TestAssertion create(JsonNode args); + AssertionMatcher create(JsonNode args); } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/AssertionMatcher.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/AssertionMatcher.java new file mode 100644 index 0000000000..4703c32c25 --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/AssertionMatcher.java @@ -0,0 +1,8 @@ +package io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions; + +import io.stargate.sgv2.jsonapi.api.v1.vectorize.APIResponse; + +public interface AssertionMatcher { + + void match(APIResponse apiResponse); +} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/BodyAssertion.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/BodyAssertion.java index ca995742db..763c1ec498 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/BodyAssertion.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/BodyAssertion.java @@ -1,29 +1,14 @@ package io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions; -import com.fasterxml.jackson.databind.JsonNode; -import io.stargate.sgv2.jsonapi.api.model.command.CommandName; import io.stargate.sgv2.jsonapi.api.v1.vectorize.APIResponse; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.TestCase; import org.hamcrest.Matcher; -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; -import java.lang.reflect.Modifier; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Map; - -import static io.stargate.sgv2.jsonapi.api.v1.ResponseAssertions.responseIsDDLSuccess; -import static io.stargate.sgv2.jsonapi.api.v1.ResponseAssertions.responseIsWriteSuccess; -import static org.hamcrest.Matchers.*; - public record BodyAssertion( String bodyPath, Matcher matcher -) implements TestAssertion{ +) implements AssertionMatcher { - public void run(APIResponse apiResponse) { + public void match(APIResponse apiResponse) { apiResponse.validatableResponse().body(bodyPath(), matcher()); } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Documents.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Documents.java index 57b346eddc..0dafee15c2 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Documents.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Documents.java @@ -7,12 +7,12 @@ public class Documents { - public static TestAssertion count(JsonNode args) { + public static AssertionMatcher count(JsonNode args) { var expectedCount = args.asInt(); return new BodyAssertion("data.documents", hasSize(expectedCount)); } - public static TestAssertion isExactly(JsonNode args) { + public static AssertionMatcher isExactly(JsonNode args) { return new BodyAssertion("data.document", jsonEquals(args)); } } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Response.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Response.java index 1aabd520ca..b1e2e31583 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Response.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Response.java @@ -7,20 +7,20 @@ public class Response { - public static TestAssertion isFindSuccess(JsonNode args) { + public static AssertionMatcher isFindSuccess(JsonNode args) { return new BodyAssertion("$", responseIsFindSuccess()); } - public static TestAssertion isFindAndSuccess(JsonNode args) { + public static AssertionMatcher isFindAndSuccess(JsonNode args) { return new BodyAssertion("$", responseIsFindAndSuccess()); } - public static TestAssertion isWriteSuccess(JsonNode args) { + public static AssertionMatcher isWriteSuccess(JsonNode args) { return new BodyAssertion("$", responseIsWriteSuccess()); } - public static TestAssertion isDDLSuccess(JsonNode args) { + public static AssertionMatcher isDDLSuccess(JsonNode args) { return new BodyAssertion("$", responseIsDDLSuccess()); } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Status.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Status.java index d724ddfdd5..f1b4fddad2 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Status.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Status.java @@ -5,7 +5,7 @@ import static net.javacrumbs.jsonunit.JsonMatchers.jsonEquals; public class Status { - public static TestAssertion isExactly(JsonNode args) { + public static AssertionMatcher isExactly(JsonNode args) { return new BodyAssertion("status", jsonEquals(args)); } } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/StatusCode.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Statuscode.java similarity index 65% rename from src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/StatusCode.java rename to src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Statuscode.java index 56d0bd8b07..fac78c63b9 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/StatusCode.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Statuscode.java @@ -1,12 +1,11 @@ package io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.APIResponse; import org.apache.http.HttpStatus; import org.testcontainers.shaded.com.fasterxml.jackson.databind.JsonNode; -public class StatusCode { +public class Statuscode { - public static TestAssertion success(JsonNode args){ + public static AssertionMatcher success(JsonNode args){ return apiResponse -> apiResponse.validatableResponse().statusCode(HttpStatus.SC_OK); } } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/TestAssertion.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/TestAssertion.java index 7bf168ccf3..1182d6b985 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/TestAssertion.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/TestAssertion.java @@ -2,60 +2,83 @@ import com.fasterxml.jackson.databind.JsonNode; import io.stargate.sgv2.jsonapi.api.model.command.CommandName; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.APIResponse; import io.stargate.sgv2.jsonapi.api.v1.vectorize.TestCase; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.TestResponse; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.lang.reflect.Modifier; -import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.Map; +import java.util.stream.Stream; -public interface TestAssertion { +public record TestAssertion( + String name, + JsonNode args, + AssertionMatcher matcher +) { - void run(APIResponse apiResponse); + public void run(TestResponse testResponse) { - static List forSuccess(CommandName commandName) { + try{ + matcher.match(testResponse.apiResponse()); + } + catch (AssertionError e) { + System.out.printf("Failed Assertion: name=%s, args=%s", name, args); + throw e; + } + catch (Exception e) { + System.out.printf("Error In Assertion: name=%s, args=%s", name, args); + throw e; + } + } + public static List forSuccess(CommandName commandName) { - List assertions = new ArrayList<>(); - assertions.add(StatusCode.success(null)); + var builder = Stream.builder() + .add(new AssertionDefinition("Statuscode.success", null)); switch (commandName) { case INSERT_ONE, INSERT_MANY -> { - assertions.add(Response.isWriteSuccess(null)); + builder.add(new AssertionDefinition("Response.isWriteSuccess", null)); } case CREATE_KEYSPACE, DROP_KEYSPACE, CREATE_NAMESPACE, DROP_NAMESPACE, DELETE_COLLECTION, CREATE_COLLECTION -> { - assertions.add(Response.isDDLSuccess(null)); + builder.add(new AssertionDefinition("Response.isDDLSuccess", null)); } } - return assertions; + return buildAssertions(builder.build()); } - public static List buildAssertions (TestCase testCase) { + public static List buildAssertions(TestCase testCase) { - List testAssertions = new ArrayList<>(); - for (Map.Entry entry : testCase.asserts().properties()){ - var args = entry.getValue(); + return buildAssertions(testCase.asserts().properties().stream() + .map(AssertionDefinition::create)); + } + + private static List buildAssertions(Stream defs) { + + return defs.map( + def -> { + var assertFactory = findAssertionFactory(def.name()); + AssertionMatcher matcher; + try { + matcher = (AssertionMatcher) assertFactory.invoke(null, def.args()); + } catch (IllegalAccessException | InvocationTargetException e) { + throw new RuntimeException(e); + } + return new TestAssertion(def.name(), def.args(), matcher); + } + ).toList(); - var assertFactory = findAssertionFactory(entry.getKey()); - try { - testAssertions.add((BodyAssertion)assertFactory.invoke(null, args)); - } catch (IllegalAccessException | InvocationTargetException e) { - throw new RuntimeException(e); - } - } - return testAssertions; } - private static Method findAssertionFactory(String key){ + private static Method findAssertionFactory(String key) { // "validatableResponse.isFindSuccess" int dot = key.indexOf('.'); String typeName = key.substring(0, dot); - String funcName = key.substring(dot + 1); + String funcName = key.substring(dot + 1); String qualifiedTypeName = "io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions." @@ -78,4 +101,12 @@ private static Method findAssertionFactory(String key){ } } + private record AssertionDefinition(String name, JsonNode args) { + + static AssertionDefinition create(Map.Entry def) { + return new AssertionDefinition(def.getKey(), def.getValue()); + } + + } + } diff --git a/src/test/resources/integration-tests/vectorize/vectorize-base.json b/src/test/resources/integration-tests/vectorize/vectorize-base.json index a27f4a9e33..69940a3b62 100644 --- a/src/test/resources/integration-tests/vectorize/vectorize-base.json +++ b/src/test/resources/integration-tests/vectorize/vectorize-base.json @@ -46,7 +46,7 @@ ], "tests": [ { - "name": "findByVectorize", + "name": "basic findMany", "command": { "find": { "sort": { @@ -56,7 +56,7 @@ }, "asserts": { "response.isFindSuccess": null, - "documents.count": 3 + "documents.count": 4 } }, { From 22d80e33f111ce5122e2782b9f240eed6e6928d0 Mon Sep 17 00:00:00 2001 From: Aaron Morton Date: Wed, 25 Feb 2026 15:19:00 +1300 Subject: [PATCH 08/89] WIP --- .../api/v1/vectorize/CassandraBackend.java | 4 +- .../jsonapi/api/v1/vectorize/TestRequest.java | 10 +- .../jsonapi/api/v1/vectorize/TestSuite.java | 4 +- .../v1/vectorize/VectorizeTestFactory.java | 27 ---- .../assertions/AssertionFactory.java | 9 -- .../assertions/AssertionMatcher.java | 145 ++++++++++++++++++ .../vectorize/assertions/BodyAssertion.java | 11 +- .../v1/vectorize/assertions/Documents.java | 16 +- .../api/v1/vectorize/assertions/Http.java | 16 ++ .../api/v1/vectorize/assertions/Response.java | 46 +++++- .../assertions/SingleTestAssertion.java | 42 +++++ .../api/v1/vectorize/assertions/Status.java | 9 +- .../v1/vectorize/assertions/Statuscode.java | 11 -- .../vectorize/assertions/TestAssertion.java | 111 +++++--------- .../assertions/TestAssertionContainer.java | 47 ++++++ .../vectorize/vectorize-base.json | 4 +- 16 files changed, 366 insertions(+), 146 deletions(-) delete mode 100644 src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/AssertionFactory.java create mode 100644 src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Http.java create mode 100644 src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/SingleTestAssertion.java delete mode 100644 src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Statuscode.java create mode 100644 src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/TestAssertionContainer.java diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/CassandraBackend.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/CassandraBackend.java index 274550ecfd..76a31a96e3 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/CassandraBackend.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/CassandraBackend.java @@ -40,7 +40,7 @@ public Optional beforeJob(TestPlan testPlan, Job job) { var env = job.withoutMatrix(testPlan); var setupRequest = new TestRequest( env.substitutor().replace("createKeyspace: ${KEYSPACE_NAME}"), - command, testPlan.target(), env, TestAssertion.forSuccess(command.commandName())); + command, testPlan.target(), env, TestAssertion.forSuccess(command)); return Optional.of(setupRequest.testNodes()); } @@ -59,7 +59,7 @@ public Optional afterJob(TestPlan testPlan, Job job) { var env = job.withoutMatrix(testPlan); var setupRequest = new TestRequest( env.substitutor().replace("dropKeyspace: ${KEYSPACE_NAME}"), - command, testPlan.target(), job.withoutMatrix(testPlan), TestAssertion.forSuccess(command.commandName())); + command, testPlan.target(), job.withoutMatrix(testPlan), TestAssertion.forSuccess(command)); return Optional.of(setupRequest.testNodes()); } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestRequest.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestRequest.java index 6c95b2f5fa..e888362095 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestRequest.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestRequest.java @@ -34,15 +34,7 @@ public DynamicContainer testNodes() { // tests for each assertion var assertionTests = testAssertions().stream().map( - testAssertion -> - dynamicTest(testAssertion.name(), () -> { - var resp = atomicResponse.get(); - if (resp == null) { - throw new IllegalStateException("Response is null"); - } - testAssertion.run(resp); - } - )) + testAssertion -> testAssertion.testNodes(atomicResponse)) .toList(); // if we have assertion tests, put them in a container diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestSuite.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestSuite.java index e7c159aced..b027f17031 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestSuite.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestSuite.java @@ -39,7 +39,7 @@ Collection testNodesForEnvironment(TestPlan testPlan, Te for (TestCommand setupCommand : setup()) { var setupRequest = new TestRequest( "SetupRequest[%s]: %s".formatted(i++, setupCommand.commandName()), - setupCommand, testPlan.target(), testEnvironment, TestAssertion.forSuccess(setupCommand.commandName())); + setupCommand, testPlan.target(), testEnvironment, TestAssertion.forSuccess(setupCommand)); nodes.add(setupRequest.testNodes()); } @@ -51,7 +51,7 @@ Collection testNodesForEnvironment(TestPlan testPlan, Te for (TestCommand cleanupCommand : cleanup()) { var cleanupRequest = new TestRequest( "CleanupRequest[%s]: %s".formatted(i++, cleanupCommand.commandName()), - cleanupCommand, testPlan.target(), testEnvironment, TestAssertion.forSuccess(cleanupCommand.commandName())); + cleanupCommand, testPlan.target(), testEnvironment, TestAssertion.forSuccess(cleanupCommand)); nodes.add(cleanupRequest.testNodes()); } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/VectorizeTestFactory.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/VectorizeTestFactory.java index daf2137038..19219119a1 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/VectorizeTestFactory.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/VectorizeTestFactory.java @@ -13,38 +13,11 @@ public class VectorizeTestFactory { private static final Logger LOGGER = LoggerFactory.getLogger(VectorizeTestFactory.class); - - - private static final int TEST_PARALLELISM = 8; // ← set this - @TestFactory Stream jobs() { var testPlan = TestPlan.create("local", List.of("all-vectorize-workflow")); return testPlan.testNode(); - - // Parallel -// int maxConcurrentJobs = 2; -// integrationTarget.workflowStarting(workflow); -// try { -// var executor = java.util.concurrent.Executors.newFixedThreadPool(maxConcurrentJobs); -// try { -// var futures = jobs.stream() -// .map(job -> java.util.concurrent.CompletableFuture.runAsync(() -> { -// LOGGER.info("Starting job {}", job.meta()); -// new IntegrationJobRunner(integrationTarget, itCollection, job).run(); -// }, executor)) -// .toList(); -// -// // wait for all, fail if any failed -// futures.forEach(java.util.concurrent.CompletableFuture::join); -// } finally { -// executor.shutdown(); -// } -// } finally { -// integrationTarget.workflowFinished(workflow); -// } - } } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/AssertionFactory.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/AssertionFactory.java deleted file mode 100644 index 0694a73528..0000000000 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/AssertionFactory.java +++ /dev/null @@ -1,9 +0,0 @@ -package io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions; - -import com.fasterxml.jackson.databind.JsonNode; - -@FunctionalInterface -public interface AssertionFactory { - - AssertionMatcher create(JsonNode args); -} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/AssertionMatcher.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/AssertionMatcher.java index 4703c32c25..ed8c7cd4dd 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/AssertionMatcher.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/AssertionMatcher.java @@ -1,8 +1,153 @@ package io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions; +import com.fasterxml.jackson.databind.JsonNode; import io.stargate.sgv2.jsonapi.api.v1.vectorize.APIResponse; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.TestCommand; +import org.assertj.core.api.AssertFactory; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.lang.reflect.Modifier; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.function.BiFunction; + +/** + * Contract for running an assertion on the response from the API. + */ +@FunctionalInterface public interface AssertionMatcher { + /** + * Match the response to the assertion. + * + * @param apiResponse response from the API + * @throws AssertionError if the match fails + */ void match(APIResponse apiResponse); + + AssertionFactoryRegistry FACTORY_REGISTRY = new AssertionFactoryRegistry(); + + /** + * Assertions int the test case are used to find Assertion factories in the code, + * they are called with the TestCommand they need to run against so you + * know what the command is, and any args from the JSON for the test case. + *

+ * Use the {@link SingleFactory} signature if your factory returns a single matcher, which is + * normal. Use the {@link MultiFactory} signature if you want to return more than one. + *

+ */ + + interface AssertionFactory extends BiFunction {} + + interface AssertionMatcherFactory extends AssertionFactory { + } + + interface TestAssertionContainerFactory extends AssertionFactory> {} + + class AssertionFactoryRegistry { + + private final Map> factoryMethods = new ConcurrentHashMap<>(); + + public void register(Class cls) { + for (var method : cls.getMethods()) { + if (isValidFactoryMethod(method)) { + var factoryKey = cls.getSimpleName().toLowerCase() + + "." + + method.getName().toLowerCase(); + + // NOTE: not checking the generic of the list + AssertionFactory wrapped = (method.getReturnType() == AssertionMatcher.class) ? + new AssertionMatcherFactoryWrapper(method) + : + new TestAssertionContainerFactoryWrapper(method); + factoryMethods.put(factoryKey, wrapped); + } + } + } + + public AssertionFactory get(String name){ + var normalName = name.toLowerCase(); + var factoryMethod = factoryMethods.get(normalName); + if (factoryMethod == null) { + loadClassFor(name); + } + + factoryMethod = factoryMethods.get(normalName); + if (factoryMethod == null) { + throw new IllegalArgumentException("Unknown assertion factory. (normalised)name: %s known=%s".formatted(name,factoryMethods.keySet())); + } + return factoryMethod; + } + + private static final String PACKAGE = + "io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions"; + + private void loadClassFor(String key) { + + int dot = key.indexOf('.'); + if (dot < 0) { + return; + } + + var typeName = key.substring(0, dot); + + var className = + PACKAGE + "." + + Character.toUpperCase(typeName.charAt(0)) + + typeName.substring(1); + + try { + Class.forName(className); + + // class static initializer should call register() + + } catch (ClassNotFoundException ignored) { + // expected if class does not exist + } + } + + private record AssertionMatcherFactoryWrapper(Method method) implements AssertionMatcherFactory { + + @SuppressWarnings("unchecked") + @Override + public AssertionMatcher apply(TestCommand testCommand, JsonNode args) { + try { + return (AssertionMatcher) method.invoke(null, testCommand, args); + } catch (IllegalAccessException | InvocationTargetException e) { + throw new RuntimeException(e); + } + } + } + + private record TestAssertionContainerFactoryWrapper(Method method) implements TestAssertionContainerFactory { + + @SuppressWarnings("unchecked") + @Override + public List apply(TestCommand testCommand, JsonNode args) { + try { + return (List) method.invoke(null, testCommand, args); + } catch (IllegalAccessException | InvocationTargetException e) { + throw new RuntimeException(e); + } + } + } + + private static boolean isValidFactoryMethod(Method method) { + + if (!Modifier.isStatic(method.getModifiers())) { + return false; + } + + if ((!List.class.isAssignableFrom(method.getReturnType())) && method.getReturnType() != AssertionMatcher.class) { + return false; + } + var p = method.getParameterTypes(); + return p.length == 2 + && p[0] == TestCommand.class + && p[1] == JsonNode.class; + } + + } } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/BodyAssertion.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/BodyAssertion.java index 763c1ec498..0f048667f9 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/BodyAssertion.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/BodyAssertion.java @@ -3,13 +3,22 @@ import io.stargate.sgv2.jsonapi.api.v1.vectorize.APIResponse; import org.hamcrest.Matcher; +/** + * Assertions that check the body of the response using a {@link Matcher} form hamcrest. + * + *

+ * Example: + * 

+ *   return new BodyAssertion("data.documents", hasSize(expectedCount));
+ *
+ *

+ */ public record BodyAssertion( String bodyPath, Matcher matcher ) implements AssertionMatcher { public void match(APIResponse apiResponse) { - apiResponse.validatableResponse().body(bodyPath(), matcher()); } } \ No newline at end of file diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Documents.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Documents.java index 0dafee15c2..cd4dbc5078 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Documents.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Documents.java @@ -1,18 +1,30 @@ package io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions; import com.fasterxml.jackson.databind.JsonNode; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.TestCommand; import static net.javacrumbs.jsonunit.JsonMatchers.jsonEquals; import static org.hamcrest.Matchers.hasSize; +/** + * Assertions that check the `document` or `documents` in the `data` field + * of the API result. + *

+ * See {@link TestAssertion} + *

+ */ public class Documents { - public static AssertionMatcher count(JsonNode args) { + static { + AssertionMatcher.FACTORY_REGISTRY.register(Documents.class); + } + + public static AssertionMatcher count(TestCommand testCommand, JsonNode args) { var expectedCount = args.asInt(); return new BodyAssertion("data.documents", hasSize(expectedCount)); } - public static AssertionMatcher isExactly(JsonNode args) { + public static AssertionMatcher isExactly(TestCommand testCommand, JsonNode args) { return new BodyAssertion("data.document", jsonEquals(args)); } } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Http.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Http.java new file mode 100644 index 0000000000..f9424f0aee --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Http.java @@ -0,0 +1,16 @@ +package io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions; + +import com.fasterxml.jackson.databind.JsonNode; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.TestCommand; +import org.apache.http.HttpStatus; + +public class Http { + + static { + AssertionMatcher.FACTORY_REGISTRY.register(Http.class); + } + + public static AssertionMatcher success(TestCommand testCommand, JsonNode args){ + return apiResponse -> apiResponse.validatableResponse().statusCode(HttpStatus.SC_OK); + } +} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Response.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Response.java index b1e2e31583..afb54bce38 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Response.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Response.java @@ -1,26 +1,64 @@ package io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions; import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.NullNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.TestCommand; + +import java.util.List; import static io.stargate.sgv2.jsonapi.api.v1.ResponseAssertions.*; import static org.hamcrest.Matchers.hasSize; + +/** + * Assertions that check the structure of the API response, e.g. should it have a `data` field + *

+ * See {@link TestAssertion} + *

+ */ public class Response { - public static AssertionMatcher isFindSuccess(JsonNode args) { + static { + AssertionMatcher.FACTORY_REGISTRY.register(Response.class); + } + + /** + * Checks the hTTP status AND the shape of the response doc + */ + public static List isSuccess(TestCommand testCommand, JsonNode args) { + + var commandName = testCommand.commandName(); + var responseDocMatcher = switch (commandName.getCommandType()){ + case DDL -> new TestAssertion.AssertionDefinition( "Response.isDDLSuccess", null); + case DML -> + switch (commandName) { + case FIND_ONE, FIND -> new TestAssertion.AssertionDefinition( "Response.isFindSuccess", null); + case FIND_ONE_AND_DELETE, FIND_ONE_AND_REPLACE, FIND_ONE_AND_UPDATE -> new TestAssertion.AssertionDefinition( "Response.isFindAndSuccess", null); + case INSERT_ONE, INSERT_MANY -> new TestAssertion.AssertionDefinition( "Response.isWriteSuccess", null); + default -> throw new IllegalStateException("No isSuccess mapping for command name: " + commandName); + }; + case ADMIN -> throw new IllegalStateException("No isSuccess mapping for command name: " + commandName); + }; + + + return TestAssertion.buildAssertions(testCommand, List.of(new TestAssertion.AssertionDefinition("Http.success", null), responseDocMatcher)); + } + + public static AssertionMatcher isFindSuccess(TestCommand testCommand, JsonNode args) { return new BodyAssertion("$", responseIsFindSuccess()); } - public static AssertionMatcher isFindAndSuccess(JsonNode args) { + public static AssertionMatcher isFindAndSuccess(TestCommand testCommand, JsonNode args) { return new BodyAssertion("$", responseIsFindAndSuccess()); } - public static AssertionMatcher isWriteSuccess(JsonNode args) { + public static AssertionMatcher isWriteSuccess(TestCommand testCommand, JsonNode args) { return new BodyAssertion("$", responseIsWriteSuccess()); } - public static AssertionMatcher isDDLSuccess(JsonNode args) { + public static AssertionMatcher isDDLSuccess(TestCommand testCommand, JsonNode args) { return new BodyAssertion("$", responseIsDDLSuccess()); } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/SingleTestAssertion.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/SingleTestAssertion.java new file mode 100644 index 0000000000..40750ef352 --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/SingleTestAssertion.java @@ -0,0 +1,42 @@ +package io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions; + +import com.fasterxml.jackson.databind.JsonNode; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.TestResponse; +import org.junit.jupiter.api.DynamicNode; + +import java.util.concurrent.atomic.AtomicReference; + +import static org.junit.jupiter.api.DynamicTest.dynamicTest; + +public record SingleTestAssertion( + String name, + JsonNode args, + AssertionMatcher matcher +) implements TestAssertion { + + public void run(TestResponse testResponse) { + + try { + matcher.match(testResponse.apiResponse()); + } catch (AssertionError e) { + System.out.printf("Failed Assertion: name=%s, args=%s", name, args); + throw e; + } catch (Exception e) { + System.out.printf("Error In Assertion: name=%s, args=%s", name, args); + throw e; + } + } + + @Override + public DynamicNode testNodes(AtomicReference testResponse) { + + return dynamicTest(name(), () -> { + var resp = testResponse.get(); + if (resp == null) { + throw new IllegalStateException("Response is null"); + } + run(resp); + } + ); + } +} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Status.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Status.java index f1b4fddad2..adc9bd703f 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Status.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Status.java @@ -1,11 +1,18 @@ package io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions; import com.fasterxml.jackson.databind.JsonNode; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.TestCommand; import static net.javacrumbs.jsonunit.JsonMatchers.jsonEquals; public class Status { - public static AssertionMatcher isExactly(JsonNode args) { + + static { + AssertionMatcher.FACTORY_REGISTRY.register(Status.class); + } + + + public static AssertionMatcher isExactly(TestCommand testCommand, JsonNode args) { return new BodyAssertion("status", jsonEquals(args)); } } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Statuscode.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Statuscode.java deleted file mode 100644 index fac78c63b9..0000000000 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Statuscode.java +++ /dev/null @@ -1,11 +0,0 @@ -package io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions; - -import org.apache.http.HttpStatus; -import org.testcontainers.shaded.com.fasterxml.jackson.databind.JsonNode; - -public class Statuscode { - - public static AssertionMatcher success(JsonNode args){ - return apiResponse -> apiResponse.validatableResponse().statusCode(HttpStatus.SC_OK); - } -} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/TestAssertion.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/TestAssertion.java index 1182d6b985..812ee10f02 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/TestAssertion.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/TestAssertion.java @@ -1,107 +1,66 @@ package io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions; import com.fasterxml.jackson.databind.JsonNode; -import io.stargate.sgv2.jsonapi.api.model.command.CommandName; import io.stargate.sgv2.jsonapi.api.v1.vectorize.TestCase; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.TestCommand; import io.stargate.sgv2.jsonapi.api.v1.vectorize.TestResponse; +import org.junit.jupiter.api.DynamicNode; -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; -import java.lang.reflect.Modifier; -import java.util.Arrays; import java.util.List; import java.util.Map; +import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Stream; -public record TestAssertion( - String name, - JsonNode args, - AssertionMatcher matcher -) { +public interface TestAssertion { - public void run(TestResponse testResponse) { + String name(); - try{ - matcher.match(testResponse.apiResponse()); - } - catch (AssertionError e) { - System.out.printf("Failed Assertion: name=%s, args=%s", name, args); - throw e; - } - catch (Exception e) { - System.out.printf("Error In Assertion: name=%s, args=%s", name, args); - throw e; - } - } - public static List forSuccess(CommandName commandName) { - - var builder = Stream.builder() - .add(new AssertionDefinition("Statuscode.success", null)); - - switch (commandName) { - case INSERT_ONE, INSERT_MANY -> { - builder.add(new AssertionDefinition("Response.isWriteSuccess", null)); - } - case CREATE_KEYSPACE, DROP_KEYSPACE, CREATE_NAMESPACE, DROP_NAMESPACE, DELETE_COLLECTION, CREATE_COLLECTION -> { - builder.add(new AssertionDefinition("Response.isDDLSuccess", null)); - } - } - return buildAssertions(builder.build()); - } + JsonNode args(); + void run(TestResponse testResponse); - public static List buildAssertions(TestCase testCase) { + DynamicNode testNodes(AtomicReference testResponse); - return buildAssertions(testCase.asserts().properties().stream() - .map(AssertionDefinition::create)); - } - private static List buildAssertions(Stream defs) { + static List forSuccess(TestCommand testCommand) { - return defs.map( - def -> { - var assertFactory = findAssertionFactory(def.name()); - AssertionMatcher matcher; - try { - matcher = (AssertionMatcher) assertFactory.invoke(null, def.args()); - } catch (IllegalAccessException | InvocationTargetException e) { - throw new RuntimeException(e); - } - return new TestAssertion(def.name(), def.args(), matcher); - } - ).toList(); + var builder = Stream.builder() + .add(new AssertionDefinition("Response.isSuccess", null)); + return buildAssertions(testCommand, builder.build()); } - private static Method findAssertionFactory(String key) { - // "validatableResponse.isFindSuccess" + static List buildAssertions(TestCase testCase) { - int dot = key.indexOf('.'); - String typeName = key.substring(0, dot); - String funcName = key.substring(dot + 1); + var defs = testCase.asserts().properties().stream() + .map(AssertionDefinition::create); + return buildAssertions(testCase.command(), defs); + } - String qualifiedTypeName = - "io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions." - + Character.toUpperCase(typeName.charAt(0)) - + typeName.substring(1).toLowerCase(); + static List buildAssertions(TestCommand testCommand, List defs) { - try { - Class cls = Class.forName(qualifiedTypeName); + return buildAssertions(testCommand, defs.stream()); + } - var factoryMethod = Arrays.stream(cls.getMethods()) - .filter(m -> m.getName().equalsIgnoreCase(funcName)) - .filter(m -> Modifier.isStatic(m.getModifiers())) - .findFirst() - .orElseThrow(); + static List buildAssertions(TestCommand testCommand, Stream defs) { + return defs.map( + def -> buildAssertion(testCommand, def) + ).toList(); + } - return factoryMethod; - } catch (ReflectiveOperationException e) { - throw new RuntimeException("Invalid assertion: " + key, e); - } + private static TestAssertion buildAssertion(TestCommand testCommand, AssertionDefinition def) { + + return switch (AssertionMatcher.FACTORY_REGISTRY.get(def.name())) { + case AssertionMatcher.AssertionMatcherFactory factory -> + new SingleTestAssertion(def.name(), def.args(), factory.apply(testCommand, def.args())); + case AssertionMatcher.TestAssertionContainerFactory factory -> + new TestAssertionContainer(def.name(), def.args(), factory.apply(testCommand, def.args)); + default -> throw new IllegalStateException("Unknown TestAssertionFactory: " + def.name()); + }; } - private record AssertionDefinition(String name, JsonNode args) { + record AssertionDefinition(String name, JsonNode args) { static AssertionDefinition create(Map.Entry def) { return new AssertionDefinition(def.getKey(), def.getValue()); diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/TestAssertionContainer.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/TestAssertionContainer.java new file mode 100644 index 0000000000..acf447b355 --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/TestAssertionContainer.java @@ -0,0 +1,47 @@ +package io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions; + +import com.fasterxml.jackson.databind.JsonNode; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.TestResponse; +import org.junit.jupiter.api.DynamicNode; + +import java.util.List; +import java.util.concurrent.atomic.AtomicReference; + +import static org.junit.jupiter.api.DynamicContainer.dynamicContainer; +import static org.junit.jupiter.api.DynamicTest.dynamicTest; + +public record TestAssertionContainer( + String name, + JsonNode args, + List assertions +) implements TestAssertion { + + @Override + public void run(TestResponse testResponse) { + for (TestAssertion assertion : assertions) { + try{ + assertion.run(testResponse); + } + catch (AssertionError e) { + System.out.printf("Failed Assertion Container: name=%s, args=%s\n\t Caused By: name=%s, args=%s", name, args, + assertion.name(), assertion.args()); + throw e; + } + catch (Exception e) { + System.out.printf("Error In Assertion Container: name=%s, args=%s\n\t Caused By: name=%s, args=%s", name, args, + assertion.name(), assertion.args()); + throw e; + } + } + } + + @Override + public DynamicNode testNodes(AtomicReference testResponse) { + + var childs = assertions.stream() + .map(assertion -> assertion.testNodes(testResponse)) + .toList(); + + return dynamicContainer(name, childs); + } +} diff --git a/src/test/resources/integration-tests/vectorize/vectorize-base.json b/src/test/resources/integration-tests/vectorize/vectorize-base.json index 69940a3b62..4793799423 100644 --- a/src/test/resources/integration-tests/vectorize/vectorize-base.json +++ b/src/test/resources/integration-tests/vectorize/vectorize-base.json @@ -55,8 +55,8 @@ } }, "asserts": { - "response.isFindSuccess": null, - "documents.count": 4 + "response.isSuccess": null, + "documents.count": 3 } }, { From 0af33b1ed336ecbcdef2c0ab76d7ea885c68837a Mon Sep 17 00:00:00 2001 From: Aaron Morton Date: Thu, 26 Feb 2026 13:40:45 +1300 Subject: [PATCH 09/89] WIP - assertion template --- .../jsonapi/api/v1/vectorize/TestPlan.java | 4 +- .../assertions/AssertionMatcher.java | 12 +++++ .../assertions/AssertionTemplates.java | 49 +++++++++++++++++++ .../api/v1/vectorize/assertions/Response.java | 15 +++--- .../assertions/TemplatedAssertions.java | 40 +++++++++++++++ .../vectorize/assertions/TestAssertion.java | 4 +- .../assertions/assertion-templates.json | 12 +++++ .../vectorize/vectorize-base.json | 4 +- 8 files changed, 126 insertions(+), 14 deletions(-) create mode 100644 src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/AssertionTemplates.java create mode 100644 src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/TemplatedAssertions.java create mode 100644 src/test/resources/integration-tests/assertions/assertion-templates.json diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestPlan.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestPlan.java index 11d60aefa7..09f40ac141 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestPlan.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestPlan.java @@ -1,8 +1,11 @@ package io.stargate.sgv2.jsonapi.api.v1.vectorize; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions.AssertionTemplates; import org.junit.jupiter.api.DynamicContainer; import org.junit.jupiter.api.DynamicNode; +import java.io.IOException; +import java.nio.file.Path; import java.util.List; import java.util.Optional; import java.util.Set; @@ -14,7 +17,6 @@ public record TestPlan(Target target, SpecFiles specFiles, Set workflows){ - public static TestPlan create(String targetName, List workflows){ var targetConfigs = TargetConfigurationss.loadAll("integration-tests/targets/targets.json"); var target = new Target(targetConfigs.configuration(targetName)); diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/AssertionMatcher.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/AssertionMatcher.java index ed8c7cd4dd..2405419e15 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/AssertionMatcher.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/AssertionMatcher.java @@ -69,6 +69,18 @@ public void register(Class cls) { public AssertionFactory get(String name){ var normalName = name.toLowerCase(); + + int pos = normalName.indexOf('.'); + if (pos < 0){ + throw new IllegalArgumentException("Name must have a dot: " + name); + } + var type = normalName.substring(0, pos); + var func = normalName.substring(pos + 1); + + if (type.equals( "template")) { + return TemplatedAssertions.getFactory(func); + } + var factoryMethod = factoryMethods.get(normalName); if (factoryMethod == null) { loadClassFor(name); diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/AssertionTemplates.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/AssertionTemplates.java new file mode 100644 index 0000000000..82476b8ce6 --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/AssertionTemplates.java @@ -0,0 +1,49 @@ +package io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.TargetConfigurationss; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.TestSpecMeta; + +import java.io.IOException; +import java.net.URISyntaxException; +import java.net.URL; +import java.nio.file.Path; +import java.nio.file.Paths; + +public record AssertionTemplates( + TestSpecMeta meta, + ObjectNode templates +) { + + private static final ObjectMapper MAPPER = new ObjectMapper(); + + public static AssertionTemplates load(){ + + final Path dir = resourceDir("integration-tests/assertions/assertion-templates.json"); + + try { + return MAPPER.readValue(dir.toFile(), AssertionTemplates.class); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + private static Path resourceDir(String path) { + String normalized = path.startsWith("/") ? path.substring(1) : path; + + ClassLoader cl = Thread.currentThread().getContextClassLoader(); + URL url = cl.getResource(normalized); + if (url == null) { + throw new IllegalArgumentException("Test resource folder not found: " + path); + } + + try { + // Works for file: URLs; if you run tests from a jar, switch to getResourceAsStream-based + // walking. + return Paths.get(url.toURI()); + } catch (URISyntaxException e) { + throw new IllegalArgumentException("Bad resource URI for: " + path + " -> " + url, e); + } + } +} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Response.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Response.java index afb54bce38..d198db2fc5 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Response.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Response.java @@ -26,23 +26,20 @@ public class Response { /** * Checks the hTTP status AND the shape of the response doc */ - public static List isSuccess(TestCommand testCommand, JsonNode args) { + public static AssertionMatcher isSuccess(TestCommand testCommand, JsonNode args) { var commandName = testCommand.commandName(); - var responseDocMatcher = switch (commandName.getCommandType()){ - case DDL -> new TestAssertion.AssertionDefinition( "Response.isDDLSuccess", null); + return switch (commandName.getCommandType()){ + case DDL -> isDDLSuccess(testCommand, args); case DML -> switch (commandName) { - case FIND_ONE, FIND -> new TestAssertion.AssertionDefinition( "Response.isFindSuccess", null); - case FIND_ONE_AND_DELETE, FIND_ONE_AND_REPLACE, FIND_ONE_AND_UPDATE -> new TestAssertion.AssertionDefinition( "Response.isFindAndSuccess", null); - case INSERT_ONE, INSERT_MANY -> new TestAssertion.AssertionDefinition( "Response.isWriteSuccess", null); + case FIND_ONE, FIND -> Response.isFindSuccess(testCommand, args); + case FIND_ONE_AND_DELETE, FIND_ONE_AND_REPLACE, FIND_ONE_AND_UPDATE -> Response.isFindAndSuccess(testCommand, args); + case INSERT_ONE, INSERT_MANY -> Response.isWriteSuccess(testCommand, args); default -> throw new IllegalStateException("No isSuccess mapping for command name: " + commandName); }; case ADMIN -> throw new IllegalStateException("No isSuccess mapping for command name: " + commandName); }; - - - return TestAssertion.buildAssertions(testCommand, List.of(new TestAssertion.AssertionDefinition("Http.success", null), responseDocMatcher)); } public static AssertionMatcher isFindSuccess(TestCommand testCommand, JsonNode args) { diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/TemplatedAssertions.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/TemplatedAssertions.java new file mode 100644 index 0000000000..9b624b1f70 --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/TemplatedAssertions.java @@ -0,0 +1,40 @@ +package io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.TestCommand; + +import java.util.List; + +public class TemplatedAssertions { + + private static final AssertionTemplates assertions = AssertionTemplates.load(); + + public static AssertionMatcher.TestAssertionContainerFactory getFactory(String templateName){ + + JsonNode template = null; + for (var entry : assertions.templates().properties()){ + if (entry.getKey().equalsIgnoreCase(templateName)) { + template = entry.getValue(); + break; + } + } + + if (template == null) { + throw new IllegalArgumentException("Assertion template not found: " + templateName); + } + if (! (template instanceof ObjectNode templateObject)){ + throw new IllegalArgumentException("Assertion template is not an object: " + templateName); + } + + return new AssertionMatcher.TestAssertionContainerFactory(){ + @Override + public List apply(TestCommand testCommand, JsonNode jsonNode) { + return templateObject.properties().stream() + .map(entry -> new TestAssertion.AssertionDefinition(entry.getKey(), entry.getValue())) + .map(def -> TestAssertion.buildAssertion(testCommand, def)) + .toList(); + } + }; + } +} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/TestAssertion.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/TestAssertion.java index 812ee10f02..b01736cef1 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/TestAssertion.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/TestAssertion.java @@ -25,7 +25,7 @@ public interface TestAssertion { static List forSuccess(TestCommand testCommand) { var builder = Stream.builder() - .add(new AssertionDefinition("Response.isSuccess", null)); + .add(new AssertionDefinition("template.isSuccess", null)); return buildAssertions(testCommand, builder.build()); } @@ -49,7 +49,7 @@ static List buildAssertions(TestCommand testCommand, Stream diff --git a/src/test/resources/integration-tests/assertions/assertion-templates.json b/src/test/resources/integration-tests/assertions/assertion-templates.json new file mode 100644 index 0000000000..848d321756 --- /dev/null +++ b/src/test/resources/integration-tests/assertions/assertion-templates.json @@ -0,0 +1,12 @@ +{ + "meta": { + "name": "assertions-templates", + "kind": "assertionTemplate" + }, + "templates": { + "isSuccess": { + "http.success" : null, + "response.isSuccess": null + } + } +} \ No newline at end of file diff --git a/src/test/resources/integration-tests/vectorize/vectorize-base.json b/src/test/resources/integration-tests/vectorize/vectorize-base.json index 4793799423..d35b10db0f 100644 --- a/src/test/resources/integration-tests/vectorize/vectorize-base.json +++ b/src/test/resources/integration-tests/vectorize/vectorize-base.json @@ -55,7 +55,7 @@ } }, "asserts": { - "response.isSuccess": null, + "template.isSuccess": null, "documents.count": 3 } }, @@ -77,7 +77,7 @@ } }, "asserts": { - "response.isFindAndSuccess": null, + "response.isSuccess": null, "documents.isExactly": { "_id": "Inception", "name": "Inception", From bcf32711e9f565464f61e4bf3d4c8581ca25806b Mon Sep 17 00:00:00 2001 From: Aaron Morton Date: Fri, 27 Feb 2026 13:34:34 +1300 Subject: [PATCH 10/89] WIP --- .../api/v1/vectorize/TestEnvironment.java | 3 +- .../assertions/TemplatedAssertions.java | 26 ++++- .../assertions/assertion-templates.json | 48 +++++++- .../vectorize/vectorize-base.json | 2 +- .../vectorize/vectorize-workflow.json | 105 ++++++++++++++++-- 5 files changed, 166 insertions(+), 18 deletions(-) diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestEnvironment.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestEnvironment.java index 85de1077af..8ae7f368c7 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestEnvironment.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestEnvironment.java @@ -110,7 +110,8 @@ public static String toSafeSchemaIdentifier(String name){ var newValue = PATTERN_NOT_WORD_CHARS.matcher(name).replaceAll("_"); if (newValue.length() > 48){ - throw new RuntimeException("Schema Identifier longer than 48 characters %s=%s".formatted(name,newValue)); + return newValue.substring(0, 47); +// throw new RuntimeException("Schema Identifier longer than 48 characters orginalName=%s, afterNormalisation==%s".formatted(name,newValue)); } return newValue; } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/TemplatedAssertions.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/TemplatedAssertions.java index 9b624b1f70..d6c6380c2d 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/TemplatedAssertions.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/TemplatedAssertions.java @@ -27,14 +27,28 @@ public static AssertionMatcher.TestAssertionContainerFactory getFactory(String t throw new IllegalArgumentException("Assertion template is not an object: " + templateName); } - return new AssertionMatcher.TestAssertionContainerFactory(){ - @Override - public List apply(TestCommand testCommand, JsonNode jsonNode) { - return templateObject.properties().stream() + return switch (templateName.toLowerCase()) { + case "issuccess" -> + (AssertionMatcher.TestAssertionContainerFactory) (testCommand, args) -> { + var commandTemplate = templateObject.get(testCommand.commandName().getApiName()); + if (commandTemplate == null) { + throw new IllegalArgumentException( + "isSuccess Assertion template not found for command: " + + testCommand.commandName().getApiName()); + } + return runTemplate((ObjectNode) commandTemplate, testCommand, args); + }; + default -> + throw new IllegalArgumentException( + "Assertion template not found: " + templateName); + }; + } + + private static List runTemplate(ObjectNode template, TestCommand testCommand, JsonNode args) { + return template.properties().stream() .map(entry -> new TestAssertion.AssertionDefinition(entry.getKey(), entry.getValue())) .map(def -> TestAssertion.buildAssertion(testCommand, def)) .toList(); - } - }; } + } diff --git a/src/test/resources/integration-tests/assertions/assertion-templates.json b/src/test/resources/integration-tests/assertions/assertion-templates.json index 848d321756..394f9bad4c 100644 --- a/src/test/resources/integration-tests/assertions/assertion-templates.json +++ b/src/test/resources/integration-tests/assertions/assertion-templates.json @@ -4,9 +4,51 @@ "kind": "assertionTemplate" }, "templates": { - "isSuccess": { - "http.success" : null, - "response.isSuccess": null + "isSuccess": { + "createCollection": { + "http.success": null, + "response.isDDLSuccess": null + }, + "createKeyspace": { + "http.success": null, + "response.isDDLSuccess": null + }, + "deleteCollection": { + "http.success": null, + "response.isDDLSuccess": null + }, + "dropKeyspace": { + "http.success": null, + "response.isDDLSuccess": null + }, + "find": { + "http.success": null, + "response.isFindSuccess": null + }, + "findOne": { + "http.success": null, + "response.isFindSuccess": null + }, + "insertOne": { + "http.success": null, + "response.isWriteSuccess": null + }, + "insertMany": { + "http.success": null, + "response.isWriteSuccess": null + }, + "findOneAndDelete": { + "http.success": null, + "response.isFindAndSuccess": null + }, + "findOneAndReplace": { + "http.success": null, + "response.isFindAndSuccess": null + }, + "findOneAndUpdate": { + "http.success": null, + "response.isFindAndSuccess": null + } } } } \ No newline at end of file diff --git a/src/test/resources/integration-tests/vectorize/vectorize-base.json b/src/test/resources/integration-tests/vectorize/vectorize-base.json index d35b10db0f..ffc69bf60f 100644 --- a/src/test/resources/integration-tests/vectorize/vectorize-base.json +++ b/src/test/resources/integration-tests/vectorize/vectorize-base.json @@ -77,7 +77,7 @@ } }, "asserts": { - "response.isSuccess": null, + "template.isSuccess": null, "documents.isExactly": { "_id": "Inception", "name": "Inception", diff --git a/src/test/resources/integration-tests/vectorize/vectorize-workflow.json b/src/test/resources/integration-tests/vectorize/vectorize-workflow.json index 6045d6a7dc..ad16917a2a 100644 --- a/src/test/resources/integration-tests/vectorize/vectorize-workflow.json +++ b/src/test/resources/integration-tests/vectorize/vectorize-workflow.json @@ -10,11 +10,10 @@ "tags": [] }, "fromEnvironment": { - "x-embedding-api-key": "x-embedding-api-key", + "x-embedding-api-key": "x_embedding_api_key", "Token": "Token" }, "variables": { - "PROVIDER": "openai", "COLLECTION_NAME": "${PROVIDER}-${MODEL}" }, @@ -33,13 +32,11 @@ "meta": { "name": "voyageAI-vectorize", "tags": [ - "disabled" ] }, "fromEnvironment": { "x-embedding-api-key": "voyageAI_KEY", - "Token" : "Token" - + "Token": "Token" }, "variables": { "PROVIDER": "voyageAI", @@ -61,11 +58,12 @@ { "meta": { "name": "jinaAI-vectorize", - "tags": ["disabled"] + "tags": [ + ] }, "fromEnvironment": { "x-embedding-api-key": "jinaAI_KEY", - "Token" : "Token" + "Token": "Token" }, "variables": { "PROVIDER": "jinaAI", @@ -83,6 +81,99 @@ "tests": [ "vectorize-header-auth" ] + }, + { + "meta": { + "name": "huggingface-non-dedicated-vectorize", + "tags": [] + }, + "fromEnvironment": { + "x-embedding-api-key": "huggingface_KEY", + "Token": "Token" + }, + "variables": { + "PROVIDER": "huggingface", + "COLLECTION_NAME": "${PROVIDER}-${MODEL}" + }, + "matrix": { + "MODEL": [ + "sentence-transformers/all-MiniLM-L6-v2", + "intfloat/multilingual-e5-large", + "intfloat/multilingual-e5-large-instruct", + "BAAI/bge-small-en-v1.5", + "BAAI/bge-base-en-v1.5", + "BAAI/bge-large-en-v1.5" + ] + }, + "tests": [ + "vectorize-header-auth" + ] + }, + { + "meta": { + "name": "mistral-vectorize", + "tags": [] + }, + "fromEnvironment": { + "x-embedding-api-key": "mistral_KEY", + "Token": "Token" + }, + "variables": { + "PROVIDER": "mistral", + "COLLECTION_NAME": "${PROVIDER}-${MODEL}" + }, + "matrix": { + "MODEL": [ + "mistral-embed" + ] + }, + "tests": [ + "vectorize-header-auth" + ] + }, + { + "meta": { + "name": "upstageAI-vectorize", + "tags": [] + }, + "fromEnvironment": { + "x-embedding-api-key": "upstageAI_KEY", + "Token": "Token" + }, + "variables": { + "PROVIDER": "upstageAI", + "COLLECTION_NAME": "${PROVIDER}-${MODEL}" + }, + "matrix": { + "MODEL": [ + "solar-embedding-1-large" + ] + }, + "tests": [ + "vectorize-header-auth" + ] + }, + { + "meta": { + "name": "nvidia-vectorize", + "tags": [] + }, + "fromEnvironment": { + "x-embedding-api-key": "Token", + "Token": "Token" + }, + "variables": { + "PROVIDER": "nvidia", + "COLLECTION_NAME": "${PROVIDER}-${MODEL}" + }, + "matrix": { + "MODEL": [ + "NV-Embed-QA" + ] + }, + "tests": [ + "vectorize-header-auth" + ] } ] } \ No newline at end of file From 0d8294f2c451e65acfbed1b2e0fa6a1be32fc0df Mon Sep 17 00:00:00 2001 From: Aaron Morton Date: Mon, 2 Mar 2026 14:43:29 +1300 Subject: [PATCH 11/89] WIP structure tidy --- .../jsonapi/api/v1/ResponseAssertions.java | 2 +- .../jsonapi/api/v1/vectorize/APIResponse.java | 3 +- .../jsonapi/api/v1/vectorize/Backend.java | 2 + .../api/v1/vectorize/CassandraBackend.java | 4 +- .../v1/vectorize/IntegrationJobRunner.java | 3 +- .../sgv2/jsonapi/api/v1/vectorize/Job.java | 10 +- .../jsonapi/api/v1/vectorize/SpecFile.java | 22 --- .../jsonapi/api/v1/vectorize/SpecFiles.java | 152 ------------------ .../sgv2/jsonapi/api/v1/vectorize/Target.java | 4 +- .../jsonapi/api/v1/vectorize/TestCase.java | 4 +- .../api/v1/vectorize/TestCaseResult.java | 1 + .../api/v1/vectorize/TestEnvironment.java | 2 +- .../jsonapi/api/v1/vectorize/TestPlan.java | 8 +- .../api/v1/vectorize/TestRequestRunner.java | 3 +- .../jsonapi/api/v1/vectorize/TestSpec.java | 15 -- .../api/v1/vectorize/TestSpecMeta.java | 6 - .../jsonapi/api/v1/vectorize/TestSuite.java | 89 ---------- .../api/v1/vectorize/TestSuiteRunner.java | 1 + .../jsonapi/api/v1/vectorize/Workflow.java | 36 ----- .../assertions/AssertionMatcher.java | 146 ----------------- .../assertions/AssertionTemplates.java | 49 ------ .../vectorize/assertions/BodyAssertion.java | 15 +- .../v1/vectorize/assertions/Describable.java | 7 + .../DescribableAssertionMatcher.java | 30 ++++ .../v1/vectorize/assertions/Documents.java | 2 +- .../api/v1/vectorize/assertions/Http.java | 7 +- .../api/v1/vectorize/assertions/Response.java | 2 +- .../assertions/SingleTestAssertion.java | 33 +++- .../api/v1/vectorize/assertions/Status.java | 2 +- .../assertions/TemplatedAssertions.java | 54 ------- .../vectorize/assertions/TestAssertion.java | 70 +++++--- .../assertions/assertion-templates.json | 2 +- .../vectorize/vectorize-base.json | 12 +- .../vectorize/vectorize-header-auth.json | 2 +- .../vectorize/vectorize-workflow.json | 18 ++- 35 files changed, 186 insertions(+), 632 deletions(-) delete mode 100644 src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/SpecFile.java delete mode 100644 src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/SpecFiles.java delete mode 100644 src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestSpec.java delete mode 100644 src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestSpecMeta.java delete mode 100644 src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestSuite.java delete mode 100644 src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/Workflow.java delete mode 100644 src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/AssertionTemplates.java create mode 100644 src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Describable.java create mode 100644 src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/DescribableAssertionMatcher.java delete mode 100644 src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/TemplatedAssertions.java diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/ResponseAssertions.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/ResponseAssertions.java index 7176eb94c2..ae02dd0235 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/ResponseAssertions.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/ResponseAssertions.java @@ -87,7 +87,7 @@ public class ResponseAssertions { FieldMatcher.errors(hasErrors)); final String msg = - "%s: Response fields %s:%s, %s:%s, %s:%s" + "%s: %s:%s, %s:%s, %s:%s" .formatted( message, Presence.REQUIRED, diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/APIResponse.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/APIResponse.java index 5f14d2b5c1..05120514be 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/APIResponse.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/APIResponse.java @@ -1,8 +1,7 @@ package io.stargate.sgv2.jsonapi.api.v1.vectorize; -import com.fasterxml.jackson.databind.node.ObjectNode; import io.restassured.response.ValidatableResponse; public record APIResponse (APIRequest apiRequest, - ValidatableResponse validatableResponse) { + ValidatableResponse validatable) { } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/Backend.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/Backend.java index eff80f178d..6e6a147e69 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/Backend.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/Backend.java @@ -1,5 +1,7 @@ package io.stargate.sgv2.jsonapi.api.v1.vectorize; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.TestSuite; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.Workflow; import org.junit.jupiter.api.DynamicNode; import java.util.Optional; diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/CassandraBackend.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/CassandraBackend.java index 76a31a96e3..916ff8582a 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/CassandraBackend.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/CassandraBackend.java @@ -40,7 +40,7 @@ public Optional beforeJob(TestPlan testPlan, Job job) { var env = job.withoutMatrix(testPlan); var setupRequest = new TestRequest( env.substitutor().replace("createKeyspace: ${KEYSPACE_NAME}"), - command, testPlan.target(), env, TestAssertion.forSuccess(command)); + command, testPlan.target(), env, TestAssertion.forSuccess( testPlan, command)); return Optional.of(setupRequest.testNodes()); } @@ -59,7 +59,7 @@ public Optional afterJob(TestPlan testPlan, Job job) { var env = job.withoutMatrix(testPlan); var setupRequest = new TestRequest( env.substitutor().replace("dropKeyspace: ${KEYSPACE_NAME}"), - command, testPlan.target(), job.withoutMatrix(testPlan), TestAssertion.forSuccess(command)); + command, testPlan.target(), job.withoutMatrix(testPlan), TestAssertion.forSuccess(testPlan,command)); return Optional.of(setupRequest.testNodes()); } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/IntegrationJobRunner.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/IntegrationJobRunner.java index 1aeec9bcf3..12551cdd5a 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/IntegrationJobRunner.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/IntegrationJobRunner.java @@ -1,7 +1,6 @@ package io.stargate.sgv2.jsonapi.api.v1.vectorize; -import java.util.ArrayList; -import java.util.List; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.SpecFiles; public class IntegrationJobRunner { diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/Job.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/Job.java index 14d851175d..93935df839 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/Job.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/Job.java @@ -1,11 +1,13 @@ package io.stargate.sgv2.jsonapi.api.v1.vectorize; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.TestSpecKind; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.TestSpecMeta; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.TestSuite; import org.junit.jupiter.api.DynamicContainer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.stream.Stream; @@ -39,14 +41,14 @@ public DynamicContainer testNode(TestPlan testPlan) { } public Stream testSuites(TestPlan testPlan) { - List allTests = new ArrayList<>(); + Stream.Builder allTests = Stream.builder(); tests() .forEach( testName -> { - allTests.addAll(testPlan.specFiles().testByName(testName)); + testPlan.specFiles().byNameAsType(TestSuite.class, testName).forEach(allTests) ; }); - return allTests.stream(); + return allTests.build(); } public TestEnvironment withoutMatrix(TestPlan testPlan) { diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/SpecFile.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/SpecFile.java deleted file mode 100644 index 5bcfc17bb4..0000000000 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/SpecFile.java +++ /dev/null @@ -1,22 +0,0 @@ -package io.stargate.sgv2.jsonapi.api.v1.vectorize; - -import com.fasterxml.jackson.databind.JsonNode; - -public class SpecFile { - - private final TestSpec spec; - private final JsonNode rootNode; - - SpecFile(TestSpec spec, JsonNode rootNode) { - this.spec = spec; - this.rootNode = rootNode; - } - - public TestSpec spec() { - return spec; - } - - public JsonNode root() { - return rootNode; - } -} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/SpecFiles.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/SpecFiles.java deleted file mode 100644 index 1f363a2817..0000000000 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/SpecFiles.java +++ /dev/null @@ -1,152 +0,0 @@ -package io.stargate.sgv2.jsonapi.api.v1.vectorize; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.node.ArrayNode; -import com.fasterxml.jackson.databind.node.ObjectNode; -import com.jayway.jsonpath.Configuration; -import com.jayway.jsonpath.JsonPath; -import com.jayway.jsonpath.spi.json.JacksonJsonNodeJsonProvider; -import com.jayway.jsonpath.spi.mapper.JacksonMappingProvider; -import java.io.IOException; -import java.io.UncheckedIOException; -import java.net.URISyntaxException; -import java.net.URL; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.ArrayList; -import java.util.List; -import java.util.stream.Stream; - -/** - * Collection of all the test spec files read for this execution. - */ -public class SpecFiles { - - private static final ObjectMapper MAPPER = new ObjectMapper(); - - private static final Configuration JSON_PATH_CONFIG = - Configuration.builder() - .jsonProvider(new JacksonJsonNodeJsonProvider()) - .mappingProvider(new JacksonMappingProvider()) - .build(); - - List specFiles; - - private SpecFiles(List specFiles) { - this.specFiles = specFiles; - - for (SpecFile file : specFiles) { - if (file.spec() instanceof TestSuite it){ - it.expand(this); - } - } - } - - public Workflow workflowFirstByName(String name) { - var path = "$.meta[?(@.name == '%s')]".formatted(name); - - var itFile = - match(TestSpec.TestSpecKind.WORKFLOW, path).stream() - .findFirst() - .orElseThrow( - () -> - new IllegalArgumentException( - "No IT file found with meta.name == '%s'".formatted(name))); - return (Workflow) itFile.spec(); - } - - public TestSuite testFirstByName(String name) { - return testByName(name).getFirst(); - } - - public List testByName(String name) { - var path = "$.meta[?(@.name == '%s')]".formatted(name); - - return match(TestSpec.TestSpecKind.TEST, path).stream() - .map(itFile -> (TestSuite) itFile.spec()) - .toList(); - } - - public Stream byKind(TestSpec.TestSpecKind kind) { - return specFiles.stream().filter(itFile -> itFile.spec().kind() == kind); - } - - private List match(TestSpec.TestSpecKind kind, String jsonPath) { - var compiled = JsonPath.compile(jsonPath); - - return byKind(kind).filter(itFile -> hasMatch(itFile.root(), compiled)).toList(); - } - - private boolean hasMatch(JsonNode root, JsonPath compiled) { - var pathResult = JsonPath.using(JSON_PATH_CONFIG).parse(root).read(compiled); - - return switch (pathResult) { - case null -> false; - case java.util.Collection c -> !c.isEmpty(); - case java.util.Map m -> !m.isEmpty(); - case ArrayNode a -> !a.isEmpty(); - case ObjectNode o -> !o.isEmpty(); - default -> true; - }; - } - - static SpecFiles loadAll(String path) { - final Path dir = resourceDir(path); - - List itFiles = new ArrayList<>(); - - try (Stream s = Files.walk(dir)) { - itFiles = - s.filter(Files::isRegularFile) - .filter(p -> p.getFileName().toString().endsWith(".json")) - .map(SpecFiles::loadOne) - .toList(); - } catch (IOException e) { - throw new UncheckedIOException("Failed reading test resources under: " + dir, e); - } - return new SpecFiles(itFiles); - } - - private static SpecFile loadOne(Path file) { - try { - var root = MAPPER.readTree(file.toFile()); - - JsonNode kindNode = root.path("meta").path("kind"); - if (!kindNode.isTextual()) { - throw new IllegalArgumentException("Missing/invalid meta.kind in " + file); - } - - var kind = kindNode.asText(); - var element = - switch (kind.toUpperCase()) { - case "TEST" -> MAPPER.treeToValue(root, TestSuite.class); - case "WORKFLOW" -> MAPPER.treeToValue(root, Workflow.class); - default -> - throw new IllegalArgumentException("Unknown meta.kind '" + kind + "' in " + file); - }; - return new SpecFile(element, root); - } catch (IOException e) { - throw new UncheckedIOException("Failed parsing JSON file: " + file, e); - } - } - - private static Path resourceDir(String path) { - String normalized = path.startsWith("/") ? path.substring(1) : path; - - ClassLoader cl = Thread.currentThread().getContextClassLoader(); - URL url = cl.getResource(normalized); - if (url == null) { - throw new IllegalArgumentException("Test resource folder not found: " + path); - } - - try { - // Works for file: URLs; if you run tests from a jar, switch to getResourceAsStream-based - // walking. - return Paths.get(url.toURI()); - } catch (URISyntaxException e) { - throw new IllegalArgumentException("Bad resource URI for: " + path + " -> " + url, e); - } - } -} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/Target.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/Target.java index 805732df42..8aa2e50a05 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/Target.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/Target.java @@ -1,5 +1,7 @@ package io.stargate.sgv2.jsonapi.api.v1.vectorize; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.TestSuite; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.Workflow; import org.junit.jupiter.api.DynamicNode; import java.util.HashMap; @@ -52,7 +54,7 @@ public Optional afterJob(TestPlan testPlan,Job job){ return backend.afterJob(testPlan, job); } - public Optional beforeTestSuite(TestPlan testPlan,TestSuite test, TestEnvironment env){ + public Optional beforeTestSuite(TestPlan testPlan, TestSuite test, TestEnvironment env){ return backend.beforeTestSuite(testPlan, test, env); } public Optional afterTestSuite(TestPlan testPlan,TestSuite test, TestEnvironment env){ diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestCase.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestCase.java index 2508eb985e..cb49caa5c5 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestCase.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestCase.java @@ -15,11 +15,11 @@ public record TestCase( String include) { - DynamicContainer testNodesForEnvironment(TestPlan testPlan, TestEnvironment testEnvironment) { + public DynamicContainer testNodesForEnvironment(TestPlan testPlan, TestEnvironment testEnvironment) { var testRequest = new TestRequest( "TestCase: name=%s".formatted(name, command.commandName()), - command(), testPlan.target(), testEnvironment, TestAssertion.buildAssertions(this)); + command(), testPlan.target(), testEnvironment, TestAssertion.buildAssertions(testPlan, this)); return testRequest.testNodes(); } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestCaseResult.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestCaseResult.java index 297adb22ca..eccc7429db 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestCaseResult.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestCaseResult.java @@ -1,6 +1,7 @@ package io.stargate.sgv2.jsonapi.api.v1.vectorize; import io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions.AssertionMatcher; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.TestSuite; public record TestCaseResult( TestSuite integrationTest, diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestEnvironment.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestEnvironment.java index 8ae7f368c7..9f0e0bbdf2 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestEnvironment.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestEnvironment.java @@ -1,5 +1,6 @@ package io.stargate.sgv2.jsonapi.api.v1.vectorize; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.TestSuite; import org.apache.commons.text.StringSubstitutor; import org.apache.commons.text.lookup.StringLookupFactory; import org.junit.jupiter.api.DynamicContainer; @@ -10,7 +11,6 @@ import java.util.Map; import java.util.Set; import java.util.regex.Pattern; -import java.util.stream.Stream; import static org.junit.jupiter.api.DynamicTest.dynamicTest; diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestPlan.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestPlan.java index 09f40ac141..146247776e 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestPlan.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestPlan.java @@ -1,11 +1,9 @@ package io.stargate.sgv2.jsonapi.api.v1.vectorize; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions.AssertionTemplates; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.*; import org.junit.jupiter.api.DynamicContainer; import org.junit.jupiter.api.DynamicNode; -import java.io.IOException; -import java.nio.file.Path; import java.util.List; import java.util.Optional; import java.util.Set; @@ -21,14 +19,14 @@ public static TestPlan create(String targetName, List workflows){ var targetConfigs = TargetConfigurationss.loadAll("integration-tests/targets/targets.json"); var target = new Target(targetConfigs.configuration(targetName)); - var specFiles = SpecFiles.loadAll("integration-tests/vectorize"); + var specFiles = SpecFiles.loadAll(List.of("integration-tests/vectorize", "integration-tests/assertions/assertion-templates.json")); return new TestPlan(target, specFiles, Set.copyOf(workflows)); } public Stream selectedWorkflows(){ - return specFiles.byKind(TestSpec.TestSpecKind.WORKFLOW) + return specFiles.byKind(TestSpecKind.WORKFLOW) .filter(specFile -> workflows.isEmpty() || workflows.contains(specFile.spec().meta().name()) ) diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestRequestRunner.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestRequestRunner.java index 06388569cc..294c2c11d0 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestRequestRunner.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestRequestRunner.java @@ -1,8 +1,7 @@ package io.stargate.sgv2.jsonapi.api.v1.vectorize; -import org.junit.jupiter.api.DynamicTest; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.TestSuite; -import java.util.Collection; import java.util.Objects; import static org.junit.jupiter.api.DynamicTest.dynamicTest; diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestSpec.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestSpec.java deleted file mode 100644 index 01f0a1ce73..0000000000 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestSpec.java +++ /dev/null @@ -1,15 +0,0 @@ -package io.stargate.sgv2.jsonapi.api.v1.vectorize; - -public interface TestSpec { - - TestSpecKind kind(); - - TestSpecMeta meta(); - - // void setJson(JsonNode json); - - enum TestSpecKind { - TEST, - WORKFLOW - } -} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestSpecMeta.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestSpecMeta.java deleted file mode 100644 index fcf919b4a2..0000000000 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestSpecMeta.java +++ /dev/null @@ -1,6 +0,0 @@ -package io.stargate.sgv2.jsonapi.api.v1.vectorize; - -import java.util.List; - -public record TestSpecMeta(String name, String kind, - List tags) {} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestSuite.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestSuite.java deleted file mode 100644 index b027f17031..0000000000 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestSuite.java +++ /dev/null @@ -1,89 +0,0 @@ -package io.stargate.sgv2.jsonapi.api.v1.vectorize; - -import io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions.AssertionMatcher; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions.TestAssertion; -import org.junit.jupiter.api.DynamicContainer; -import org.junit.jupiter.api.DynamicNode; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; - -import static org.junit.jupiter.api.DynamicContainer.dynamicContainer; - -public record TestSuite(TestSpecMeta meta, List setup, List tests, List cleanup) - implements TestSpec { - - @Override - public TestSpecKind kind() { - return TestSpecKind.TEST; - } - - public DynamicContainer testNode(TestPlan testPlan, List allEnvs) { - - var desc = "TestSuite: %s ".formatted( - meta.name()); - - return dynamicContainer( - desc, - allEnvs.stream() - .map(testEnv -> testEnv.testNode(testPlan, this)) - ); - } - - Collection testNodesForEnvironment(TestPlan testPlan, TestEnvironment testEnvironment) { - - List nodes = new ArrayList<>(); - - int i = 1; - for (TestCommand setupCommand : setup()) { - var setupRequest = new TestRequest( - "SetupRequest[%s]: %s".formatted(i++, setupCommand.commandName()), - setupCommand, testPlan.target(), testEnvironment, TestAssertion.forSuccess(setupCommand)); - - nodes.add(setupRequest.testNodes()); - } - - for (var testCase : tests()) { - nodes.add(testCase.testNodesForEnvironment(testPlan, testEnvironment)); - } - - for (TestCommand cleanupCommand : cleanup()) { - var cleanupRequest = new TestRequest( - "CleanupRequest[%s]: %s".formatted(i++, cleanupCommand.commandName()), - cleanupCommand, testPlan.target(), testEnvironment, TestAssertion.forSuccess(cleanupCommand)); - nodes.add(cleanupRequest.testNodes()); - } - - return nodes; - - } - public void expand(SpecFiles itCollection) { - - List expandedSetup = new ArrayList<>(); - for (TestCommand command : setup) { - if (command.includeFrom() != null){ - var includedTest = itCollection.testFirstByName(command.includeFrom()); - expandedSetup.addAll(includedTest.setup()); - } - else { - expandedSetup.add(command); - } - } - setup.clear(); - setup.addAll(expandedSetup); - - List expandedTests = new ArrayList<>(); - for (TestCase item : tests) { - if (item.include() != null ){ - var includedTest = itCollection.testFirstByName(item.include()); - expandedTests.addAll(includedTest.tests()); - } - else { - expandedTests.add(item); - } - } - tests.clear(); - tests.addAll(expandedTests); - } -} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestSuiteRunner.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestSuiteRunner.java index 3aaabe045a..b697b4e04c 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestSuiteRunner.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestSuiteRunner.java @@ -1,6 +1,7 @@ package io.stargate.sgv2.jsonapi.api.v1.vectorize; import com.fasterxml.jackson.databind.ObjectMapper; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.TestSuite; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/Workflow.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/Workflow.java deleted file mode 100644 index ed3666de1e..0000000000 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/Workflow.java +++ /dev/null @@ -1,36 +0,0 @@ -package io.stargate.sgv2.jsonapi.api.v1.vectorize; - -import org.junit.jupiter.api.DynamicContainer; - -import java.util.List; -import java.util.stream.Stream; - -import static org.junit.jupiter.api.DynamicContainer.dynamicContainer; - -public record Workflow(TestSpecMeta meta, List jobs) implements TestSpec { - - @Override - public TestSpecKind kind() { - return TestSpecKind.WORKFLOW; - } - - public Stream activeJobs(){ - return jobs().stream() - .filter(job -> !job.meta().tags().contains("disabled")); - } - - - public DynamicContainer testNode(TestPlan testPlan) { - - var desc = "Workflow: %s ".formatted( - meta.name()); - - var jobNodes = activeJobs() - .map(job -> job.testNode(testPlan)); - - return dynamicContainer( - desc, - testPlan.addLifecycle(this, jobNodes)); - } - -} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/AssertionMatcher.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/AssertionMatcher.java index 2405419e15..70d4225013 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/AssertionMatcher.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/AssertionMatcher.java @@ -1,17 +1,6 @@ package io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions; -import com.fasterxml.jackson.databind.JsonNode; import io.stargate.sgv2.jsonapi.api.v1.vectorize.APIResponse; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.TestCommand; -import org.assertj.core.api.AssertFactory; - -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; -import java.lang.reflect.Modifier; -import java.util.List; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; -import java.util.function.BiFunction; /** * Contract for running an assertion on the response from the API. @@ -27,139 +16,4 @@ public interface AssertionMatcher { */ void match(APIResponse apiResponse); - AssertionFactoryRegistry FACTORY_REGISTRY = new AssertionFactoryRegistry(); - - /** - * Assertions int the test case are used to find Assertion factories in the code, - * they are called with the TestCommand they need to run against so you - * know what the command is, and any args from the JSON for the test case. - *

- * Use the {@link SingleFactory} signature if your factory returns a single matcher, which is - * normal. Use the {@link MultiFactory} signature if you want to return more than one. - *

- */ - - interface AssertionFactory extends BiFunction {} - - interface AssertionMatcherFactory extends AssertionFactory { - } - - interface TestAssertionContainerFactory extends AssertionFactory> {} - - class AssertionFactoryRegistry { - - private final Map> factoryMethods = new ConcurrentHashMap<>(); - - public void register(Class cls) { - for (var method : cls.getMethods()) { - if (isValidFactoryMethod(method)) { - var factoryKey = cls.getSimpleName().toLowerCase() - + "." - + method.getName().toLowerCase(); - - // NOTE: not checking the generic of the list - AssertionFactory wrapped = (method.getReturnType() == AssertionMatcher.class) ? - new AssertionMatcherFactoryWrapper(method) - : - new TestAssertionContainerFactoryWrapper(method); - factoryMethods.put(factoryKey, wrapped); - } - } - } - - public AssertionFactory get(String name){ - var normalName = name.toLowerCase(); - - int pos = normalName.indexOf('.'); - if (pos < 0){ - throw new IllegalArgumentException("Name must have a dot: " + name); - } - var type = normalName.substring(0, pos); - var func = normalName.substring(pos + 1); - - if (type.equals( "template")) { - return TemplatedAssertions.getFactory(func); - } - - var factoryMethod = factoryMethods.get(normalName); - if (factoryMethod == null) { - loadClassFor(name); - } - - factoryMethod = factoryMethods.get(normalName); - if (factoryMethod == null) { - throw new IllegalArgumentException("Unknown assertion factory. (normalised)name: %s known=%s".formatted(name,factoryMethods.keySet())); - } - return factoryMethod; - } - - private static final String PACKAGE = - "io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions"; - - private void loadClassFor(String key) { - - int dot = key.indexOf('.'); - if (dot < 0) { - return; - } - - var typeName = key.substring(0, dot); - - var className = - PACKAGE + "." - + Character.toUpperCase(typeName.charAt(0)) - + typeName.substring(1); - - try { - Class.forName(className); - - // class static initializer should call register() - - } catch (ClassNotFoundException ignored) { - // expected if class does not exist - } - } - - private record AssertionMatcherFactoryWrapper(Method method) implements AssertionMatcherFactory { - - @SuppressWarnings("unchecked") - @Override - public AssertionMatcher apply(TestCommand testCommand, JsonNode args) { - try { - return (AssertionMatcher) method.invoke(null, testCommand, args); - } catch (IllegalAccessException | InvocationTargetException e) { - throw new RuntimeException(e); - } - } - } - - private record TestAssertionContainerFactoryWrapper(Method method) implements TestAssertionContainerFactory { - - @SuppressWarnings("unchecked") - @Override - public List apply(TestCommand testCommand, JsonNode args) { - try { - return (List) method.invoke(null, testCommand, args); - } catch (IllegalAccessException | InvocationTargetException e) { - throw new RuntimeException(e); - } - } - } - - private static boolean isValidFactoryMethod(Method method) { - - if (!Modifier.isStatic(method.getModifiers())) { - return false; - } - - if ((!List.class.isAssignableFrom(method.getReturnType())) && method.getReturnType() != AssertionMatcher.class) { - return false; - } - var p = method.getParameterTypes(); - return p.length == 2 - && p[0] == TestCommand.class - && p[1] == JsonNode.class; - } - - } } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/AssertionTemplates.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/AssertionTemplates.java deleted file mode 100644 index 82476b8ce6..0000000000 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/AssertionTemplates.java +++ /dev/null @@ -1,49 +0,0 @@ -package io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions; - -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.node.ObjectNode; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.TargetConfigurationss; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.TestSpecMeta; - -import java.io.IOException; -import java.net.URISyntaxException; -import java.net.URL; -import java.nio.file.Path; -import java.nio.file.Paths; - -public record AssertionTemplates( - TestSpecMeta meta, - ObjectNode templates -) { - - private static final ObjectMapper MAPPER = new ObjectMapper(); - - public static AssertionTemplates load(){ - - final Path dir = resourceDir("integration-tests/assertions/assertion-templates.json"); - - try { - return MAPPER.readValue(dir.toFile(), AssertionTemplates.class); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - - private static Path resourceDir(String path) { - String normalized = path.startsWith("/") ? path.substring(1) : path; - - ClassLoader cl = Thread.currentThread().getContextClassLoader(); - URL url = cl.getResource(normalized); - if (url == null) { - throw new IllegalArgumentException("Test resource folder not found: " + path); - } - - try { - // Works for file: URLs; if you run tests from a jar, switch to getResourceAsStream-based - // walking. - return Paths.get(url.toURI()); - } catch (URISyntaxException e) { - throw new IllegalArgumentException("Bad resource URI for: " + path + " -> " + url, e); - } - } -} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/BodyAssertion.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/BodyAssertion.java index 0f048667f9..6416d294f3 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/BodyAssertion.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/BodyAssertion.java @@ -2,6 +2,7 @@ import io.stargate.sgv2.jsonapi.api.v1.vectorize.APIResponse; import org.hamcrest.Matcher; +import org.hamcrest.StringDescription; /** * Assertions that check the body of the response using a {@link Matcher} form hamcrest. @@ -16,9 +17,19 @@ public record BodyAssertion( String bodyPath, Matcher matcher -) implements AssertionMatcher { +) implements Describable, AssertionMatcher { + @Override public void match(APIResponse apiResponse) { - apiResponse.validatableResponse().body(bodyPath(), matcher()); + apiResponse.validatable().body(bodyPath(), matcher()); + } + + @Override + public String describe() { + var describable = new StringDescription(); + matcher.describeTo(describable); + + // called should truncate if it wants to limit it + return "body('%s') - %s".formatted(bodyPath(), describable.toString()); } } \ No newline at end of file diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Describable.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Describable.java new file mode 100644 index 0000000000..fc1a0a5f67 --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Describable.java @@ -0,0 +1,7 @@ +package io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions; + +@FunctionalInterface +public interface Describable { + + String describe(); +} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/DescribableAssertionMatcher.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/DescribableAssertionMatcher.java new file mode 100644 index 0000000000..9e001ac0c6 --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/DescribableAssertionMatcher.java @@ -0,0 +1,30 @@ +package io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions; + +import io.stargate.sgv2.jsonapi.api.v1.vectorize.APIResponse; +import org.jspecify.annotations.NonNull; + +public record DescribableAssertionMatcher( + String description, + AssertionMatcher matcher +) implements Describable, AssertionMatcher { + + + public static DescribableAssertionMatcher described(String description, AssertionMatcher matcher) { + return new DescribableAssertionMatcher(description, matcher); + } + + @Override + public void match(APIResponse apiResponse) { + matcher.match(apiResponse); + } + + @Override + public String describe() { + return toString(); + } + + @Override + public @NonNull String toString() { + return description(); + } +} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Documents.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Documents.java index cd4dbc5078..3deb8135af 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Documents.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Documents.java @@ -16,7 +16,7 @@ public class Documents { static { - AssertionMatcher.FACTORY_REGISTRY.register(Documents.class); + AssertionFactory.REGISTRY.register(Documents.class); } public static AssertionMatcher count(TestCommand testCommand, JsonNode args) { diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Http.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Http.java index f9424f0aee..f1afc680dd 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Http.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Http.java @@ -4,13 +4,16 @@ import io.stargate.sgv2.jsonapi.api.v1.vectorize.TestCommand; import org.apache.http.HttpStatus; +import static io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions.DescribableAssertionMatcher.described; + public class Http { static { - AssertionMatcher.FACTORY_REGISTRY.register(Http.class); + AssertionFactory.REGISTRY.register(Http.class); } public static AssertionMatcher success(TestCommand testCommand, JsonNode args){ - return apiResponse -> apiResponse.validatableResponse().statusCode(HttpStatus.SC_OK); + return described("http status is groovy", apiResponse -> apiResponse.validatable().statusCode(HttpStatus.SC_OK)); +// return apiResponse -> apiResponse.validatable().statusCode(HttpStatus.SC_OK); } } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Response.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Response.java index d198db2fc5..f085e91877 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Response.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Response.java @@ -20,7 +20,7 @@ public class Response { static { - AssertionMatcher.FACTORY_REGISTRY.register(Response.class); + AssertionFactory.REGISTRY.register(Response.class); } /** diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/SingleTestAssertion.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/SingleTestAssertion.java index 40750ef352..1cfd9ac457 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/SingleTestAssertion.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/SingleTestAssertion.java @@ -2,11 +2,17 @@ import com.fasterxml.jackson.databind.JsonNode; import io.stargate.sgv2.jsonapi.api.v1.vectorize.TestResponse; +import io.stargate.sgv2.jsonapi.service.schema.tables.ApiSupportDef; +import org.hamcrest.SelfDescribing; +import org.hamcrest.StringDescription; import org.junit.jupiter.api.DynamicNode; +import java.lang.reflect.Executable; +import java.util.Objects; import java.util.concurrent.atomic.AtomicReference; import static org.junit.jupiter.api.DynamicTest.dynamicTest; +import static org.junit.jupiter.api.DynamicTest.stream; public record SingleTestAssertion( String name, @@ -30,11 +36,36 @@ public void run(TestResponse testResponse) { @Override public DynamicNode testNodes(AtomicReference testResponse) { - return dynamicTest(name(), () -> { + var original = (matcher instanceof Describable d) ? + d.describe() + : + null; + + var truncated = ( original != null && original.length() > 60) ? + original.substring(0, 57) + "..." + : + original; + + var testDesc = truncated == null ? + name() + : + "%s [%s]".formatted(name(), truncated); + + // if we truncated the description of the test, we then want to pipe to std out when running + // because it will not be full in the test tree + var stdoutMessage = (original != null && !Objects.equals(truncated, original)) ? + "%s [%s]".formatted(name(), original) + : + null; + + return dynamicTest(testDesc, () -> { var resp = testResponse.get(); if (resp == null) { throw new IllegalStateException("Response is null"); } + if (stdoutMessage != null) { + System.out.printf(stdoutMessage); + } run(resp); } ); diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Status.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Status.java index adc9bd703f..4ae2703a89 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Status.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Status.java @@ -8,7 +8,7 @@ public class Status { static { - AssertionMatcher.FACTORY_REGISTRY.register(Status.class); + AssertionFactory.REGISTRY.register(Status.class); } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/TemplatedAssertions.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/TemplatedAssertions.java deleted file mode 100644 index d6c6380c2d..0000000000 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/TemplatedAssertions.java +++ /dev/null @@ -1,54 +0,0 @@ -package io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.ObjectNode; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.TestCommand; - -import java.util.List; - -public class TemplatedAssertions { - - private static final AssertionTemplates assertions = AssertionTemplates.load(); - - public static AssertionMatcher.TestAssertionContainerFactory getFactory(String templateName){ - - JsonNode template = null; - for (var entry : assertions.templates().properties()){ - if (entry.getKey().equalsIgnoreCase(templateName)) { - template = entry.getValue(); - break; - } - } - - if (template == null) { - throw new IllegalArgumentException("Assertion template not found: " + templateName); - } - if (! (template instanceof ObjectNode templateObject)){ - throw new IllegalArgumentException("Assertion template is not an object: " + templateName); - } - - return switch (templateName.toLowerCase()) { - case "issuccess" -> - (AssertionMatcher.TestAssertionContainerFactory) (testCommand, args) -> { - var commandTemplate = templateObject.get(testCommand.commandName().getApiName()); - if (commandTemplate == null) { - throw new IllegalArgumentException( - "isSuccess Assertion template not found for command: " - + testCommand.commandName().getApiName()); - } - return runTemplate((ObjectNode) commandTemplate, testCommand, args); - }; - default -> - throw new IllegalArgumentException( - "Assertion template not found: " + templateName); - }; - } - - private static List runTemplate(ObjectNode template, TestCommand testCommand, JsonNode args) { - return template.properties().stream() - .map(entry -> new TestAssertion.AssertionDefinition(entry.getKey(), entry.getValue())) - .map(def -> TestAssertion.buildAssertion(testCommand, def)) - .toList(); - } - -} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/TestAssertion.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/TestAssertion.java index b01736cef1..887c1300a1 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/TestAssertion.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/TestAssertion.java @@ -3,11 +3,16 @@ import com.fasterxml.jackson.databind.JsonNode; import io.stargate.sgv2.jsonapi.api.v1.vectorize.TestCase; import io.stargate.sgv2.jsonapi.api.v1.vectorize.TestCommand; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.TestPlan; import io.stargate.sgv2.jsonapi.api.v1.vectorize.TestResponse; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.AssertionTemplateSpec; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.SpecFiles; +import org.assertj.core.api.AssertFactory; import org.junit.jupiter.api.DynamicNode; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Stream; @@ -22,50 +27,73 @@ public interface TestAssertion { DynamicNode testNodes(AtomicReference testResponse); - static List forSuccess(TestCommand testCommand) { + static List forSuccess(TestPlan testPlan, TestCommand testCommand) { var builder = Stream.builder() - .add(new AssertionDefinition("template.isSuccess", null)); + .add(new AssertionDefinition("Templated.isSuccess", null)); - return buildAssertions(testCommand, builder.build()); + return buildAssertions(testPlan, testCommand, builder.build()); } - static List buildAssertions(TestCase testCase) { + static List buildAssertions(TestPlan testPlan, TestCase testCase) { var defs = testCase.asserts().properties().stream() .map(AssertionDefinition::create); - return buildAssertions(testCase.command(), defs); + return buildAssertions(testPlan, testCase.command(), defs); } - static List buildAssertions(TestCommand testCommand, List defs) { - - return buildAssertions(testCommand, defs.stream()); - } - - static List buildAssertions(TestCommand testCommand, Stream defs) { + static List buildAssertions(TestPlan testPlan, TestCommand testCommand, Stream defs) { return defs.map( - def -> buildAssertion(testCommand, def) + def -> buildAssertion(testPlan, testCommand, def) ).toList(); } - public static TestAssertion buildAssertion(TestCommand testCommand, AssertionDefinition def) { - - return switch (AssertionMatcher.FACTORY_REGISTRY.get(def.name())) { - case AssertionMatcher.AssertionMatcherFactory factory -> - new SingleTestAssertion(def.name(), def.args(), factory.apply(testCommand, def.args())); - case AssertionMatcher.TestAssertionContainerFactory factory -> - new TestAssertionContainer(def.name(), def.args(), factory.apply(testCommand, def.args)); - default -> throw new IllegalStateException("Unknown TestAssertionFactory: " + def.name()); - }; + static TestAssertion buildAssertion(TestPlan testPlan, TestCommand testCommand, AssertionDefinition def) { + return def.addFactory(AssertionFactory.REGISTRY).build(testPlan, testCommand); } + /** + *

+ */ record AssertionDefinition(String name, JsonNode args) { static AssertionDefinition create(Map.Entry def) { return new AssertionDefinition(def.getKey(), def.getValue()); } + AssertionDefWithFactory addFactory(AssertionFactoryRegistry registry) { + + var factory = registry.getWrapped(name()); + if (factory == null) { + throw new IllegalStateException("Unknown Assertion Factory name=" + name()); + } + return new AssertionDefWithFactory(factory, args); + } } + /** + *

+ */ + record AssertionDefWithFactory(AssertionFactory.WrappedMethod method, JsonNode args){ + + TestAssertion build(TestPlan testPlan, TestCommand testCommand) { + + return switch (method) { + case AssertionFactory.WrappedAssertionMatcherFactory factory -> + new SingleTestAssertion(method.properName(), args(), factory.create(testCommand, args())); + + case AssertionFactory.TemplatedAssertionFactory factory ->{ + + var template = testPlan.specFiles().byType(AssertionTemplateSpec.class) + .flatMap(assertTemplate -> assertTemplate.templateFor(method.properName()).stream()) + .findFirst() + .orElseThrow(() -> new IllegalStateException("Unknown Assertion Template name=" + method.properName())); + + yield new TestAssertionContainer(method.properName(), args(), factory.create(testPlan, template, testCommand, args())); + } + }; + } + + } } diff --git a/src/test/resources/integration-tests/assertions/assertion-templates.json b/src/test/resources/integration-tests/assertions/assertion-templates.json index 394f9bad4c..76bcea748e 100644 --- a/src/test/resources/integration-tests/assertions/assertion-templates.json +++ b/src/test/resources/integration-tests/assertions/assertion-templates.json @@ -1,7 +1,7 @@ { "meta": { "name": "assertions-templates", - "kind": "assertionTemplate" + "kind": "assertion_template" }, "templates": { "isSuccess": { diff --git a/src/test/resources/integration-tests/vectorize/vectorize-base.json b/src/test/resources/integration-tests/vectorize/vectorize-base.json index ffc69bf60f..43f07afb92 100644 --- a/src/test/resources/integration-tests/vectorize/vectorize-base.json +++ b/src/test/resources/integration-tests/vectorize/vectorize-base.json @@ -1,7 +1,7 @@ { "meta": { "name": "vectorize-base", - "kind" : "test" + "kind" : "test_suite" }, "setup": [ { @@ -55,8 +55,8 @@ } }, "asserts": { - "template.isSuccess": null, - "documents.count": 3 + "Templated.isSuccess": null, + "Documents.count": 3 } }, { @@ -77,8 +77,8 @@ } }, "asserts": { - "template.isSuccess": null, - "documents.isExactly": { + "Templated.isSuccess": null, + "Documents.isExactly": { "_id": "Inception", "name": "Inception", "genre": "Science Fiction", @@ -87,7 +87,7 @@ ], "status": "active" }, - "status.isExactly" : { + "Status.isExactly" : { "matchedCount": 1, "modifiedCount": 1 } diff --git a/src/test/resources/integration-tests/vectorize/vectorize-header-auth.json b/src/test/resources/integration-tests/vectorize/vectorize-header-auth.json index 11d61833f1..2aa9cd3053 100644 --- a/src/test/resources/integration-tests/vectorize/vectorize-header-auth.json +++ b/src/test/resources/integration-tests/vectorize/vectorize-header-auth.json @@ -1,7 +1,7 @@ { "meta": { "name": "vectorize-header-auth", - "kind": "test" + "kind": "test_suite" }, "setup": [ { diff --git a/src/test/resources/integration-tests/vectorize/vectorize-workflow.json b/src/test/resources/integration-tests/vectorize/vectorize-workflow.json index ad16917a2a..5cbfce0258 100644 --- a/src/test/resources/integration-tests/vectorize/vectorize-workflow.json +++ b/src/test/resources/integration-tests/vectorize/vectorize-workflow.json @@ -32,6 +32,7 @@ "meta": { "name": "voyageAI-vectorize", "tags": [ + "disabled" ] }, "fromEnvironment": { @@ -59,6 +60,7 @@ "meta": { "name": "jinaAI-vectorize", "tags": [ + "disabled" ] }, "fromEnvironment": { @@ -85,7 +87,9 @@ { "meta": { "name": "huggingface-non-dedicated-vectorize", - "tags": [] + "tags": [ + "disabled" + ] }, "fromEnvironment": { "x-embedding-api-key": "huggingface_KEY", @@ -112,7 +116,9 @@ { "meta": { "name": "mistral-vectorize", - "tags": [] + "tags": [ + "disabled" + ] }, "fromEnvironment": { "x-embedding-api-key": "mistral_KEY", @@ -134,7 +140,9 @@ { "meta": { "name": "upstageAI-vectorize", - "tags": [] + "tags": [ + "disabled" + ] }, "fromEnvironment": { "x-embedding-api-key": "upstageAI_KEY", @@ -156,7 +164,9 @@ { "meta": { "name": "nvidia-vectorize", - "tags": [] + "tags": [ + "disabled" + ] }, "fromEnvironment": { "x-embedding-api-key": "Token", From 2507b6836f27d7bdb7399131fa768facfdad3b90 Mon Sep 17 00:00:00 2001 From: Aaron Morton Date: Mon, 2 Mar 2026 14:44:05 +1300 Subject: [PATCH 12/89] missinh files --- .../assertions/AssertionFactory.java | 126 +++++++++++++++++ .../assertions/AssertionFactoryRegistry.java | 51 +++++++ .../vectorize/assertions/AssertionName.java | 44 ++++++ .../api/v1/vectorize/assertions/Http.java | 1 - .../v1/vectorize/assertions/Templated.java | 35 +++++ .../testspec/AssertionTemplateSpec.java | 21 +++ .../api/v1/vectorize/testspec/SpecFile.java | 22 +++ .../api/v1/vectorize/testspec/SpecFiles.java | 132 ++++++++++++++++++ .../api/v1/vectorize/testspec/TestSpec.java | 15 ++ .../v1/vectorize/testspec/TestSpecKind.java | 14 ++ .../v1/vectorize/testspec/TestSpecMeta.java | 10 ++ .../api/v1/vectorize/testspec/TestSuite.java | 90 ++++++++++++ .../api/v1/vectorize/testspec/Workflow.java | 34 +++++ 13 files changed, 594 insertions(+), 1 deletion(-) create mode 100644 src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/AssertionFactory.java create mode 100644 src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/AssertionFactoryRegistry.java create mode 100644 src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/AssertionName.java create mode 100644 src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Templated.java create mode 100644 src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/AssertionTemplateSpec.java create mode 100644 src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/SpecFile.java create mode 100644 src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/SpecFiles.java create mode 100644 src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/TestSpec.java create mode 100644 src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/TestSpecKind.java create mode 100644 src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/TestSpecMeta.java create mode 100644 src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/TestSuite.java create mode 100644 src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/Workflow.java diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/AssertionFactory.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/AssertionFactory.java new file mode 100644 index 0000000000..08ce531266 --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/AssertionFactory.java @@ -0,0 +1,126 @@ +package io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions; + +import com.fasterxml.jackson.databind.JsonNode; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.TestCommand; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.TestPlan; + +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.lang.reflect.Modifier; +import java.util.List; + +/** + + */ + +public sealed interface AssertionFactory { + + public static final AssertionFactoryRegistry REGISTRY = new AssertionFactoryRegistry(); + + @FunctionalInterface + non-sealed interface AssertionMatcherFactory extends AssertionFactory { + AssertionMatcher create(TestCommand testCommand, JsonNode args); + } + + @FunctionalInterface + non-sealed interface TemplatedAssertionFactory extends AssertionFactory { + List create(TestPlan testPlan, JsonNode template, TestCommand testCommand, JsonNode args); + } + + static boolean isValidFactoryMethod(Method method) { + + if (!Modifier.isStatic(method.getModifiers())) { + return false; + } + + // NOTE: not checked the type of the list, lazy + if (List.class.isAssignableFrom(method.getReturnType())) { + var p = method.getParameterTypes(); + return p.length == 4 + && p[0] == TestPlan.class + && p[1] == JsonNode.class + && p[2] == TestCommand.class + && p[3] == JsonNode.class; + } + + if (method.getReturnType() == AssertionMatcher.class) { + var p = method.getParameterTypes(); + return p.length == 2 + && p[0] == TestCommand.class + && p[1] == JsonNode.class; + } + return false; + } + + abstract sealed class WrappedMethod + permits WrappedAssertionMatcherFactory, WrappedTemplatedAssertionFactory { + + private final Class clazz; + private final Method method; + private final AssertionName assertionName; + + protected WrappedMethod(Class clazz, Method method) { + this.clazz = clazz; + this.method = method; + this.assertionName = new AssertionName(clazz.getSimpleName(), method.getName()); + } + + static WrappedMethod of(Class clazz, Method method) { + return (method.getReturnType() == AssertionMatcher.class) ? + new WrappedAssertionMatcherFactory(clazz, method) + : + new WrappedTemplatedAssertionFactory(clazz, method); + } + + public Class clazz() { + return clazz; + } + + public Method method() { + return method; + } + + public String properName() { + return AssertionName.properName(method); + } + + public AssertionName assertionName() { + return assertionName; + } + + @SuppressWarnings("unchecked") + protected T invoke(Object... args) { + try { + return (T) method.invoke(null, args); + } catch (IllegalAccessException | InvocationTargetException e) { + throw new RuntimeException(e); + } + } + } + + final class WrappedAssertionMatcherFactory extends WrappedMethod + implements AssertionMatcherFactory { + + WrappedAssertionMatcherFactory(Class clazz, Method method) { + super(clazz, method); + } + + @Override + public AssertionMatcher create(TestCommand testCommand, JsonNode args) { + return invoke(testCommand, args); + } + } + + final class WrappedTemplatedAssertionFactory extends WrappedMethod + implements TemplatedAssertionFactory { + + WrappedTemplatedAssertionFactory(Class clazz, Method method) { + super(clazz, method); + } + + @Override + public List create(TestPlan testPlan, JsonNode template, TestCommand testCommand, JsonNode args) { + return invoke(testPlan, template, testCommand, args); + } + } +} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/AssertionFactoryRegistry.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/AssertionFactoryRegistry.java new file mode 100644 index 0000000000..fb51179491 --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/AssertionFactoryRegistry.java @@ -0,0 +1,51 @@ +package io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions; + +import com.fasterxml.jackson.databind.JsonNode; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.TestCommand; + +import java.lang.reflect.Method; +import java.lang.reflect.Modifier; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +public class AssertionFactoryRegistry { + + private final Map factoryMethods = new ConcurrentHashMap<>(); + + + public void register(Class cls) { + for (var method : cls.getMethods()) { + if ( AssertionFactory.isValidFactoryMethod(method)) { + var wrapped = AssertionFactory.WrappedMethod.of(cls, method); + factoryMethods.put(wrapped.assertionName(), wrapped); + } + } + } + + public AssertionFactory.WrappedMethod getWrapped(String fullKey) { + var normalisedName = AssertionName.from(fullKey); + + var factoryMethod = factoryMethods.get(normalisedName); + if (factoryMethod == null) { + loadClassFor(normalisedName); + } + + factoryMethod = factoryMethods.get(normalisedName); + if (factoryMethod == null) { + throw new IllegalArgumentException("Unknown assertion factory. (normalised)name: %s known=%s".formatted(normalisedName, factoryMethods.keySet())); + } + return factoryMethod; + } + + private void loadClassFor(AssertionName normalisedName) { + + try { + Class.forName(normalisedName.properClassName()); + // class static initializer should call register() + } catch (ClassNotFoundException e) { + throw new IllegalArgumentException("Unknown assertion factory. normalisedName=%s, properClassName()=%s".formatted(normalisedName, normalisedName.properClassName()), e); + } + } + +} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/AssertionName.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/AssertionName.java new file mode 100644 index 0000000000..96ec5b54b7 --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/AssertionName.java @@ -0,0 +1,44 @@ +package io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions; + +import java.lang.reflect.Method; + +public record AssertionName(String typeName, String funcName) { + + private static final String PACKAGE = + "io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions"; + + public AssertionName{ + typeName = typeName.toLowerCase(); + funcName = funcName.toLowerCase(); + } + + public static AssertionName from(String fullKey) { + + int pos = fullKey.indexOf('.'); + if (pos < 0) { + throw new IllegalArgumentException("fullKey must have a dot: " + fullKey); + } + var type = fullKey.substring(0, pos); + var func = fullKey.substring(pos + 1); + return new AssertionName(type, func); + } + + public static String properName(Class clazz, Method method ) { + return clazz.getSimpleName() + '.' + method.getName(); + } + + public static String properName(Method method ) { + return method.getName(); + } + + + public String properClassName() { + return PACKAGE + "." + + Character.toUpperCase(typeName.charAt(0)) + + typeName.substring(1); + } + + public String normalisedKey() { + return typeName + "." + funcName; + } +} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Http.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Http.java index f1afc680dd..387aa05b39 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Http.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Http.java @@ -14,6 +14,5 @@ public class Http { public static AssertionMatcher success(TestCommand testCommand, JsonNode args){ return described("http status is groovy", apiResponse -> apiResponse.validatable().statusCode(HttpStatus.SC_OK)); -// return apiResponse -> apiResponse.validatable().statusCode(HttpStatus.SC_OK); } } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Templated.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Templated.java new file mode 100644 index 0000000000..4c14df897c --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Templated.java @@ -0,0 +1,35 @@ +package io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.TestCommand; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.TestPlan; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.AssertionTemplateSpec; + +import java.util.List; + +public class Templated { + + static { + AssertionFactory.REGISTRY.register(Templated.class); + } + + public static List isSuccess(TestPlan testPlan, JsonNode template, TestCommand testCommand, JsonNode args){ + var commandTemplate = template.get(testCommand.commandName().getApiName()); + if (commandTemplate == null) { + throw new IllegalArgumentException( + "isSuccess Assertion template not found for command: " + + testCommand.commandName().getApiName()); + } + return runTemplate(testPlan, (ObjectNode) commandTemplate, testCommand, args); + } + + + private static List runTemplate(TestPlan testPlan, ObjectNode template, TestCommand testCommand, JsonNode args) { + return template.properties().stream() + .map(entry -> new TestAssertion.AssertionDefinition(entry.getKey(), entry.getValue())) + .map(def -> TestAssertion.buildAssertion(testPlan, testCommand, def)) + .toList(); + } + +} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/AssertionTemplateSpec.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/AssertionTemplateSpec.java new file mode 100644 index 0000000000..a67c679468 --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/AssertionTemplateSpec.java @@ -0,0 +1,21 @@ +package io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import io.quarkus.smallrye.health.runtime.SmallRyeIndividualHealthGroupHandler; +import io.stargate.sgv2.jsonapi.api.v1.util.scenarios.ThreeClusteringKeysTableScenario; +import io.stargate.sgv2.jsonapi.service.schema.collections.CqlColumnMatcher; + +import java.util.Map; +import java.util.Optional; + +public record AssertionTemplateSpec( + TestSpecMeta meta, + Map templates +) implements TestSpec { + + public Optional templateFor(String name){ + + return Optional.ofNullable(templates.get(name)); + } +} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/SpecFile.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/SpecFile.java new file mode 100644 index 0000000000..a3ddf724d7 --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/SpecFile.java @@ -0,0 +1,22 @@ +package io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec; + +import com.fasterxml.jackson.databind.JsonNode; +import org.jspecify.annotations.NonNull; + +import java.io.File; + +public record SpecFile( + File file, + TestSpec spec, + JsonNode root) { + + @Override + public @NonNull String toString() { + return new StringBuilder("SpecFile{") + .append("file=") + .append(file) + .append("spec.meta=") + .append(spec.meta()) + .toString(); + } +} \ No newline at end of file diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/SpecFiles.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/SpecFiles.java new file mode 100644 index 0000000000..a526b34914 --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/SpecFiles.java @@ -0,0 +1,132 @@ +package io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec; + +import com.fasterxml.jackson.databind.MapperFeature; +import com.fasterxml.jackson.databind.ObjectMapper; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.net.URISyntaxException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.List; +import java.util.function.Predicate; +import java.util.stream.Stream; + +/** + * Collection of all the test spec files read for this execution. + */ +public class SpecFiles { + + private static final ObjectMapper MAPPER = new ObjectMapper() + .configure(MapperFeature.ACCEPT_CASE_INSENSITIVE_ENUMS, true); + + private final List specFiles; + + private SpecFiles(List specFiles) { + this.specFiles = specFiles; + + for (SpecFile file : specFiles) { + if (file.spec() instanceof TestSuite it){ + it.expand(this); + } + } + } + + public static SpecFiles loadAll(List paths) { + + var specFiles = resourceDirs(paths) + .flatMap(SpecFiles::loadAll) + .toList(); + return new SpecFiles(specFiles); + } + + public Stream byKind(TestSpecKind kind) { + return specFiles.stream() + .filter(itFile -> itFile.spec().meta().kind() == kind); + } + + public Stream byType(Class clazz) { + return byKind(TestSpecKind.fromType(clazz)) + .map(specFile -> specFile.spec().asSpecType(clazz)); + } + + public Stream byName(TestSpecKind kind, String name) { + return match(kind, specFiles -> specFiles.meta().name().equals(name)); + } + + public Stream byNameAsType(Class clazz, String name) { + return match(TestSpecKind.fromType(clazz), specFiles -> specFiles.meta().name().equals(name)) + .map(specFile -> specFile.spec().asSpecType(clazz)); + } + + + private Stream match(TestSpecKind kind, Predicate predicate) { + return byKind(kind) + .filter(specFile -> predicate.test(specFile.spec())); + } + + private static Stream loadAll(Path path) { + + try (Stream pathStream = Files.walk(path)) { + return pathStream.filter(Files::isRegularFile) + .filter(SpecFiles::isJsonFile) + .map(SpecFiles::loadOne) + .toList() // force so the files are read before closing + .stream(); + } catch (IOException e) { + throw new UncheckedIOException("Failed reading test resources under: " + path , e); + } + } + + private static SpecFile loadOne(Path path) { + var file = path.toFile(); + try { + var root = MAPPER.readTree(file); + + var kindNode = root.path("meta").path("kind"); + if (!kindNode.isTextual()) { + throw new IllegalArgumentException("Missing/invalid meta.kind in " + file); + } + + var element = + switch (TestSpecKind.valueOf(kindNode.asText().toUpperCase())) { + case ASSERTION_TEMPLATE -> MAPPER.treeToValue(root, AssertionTemplateSpec.class); + case TEST_SUITE -> MAPPER.treeToValue(root, TestSuite.class); + case WORKFLOW -> MAPPER.treeToValue(root, Workflow.class); + + }; + return new SpecFile(file, element, root); + } catch (IOException e) { + throw new UncheckedIOException("Failed parsing JSON file: " + file, e); + } + } + + private static boolean isJsonFile(Path file) { + return file.getFileName().toString().endsWith(".json"); + } + + private static Stream resourceDirs(List paths) { + + var cl = Thread.currentThread().getContextClassLoader(); + + return paths.stream().map( + path -> { + String normalized = path.startsWith("/") ? path.substring(1) : path; + + var url = cl.getResource(normalized); + if (url == null) { + throw new IllegalArgumentException("Test resource folder not found: " + path); + } + + try { + // Works for file: URLs; if you run tests from a jar, switch to getResourceAsStream-based + // walking. + return Paths.get(url.toURI()); + } catch (URISyntaxException e) { + throw new IllegalArgumentException("Bad resource URI for: " + path + " -> " + url, e); + } + } + ); + } +} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/TestSpec.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/TestSpec.java new file mode 100644 index 0000000000..f1871db67b --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/TestSpec.java @@ -0,0 +1,15 @@ +package io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec; + +public sealed interface TestSpec permits Workflow, TestSuite, AssertionTemplateSpec { + + TestSpecMeta meta(); + + default T asSpecType(Class type) { + + if (!type.isInstance(this)) { + throw new IllegalArgumentException( + "TestSpec is not of required type. expected=%s, spec.meta=%s".formatted(type, meta())); + } + return type.cast(this); + } +} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/TestSpecKind.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/TestSpecKind.java new file mode 100644 index 0000000000..f1a8a28171 --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/TestSpecKind.java @@ -0,0 +1,14 @@ +package io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec; + +public enum TestSpecKind { + ASSERTION_TEMPLATE, + TEST_SUITE, + WORKFLOW; + + public static TestSpecKind fromType(Class clazz) { + if (clazz == AssertionTemplateSpec.class) { return ASSERTION_TEMPLATE; } + if (clazz == TestSuite.class) { return TEST_SUITE; } + if (clazz == Workflow.class) { return WORKFLOW; } + throw new IllegalArgumentException("Unknown TestSpec type: " + clazz); + } +} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/TestSpecMeta.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/TestSpecMeta.java new file mode 100644 index 0000000000..d42034dbe2 --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/TestSpecMeta.java @@ -0,0 +1,10 @@ +package io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec; + +import java.util.List; + +public record TestSpecMeta(String name, + TestSpecKind kind, + List tags) { + + +} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/TestSuite.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/TestSuite.java new file mode 100644 index 0000000000..7f67c6786a --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/TestSuite.java @@ -0,0 +1,90 @@ +package io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec; + +import io.stargate.sgv2.jsonapi.api.v1.vectorize.*; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions.TestAssertion; +import org.junit.jupiter.api.DynamicContainer; +import org.junit.jupiter.api.DynamicNode; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +import static org.junit.jupiter.api.DynamicContainer.dynamicContainer; + +public record TestSuite(TestSpecMeta meta, List setup, List tests, List cleanup) + implements TestSpec { + + + public DynamicContainer testNode(TestPlan testPlan, List allEnvs) { + + var desc = "TestSuite: %s ".formatted( + meta.name()); + + return dynamicContainer( + desc, + allEnvs.stream() + .map(testEnv -> testEnv.testNode(testPlan, this)) + ); + } + + public Collection testNodesForEnvironment(TestPlan testPlan, TestEnvironment testEnvironment) { + + List nodes = new ArrayList<>(); + + int i = 1; + for (TestCommand setupCommand : setup()) { + var setupRequest = new TestRequest( + "SetupRequest[%s]: %s".formatted(i++, setupCommand.commandName()), + setupCommand, testPlan.target(), testEnvironment, TestAssertion.forSuccess(testPlan, setupCommand)); + + nodes.add(setupRequest.testNodes()); + } + + for (var testCase : tests()) { + nodes.add(testCase.testNodesForEnvironment(testPlan, testEnvironment)); + } + + for (TestCommand cleanupCommand : cleanup()) { + var cleanupRequest = new TestRequest( + "CleanupRequest[%s]: %s".formatted(i++, cleanupCommand.commandName()), + cleanupCommand, testPlan.target(), testEnvironment, TestAssertion.forSuccess(testPlan,cleanupCommand)); + nodes.add(cleanupRequest.testNodes()); + } + + return nodes; + + } + + public void expand(SpecFiles specFiles) { + + List expandedSetup = new ArrayList<>(); + for (TestCommand command : setup) { + if (command.includeFrom() != null) { + var includedTest = specFiles.byNameAsType(TestSuite.class, command.includeFrom()) + .findFirst() + .orElseThrow(() -> new IllegalStateException("Included TestSuite Setup not found. parent=%s, included=%s".formatted(meta().name(), command.includeFrom()))); + + expandedSetup.addAll(includedTest.setup()); + } else { + expandedSetup.add(command); + } + } + setup.clear(); + setup.addAll(expandedSetup); + + List expandedTests = new ArrayList<>(); + for (TestCase testCase : tests) { + if (testCase.include() != null) { + var includedTest = specFiles.byNameAsType(TestSuite.class, testCase.include()) + .findFirst() + .orElseThrow(() -> new IllegalStateException("Included TestSuite TestCase not found. parent=%s, included=%s".formatted(meta().name(), testCase.include()))); + + expandedTests.addAll(includedTest.tests()); + } else { + expandedTests.add(testCase); + } + } + tests.clear(); + tests.addAll(expandedTests); + } +} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/Workflow.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/Workflow.java new file mode 100644 index 0000000000..c9b9d3da24 --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/Workflow.java @@ -0,0 +1,34 @@ +package io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec; + +import io.stargate.sgv2.jsonapi.api.v1.vectorize.Job; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.TestPlan; +import org.junit.jupiter.api.DynamicContainer; + +import java.util.List; +import java.util.stream.Stream; + +import static org.junit.jupiter.api.DynamicContainer.dynamicContainer; + +public record Workflow(TestSpecMeta meta, List jobs) implements TestSpec { + + + public Stream activeJobs(){ + return jobs().stream() + .filter(job -> !job.meta().tags().contains("disabled")); + } + + + public DynamicContainer testNode(TestPlan testPlan) { + + var desc = "Workflow: %s ".formatted( + meta.name()); + + var jobNodes = activeJobs() + .map(job -> job.testNode(testPlan)); + + return dynamicContainer( + desc, + testPlan.addLifecycle(this, jobNodes)); + } + +} From ac60249857aa78a7e78989852fc738305ca9ba1e Mon Sep 17 00:00:00 2001 From: Aaron Morton Date: Tue, 3 Mar 2026 13:07:03 +1300 Subject: [PATCH 13/89] WIP - begin added TestURI --- .../api/v1/vectorize/AstraBackend.java | 14 -- .../jsonapi/api/v1/vectorize/Backend.java | 38 ----- .../sgv2/jsonapi/api/v1/vectorize/Job.java | 10 +- .../sgv2/jsonapi/api/v1/vectorize/Target.java | 39 +++-- .../jsonapi/api/v1/vectorize/TestCase.java | 5 +- .../api/v1/vectorize/TestEnvironment.java | 11 +- .../jsonapi/api/v1/vectorize/TestPlan.java | 42 +++-- .../jsonapi/api/v1/vectorize/TestRequest.java | 22 ++- .../assertions/SingleTestAssertion.java | 4 +- .../vectorize/assertions/TestAssertion.java | 3 +- .../assertions/TestAssertionContainer.java | 6 +- .../v1/vectorize/backends/AstraBackend.java | 12 ++ .../api/v1/vectorize/backends/Backend.java | 12 ++ .../{ => backends}/CassandraBackend.java | 16 +- .../lifecycle/TestPlanLifecycle.java | 36 ++++ .../api/v1/vectorize/testspec/TestSuite.java | 19 ++- .../api/v1/vectorize/testspec/TestUri.java | 159 ++++++++++++++++++ .../api/v1/vectorize/testspec/Workflow.java | 8 +- .../vectorize/vectorize-workflow.json | 3 +- 19 files changed, 343 insertions(+), 116 deletions(-) delete mode 100644 src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/AstraBackend.java delete mode 100644 src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/Backend.java create mode 100644 src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/backends/AstraBackend.java create mode 100644 src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/backends/Backend.java rename src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/{ => backends}/CassandraBackend.java (72%) create mode 100644 src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/lifecycle/TestPlanLifecycle.java create mode 100644 src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/TestUri.java diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/AstraBackend.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/AstraBackend.java deleted file mode 100644 index e7296f0e4d..0000000000 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/AstraBackend.java +++ /dev/null @@ -1,14 +0,0 @@ -package io.stargate.sgv2.jsonapi.api.v1.vectorize; - -import org.apache.commons.lang3.RandomStringUtils; - -import static io.stargate.sgv2.jsonapi.api.v1.vectorize.TestEnvironment.toSafeSchemaIdentifier; - -public class AstraBackend extends Backend { - - @Override - public void updateJobForTarget(Job job) { - - job.variables().put("KEYSPACE_NAME", "default_keyspace"); - } -} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/Backend.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/Backend.java deleted file mode 100644 index 6e6a147e69..0000000000 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/Backend.java +++ /dev/null @@ -1,38 +0,0 @@ -package io.stargate.sgv2.jsonapi.api.v1.vectorize; - -import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.TestSuite; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.Workflow; -import org.junit.jupiter.api.DynamicNode; - -import java.util.Optional; - -public abstract class Backend { - - public void updateJobForTarget(Job job){ - } - - public Optional beforeWorkflow(TestPlan testPlan, Workflow workflow) { - return Optional.empty(); - } - - public Optional afterWorkflow(TestPlan testPlan, Workflow workflow) { - return Optional.empty(); - } - - public Optional beforeJob(TestPlan testPlan, Job job) { - return Optional.empty(); - } - - public Optional afterJob(TestPlan testPlan, Job job) { - return Optional.empty(); - } - - public Optional beforeTestSuite(TestPlan testPlan, TestSuite test, TestEnvironment env) { - return Optional.empty(); - } - - public Optional afterTestSuite(TestPlan testPlan, TestSuite test, TestEnvironment env) { - return Optional.empty(); - } -} - diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/Job.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/Job.java index 93935df839..2a7b7b92c2 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/Job.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/Job.java @@ -3,6 +3,7 @@ import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.TestSpecKind; import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.TestSpecMeta; import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.TestSuite; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.TestUri; import org.junit.jupiter.api.DynamicContainer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -25,7 +26,9 @@ public record Job( private static final Logger LOGGER = LoggerFactory.getLogger(Job.class); - public DynamicContainer testNode(TestPlan testPlan) { + public DynamicContainer testNode(TestPlan testPlan, TestUri.Builder uriBuilder) { + + uriBuilder.addSegment(TestUri.Segment.JOB, meta.name()); var desc = "Job: %s ".formatted( meta.name()); @@ -33,11 +36,12 @@ public DynamicContainer testNode(TestPlan testPlan) { testPlan.updateJobForTarget(this); var allEnvs = allEnvironments(testPlan); var testSuiteNodes = testSuites(testPlan) - .map(testSuite -> testSuite.testNode(testPlan, allEnvs)); + .map(testSuite -> testSuite.testNode(testPlan, uriBuilder.clone(), allEnvs)); return dynamicContainer( desc, - testPlan.addLifecycle(this, testSuiteNodes)); + uriBuilder.build().uri(), + testPlan.addLifecycle(uriBuilder.clone(),this, testSuiteNodes)); } public Stream testSuites(TestPlan testPlan) { diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/Target.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/Target.java index 8aa2e50a05..d7460153cb 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/Target.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/Target.java @@ -1,16 +1,21 @@ package io.stargate.sgv2.jsonapi.api.v1.vectorize; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.backends.AstraBackend; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.backends.Backend; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.backends.CassandraBackend; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.lifecycle.TestPlanLifecycle; import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.TestSuite; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.TestUri; import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.Workflow; import org.junit.jupiter.api.DynamicNode; import java.util.HashMap; import java.util.Optional; -public class Target { +public class Target implements TestPlanLifecycle { private final TargetConfiguration targetConfiguration; - private final Backend backend; + private final Backend backend; private final TestEnvironment env; public Target(TargetConfiguration targetConfiguration) { @@ -40,24 +45,30 @@ public APIRequest apiRequest(TestCommand testCommand, TestEnvironment env){ return new APIRequest(targetConfiguration.connection(), env, testCommand.withEnvironment(env)); } - public Optional beforeWorkflow(TestPlan testPlan, Workflow workflow){ - return backend.beforeWorkflow(testPlan, workflow); + @Override + public Optional beforeWorkflow(TestPlan testPlan, TestUri.Builder uriBuilder, Workflow workflow){ + return backend.beforeWorkflow(testPlan, uriBuilder,workflow); } - public Optional afterWorkflow(TestPlan testPlan,Workflow workflow){ - return backend.afterWorkflow(testPlan, workflow); + @Override + public Optional afterWorkflow(TestPlan testPlan, TestUri.Builder uriBuilder,Workflow workflow){ + return backend.afterWorkflow(testPlan, uriBuilder,workflow); } - public Optional beforeJob(TestPlan testPlan,Job job){ - return backend.beforeJob(testPlan, job); + @Override + public Optional beforeJob(TestPlan testPlan, TestUri.Builder uriBuilder,Job job){ + return backend.beforeJob(testPlan, uriBuilder,job); } - public Optional afterJob(TestPlan testPlan,Job job){ - return backend.afterJob(testPlan, job); + @Override + public Optional afterJob(TestPlan testPlan,TestUri.Builder uriBuilder,Job job){ + return backend.afterJob(testPlan,uriBuilder, job); } - public Optional beforeTestSuite(TestPlan testPlan, TestSuite test, TestEnvironment env){ - return backend.beforeTestSuite(testPlan, test, env); + @Override + public Optional beforeTestSuite(TestPlan testPlan, TestUri.Builder uriBuilder,TestSuite test, TestEnvironment env){ + return backend.beforeTestSuite(testPlan,uriBuilder, test, env); } - public Optional afterTestSuite(TestPlan testPlan,TestSuite test, TestEnvironment env){ - return backend.afterTestSuite(testPlan, test, env); + @Override + public Optional afterTestSuite(TestPlan testPlan, TestUri.Builder uriBuilder,TestSuite test, TestEnvironment env){ + return backend.afterTestSuite(testPlan, uriBuilder,test, env); } } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestCase.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestCase.java index cb49caa5c5..6e03d965ab 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestCase.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestCase.java @@ -4,6 +4,7 @@ import com.fasterxml.jackson.databind.node.ObjectNode; import io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions.AssertionMatcher; import io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions.TestAssertion; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.TestUri; import org.junit.jupiter.api.DynamicContainer; public record TestCase( @@ -15,13 +16,13 @@ public record TestCase( String include) { - public DynamicContainer testNodesForEnvironment(TestPlan testPlan, TestEnvironment testEnvironment) { + public DynamicContainer testNodesForEnvironment(TestPlan testPlan, TestUri.Builder uriBuilder, TestEnvironment testEnvironment) { var testRequest = new TestRequest( "TestCase: name=%s".formatted(name, command.commandName()), command(), testPlan.target(), testEnvironment, TestAssertion.buildAssertions(testPlan, this)); - return testRequest.testNodes(); + return testRequest.testNodes(uriBuilder); } } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestEnvironment.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestEnvironment.java index 9f0e0bbdf2..327f6dbc7f 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestEnvironment.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestEnvironment.java @@ -1,6 +1,7 @@ package io.stargate.sgv2.jsonapi.api.v1.vectorize; import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.TestSuite; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.TestUri; import org.apache.commons.text.StringSubstitutor; import org.apache.commons.text.lookup.StringLookupFactory; import org.junit.jupiter.api.DynamicContainer; @@ -31,15 +32,17 @@ public TestEnvironment(Map vars) { this.vars.putAll(vars); } - public DynamicContainer testNode(TestPlan testPlan, TestSuite testSuite) { + public DynamicContainer testNode(TestPlan testPlan, TestUri.Builder uriBuilder, TestSuite testSuite) { - var desc = "TestEnv: %s ".formatted(description()); + var d = description(); + uriBuilder.addSegment(TestUri.Segment.ENV, d); + var desc = "TestEnv: %s ".formatted(d); - var envNodes = testSuite.testNodesForEnvironment(testPlan, this).stream(); + var envNodes = testSuite.testNodesForEnvironment(testPlan, uriBuilder.clone(), this).stream(); return DynamicContainer.dynamicContainer( desc, - testPlan.addLifecycle(testSuite, this, envNodes)); + testPlan.addLifecycle(uriBuilder.clone(), testSuite, this, envNodes)); } private String description(){ diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestPlan.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestPlan.java index 146247776e..0b88e28a6e 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestPlan.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestPlan.java @@ -1,6 +1,7 @@ package io.stargate.sgv2.jsonapi.api.v1.vectorize; import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.*; +import org.eclipse.aether.util.artifact.OverlayArtifactTypeRegistry; import org.junit.jupiter.api.DynamicContainer; import org.junit.jupiter.api.DynamicNode; @@ -41,11 +42,15 @@ public Stream testNode() { workflows.isEmpty() ? "" : String.join(", ", workflows) ); + var uriBuilder = TestUri.builder(TestUri.Scheme.TESTRUN) + .addSegment(TestUri.Segment.TARGET, target.configuration().name()); + return Stream.of( dynamicContainer( desc, + uriBuilder.build().uri(), selectedWorkflows() - .map(workflow -> workflow.testNode(this)) + .map(workflow -> workflow.testNode(this, uriBuilder.clone())) ) ); } @@ -54,10 +59,10 @@ public void updateJobForTarget(Job job){ target.updateJobForTarget(job); } - private static Optional containerIfPresent(String namePrefix, TestSpecMeta meta, Optional dynamicNode){ + private static Optional containerIfPresent(TestUri.Builder uriBuilder, String namePrefix, TestSpecMeta meta, Optional dynamicNode){ return dynamicNode .map( - node -> dynamicContainer(namePrefix + ": " + meta.name(), Stream.of(node)) + node -> dynamicContainer(namePrefix + ": " + meta.name(), uriBuilder.build().uri(), Stream.of(node)) ); } @@ -65,35 +70,44 @@ private static Stream streamIfPresent(Optional co return container.stream().flatMap(Stream::of); } - private static Stream lifecycleNodes(String namePrefix, TestSpecMeta meta, Supplier> nodeSupplier){ + private static Stream lifecycleNodes(TestUri.Builder uriBuilder, String namePrefix, TestSpecMeta meta, Supplier> nodeSupplier){ var targetDynamicNode = nodeSupplier.get(); - return streamIfPresent(containerIfPresent(namePrefix, meta, targetDynamicNode)); + return streamIfPresent(containerIfPresent(uriBuilder, namePrefix, meta, targetDynamicNode)); } - public Stream addLifecycle(Workflow workflow, Stream dynamicNodes){ + public Stream addLifecycle(TestUri.Builder uriBuilder, Workflow workflow, Stream dynamicNodes){ + + var beforeUriBuilder = uriBuilder.clone().addSegment(TestUri.Segment.LIFECYCLE, "before-workflow"); + var afterUriBuilder = uriBuilder.clone().addSegment(TestUri.Segment.LIFECYCLE, "after-workflow"); return Stream.concat( - lifecycleNodes("Before Workflow", workflow.meta(), () -> target.beforeWorkflow(this, workflow)), + lifecycleNodes(beforeUriBuilder, "Before Workflow", workflow.meta(), () -> target.beforeWorkflow(this,beforeUriBuilder, workflow)), Stream.concat(dynamicNodes, - lifecycleNodes("After Workflow", workflow.meta(), () -> target.afterWorkflow(this, workflow))) + lifecycleNodes(afterUriBuilder, "After Workflow", workflow.meta(), () -> target.afterWorkflow(this, afterUriBuilder, workflow))) ); } - public Stream addLifecycle(Job job, Stream dynamicNodes){ + public Stream addLifecycle(TestUri.Builder uriBuilder, Job job, Stream dynamicNodes){ + + var beforeUriBuilder = uriBuilder.clone().addSegment(TestUri.Segment.LIFECYCLE, "before-job"); + var afterUriBuilder = uriBuilder.clone().addSegment(TestUri.Segment.LIFECYCLE, "after-job"); return Stream.concat( - lifecycleNodes("Before Job", job.meta(), () -> target.beforeJob(this, job)), + lifecycleNodes(beforeUriBuilder, "Before Job", job.meta(), () -> target.beforeJob(this,beforeUriBuilder, job)), Stream.concat(dynamicNodes, - lifecycleNodes("After Job", job.meta(), () -> target.afterJob(this, job))) + lifecycleNodes(afterUriBuilder, "After Job", job.meta(), () -> target.afterJob(this,afterUriBuilder, job))) ); } - public Stream addLifecycle(TestSuite testSuite, TestEnvironment environment, Stream dynamicNodes){ + public Stream addLifecycle(TestUri.Builder uriBuilder, TestSuite testSuite, TestEnvironment environment, Stream dynamicNodes){ + + var beforeUriBuilder = uriBuilder.clone().addSegment(TestUri.Segment.LIFECYCLE, "before-test-suite"); + var afterUriBuilder = uriBuilder.clone().addSegment(TestUri.Segment.LIFECYCLE, "after-test-suite"); return Stream.concat( - lifecycleNodes("Before TestSuite", testSuite.meta(), () -> target.beforeTestSuite(this, testSuite, environment)), + lifecycleNodes(beforeUriBuilder, "Before TestSuite", testSuite.meta(), () -> target.beforeTestSuite(this, beforeUriBuilder,testSuite, environment)), Stream.concat(dynamicNodes, - lifecycleNodes("After TestSuite", testSuite.meta(), () -> target.afterTestSuite(this, testSuite, environment))) + lifecycleNodes( afterUriBuilder, "After TestSuite", testSuite.meta(), () -> target.afterTestSuite(this, afterUriBuilder,testSuite, environment))) ); } } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestRequest.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestRequest.java index e888362095..8233b640af 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestRequest.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestRequest.java @@ -1,12 +1,14 @@ package io.stargate.sgv2.jsonapi.api.v1.vectorize; import io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions.TestAssertion; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.TestUri; import org.junit.jupiter.api.DynamicContainer; import org.junit.jupiter.api.DynamicNode; import java.util.ArrayList; import java.util.List; import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.Stream; import static org.junit.jupiter.api.DynamicContainer.dynamicContainer; import static org.junit.jupiter.api.DynamicTest.dynamicTest; @@ -24,24 +26,32 @@ public TestResponse execute() { return new TestResponse(this, apiRequest, apiRequest.execute()); } - public DynamicContainer testNodes() { + public DynamicContainer testNodes(TestUri.Builder uriBuilder) { + + uriBuilder.addSegment(TestUri.Segment.REQUEST, name()); + + Stream.Builder nodesBuilder = Stream.builder();; - List nodes = new ArrayList<>(); AtomicReference atomicResponse = new AtomicReference<>(); // Execute the request, and set so the assertions can pull the response after. - nodes.add(dynamicTest("Command: " + testCommand.commandName().getApiName(), () -> atomicResponse.set(execute()))); + var commandUriBuilder = uriBuilder.clone(); + commandUriBuilder.addSegment(TestUri.Segment.COMMAND, testCommand.commandName().getApiName()); + nodesBuilder.add(dynamicTest("Command: " + testCommand.commandName().getApiName(), + commandUriBuilder.build().uri(), + () -> atomicResponse.set(execute()))); // tests for each assertion + var assertionsUriBuilder = uriBuilder.clone().addSegment(TestUri.Segment.ASSERTION_CONTAINER, "assertions"); var assertionTests = testAssertions().stream().map( - testAssertion -> testAssertion.testNodes(atomicResponse)) + testAssertion -> testAssertion.testNodes(assertionsUriBuilder.clone(), atomicResponse)) .toList(); // if we have assertion tests, put them in a container if (!assertionTests.isEmpty()) { - nodes.add(dynamicContainer("Assertions", assertionTests)); + nodesBuilder.add(dynamicContainer("Assertions", assertionTests)); } - return dynamicContainer("Request: " + name, nodes); + return dynamicContainer("Request: " + name, uriBuilder.build().uri(), nodesBuilder.build()); } } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/SingleTestAssertion.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/SingleTestAssertion.java index 1cfd9ac457..387164c560 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/SingleTestAssertion.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/SingleTestAssertion.java @@ -2,6 +2,7 @@ import com.fasterxml.jackson.databind.JsonNode; import io.stargate.sgv2.jsonapi.api.v1.vectorize.TestResponse; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.TestUri; import io.stargate.sgv2.jsonapi.service.schema.tables.ApiSupportDef; import org.hamcrest.SelfDescribing; import org.hamcrest.StringDescription; @@ -34,8 +35,9 @@ public void run(TestResponse testResponse) { } @Override - public DynamicNode testNodes(AtomicReference testResponse) { + public DynamicNode testNodes(TestUri.Builder uriBuilder, AtomicReference testResponse) { + uriBuilder.addSegment(TestUri.Segment.ASSERTION, name()); var original = (matcher instanceof Describable d) ? d.describe() : diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/TestAssertion.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/TestAssertion.java index 887c1300a1..6b9c5ccb36 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/TestAssertion.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/TestAssertion.java @@ -7,6 +7,7 @@ import io.stargate.sgv2.jsonapi.api.v1.vectorize.TestResponse; import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.AssertionTemplateSpec; import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.SpecFiles; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.TestUri; import org.assertj.core.api.AssertFactory; import org.junit.jupiter.api.DynamicNode; @@ -24,7 +25,7 @@ public interface TestAssertion { void run(TestResponse testResponse); - DynamicNode testNodes(AtomicReference testResponse); + DynamicNode testNodes(TestUri.Builder uriBuilder, AtomicReference testResponse); static List forSuccess(TestPlan testPlan, TestCommand testCommand) { diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/TestAssertionContainer.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/TestAssertionContainer.java index acf447b355..1af420a1b9 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/TestAssertionContainer.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/TestAssertionContainer.java @@ -2,6 +2,7 @@ import com.fasterxml.jackson.databind.JsonNode; import io.stargate.sgv2.jsonapi.api.v1.vectorize.TestResponse; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.TestUri; import org.junit.jupiter.api.DynamicNode; import java.util.List; @@ -36,10 +37,11 @@ public void run(TestResponse testResponse) { } @Override - public DynamicNode testNodes(AtomicReference testResponse) { + public DynamicNode testNodes(TestUri.Builder uriBuilder, AtomicReference testResponse) { + uriBuilder.addSegment(TestUri.Segment.ASSERTION, name()); var childs = assertions.stream() - .map(assertion -> assertion.testNodes(testResponse)) + .map(assertion -> assertion.testNodes(uriBuilder.clone(), testResponse)) .toList(); return dynamicContainer(name, childs); diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/backends/AstraBackend.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/backends/AstraBackend.java new file mode 100644 index 0000000000..528fadf7bf --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/backends/AstraBackend.java @@ -0,0 +1,12 @@ +package io.stargate.sgv2.jsonapi.api.v1.vectorize.backends; + +import io.stargate.sgv2.jsonapi.api.v1.vectorize.Job; + +public class AstraBackend extends Backend { + + @Override + public void updateJobForTarget(Job job) { + + job.variables().put("KEYSPACE_NAME", "default_keyspace"); + } +} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/backends/Backend.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/backends/Backend.java new file mode 100644 index 0000000000..95acf200eb --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/backends/Backend.java @@ -0,0 +1,12 @@ +package io.stargate.sgv2.jsonapi.api.v1.vectorize.backends; + +import io.stargate.sgv2.jsonapi.api.v1.vectorize.Job; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.lifecycle.TestPlanLifecycle; + +public abstract class Backend implements TestPlanLifecycle { + + public void updateJobForTarget(Job job){ + } + +} + diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/CassandraBackend.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/backends/CassandraBackend.java similarity index 72% rename from src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/CassandraBackend.java rename to src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/backends/CassandraBackend.java index 916ff8582a..ac2585d4c2 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/CassandraBackend.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/backends/CassandraBackend.java @@ -1,8 +1,12 @@ -package io.stargate.sgv2.jsonapi.api.v1.vectorize; +package io.stargate.sgv2.jsonapi.api.v1.vectorize.backends; import com.fasterxml.jackson.databind.ObjectMapper; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions.AssertionMatcher; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.Job; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.TestCommand; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.TestPlan; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.TestRequest; import io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions.TestAssertion; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.TestUri; import org.apache.commons.lang3.RandomStringUtils; import org.junit.jupiter.api.DynamicNode; @@ -26,7 +30,7 @@ public void updateJobForTarget(Job job) { } @Override - public Optional beforeJob(TestPlan testPlan, Job job) { + public Optional beforeJob(TestPlan testPlan, TestUri.Builder uriBuilder, Job job) { var command = TestCommand.fromJson( """ @@ -42,11 +46,11 @@ public Optional beforeJob(TestPlan testPlan, Job job) { env.substitutor().replace("createKeyspace: ${KEYSPACE_NAME}"), command, testPlan.target(), env, TestAssertion.forSuccess( testPlan, command)); - return Optional.of(setupRequest.testNodes()); + return Optional.of(setupRequest.testNodes(uriBuilder)); } @Override - public Optional afterJob(TestPlan testPlan, Job job) { + public Optional afterJob(TestPlan testPlan, TestUri.Builder uriBuilder,Job job) { var command = TestCommand.fromJson( """ { @@ -61,6 +65,6 @@ public Optional afterJob(TestPlan testPlan, Job job) { env.substitutor().replace("dropKeyspace: ${KEYSPACE_NAME}"), command, testPlan.target(), job.withoutMatrix(testPlan), TestAssertion.forSuccess(testPlan,command)); - return Optional.of(setupRequest.testNodes()); + return Optional.of(setupRequest.testNodes(uriBuilder)); } } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/lifecycle/TestPlanLifecycle.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/lifecycle/TestPlanLifecycle.java new file mode 100644 index 0000000000..31eb1dee36 --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/lifecycle/TestPlanLifecycle.java @@ -0,0 +1,36 @@ +package io.stargate.sgv2.jsonapi.api.v1.vectorize.lifecycle; + +import io.stargate.sgv2.jsonapi.api.v1.vectorize.Job; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.TestEnvironment; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.TestPlan; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.TestSuite; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.TestUri; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.Workflow; +import org.junit.jupiter.api.DynamicNode; + +import java.util.Optional; + +public interface TestPlanLifecycle { + + default Optional beforeWorkflow(TestPlan testPlan, TestUri.Builder uriBuilder, Workflow workflow){ + return Optional.empty(); + } + default Optional afterWorkflow(TestPlan testPlan,TestUri.Builder uriBuilder, Workflow workflow){ + return Optional.empty(); + } + + default Optional beforeJob(TestPlan testPlan,TestUri.Builder uriBuilder, Job job){ + return Optional.empty(); + } + default Optional afterJob(TestPlan testPlan, TestUri.Builder uriBuilder, Job job){ + return Optional.empty(); + } + + default Optional beforeTestSuite(TestPlan testPlan, TestUri.Builder uriBuilder,TestSuite test, TestEnvironment env){ + return Optional.empty(); + } + default Optional afterTestSuite(TestPlan testPlan, TestUri.Builder uriBuilder,TestSuite test, TestEnvironment env){ + return Optional.empty(); + } + +} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/TestSuite.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/TestSuite.java index 7f67c6786a..b8e0da0c30 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/TestSuite.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/TestSuite.java @@ -15,40 +15,47 @@ public record TestSuite(TestSpecMeta meta, List setup, List allEnvs) { + public DynamicContainer testNode(TestPlan testPlan, TestUri.Builder uriBuilder, List allEnvs) { + + uriBuilder.addSegment(TestUri.Segment.SUITE, meta().name()); var desc = "TestSuite: %s ".formatted( meta.name()); return dynamicContainer( desc, + uriBuilder.build().uri(), allEnvs.stream() - .map(testEnv -> testEnv.testNode(testPlan, this)) + .map(testEnv -> testEnv.testNode(testPlan, uriBuilder.clone(), this)) ); } - public Collection testNodesForEnvironment(TestPlan testPlan, TestEnvironment testEnvironment) { + public Collection testNodesForEnvironment(TestPlan testPlan, TestUri.Builder uriBuilder, TestEnvironment testEnvironment) { List nodes = new ArrayList<>(); int i = 1; + var setupUriBuilder = uriBuilder.clone().addSegment(TestUri.Segment.STAGE, "setup"); + for (TestCommand setupCommand : setup()) { var setupRequest = new TestRequest( "SetupRequest[%s]: %s".formatted(i++, setupCommand.commandName()), setupCommand, testPlan.target(), testEnvironment, TestAssertion.forSuccess(testPlan, setupCommand)); - nodes.add(setupRequest.testNodes()); + nodes.add(setupRequest.testNodes(setupUriBuilder.clone())); } + var testUriBuilder = uriBuilder.clone().addSegment(TestUri.Segment.STAGE, "test"); for (var testCase : tests()) { - nodes.add(testCase.testNodesForEnvironment(testPlan, testEnvironment)); + nodes.add(testCase.testNodesForEnvironment(testPlan, testUriBuilder.clone(), testEnvironment)); } + var cleanupUriBuilder = uriBuilder.clone().addSegment(TestUri.Segment.STAGE, "cleanup"); for (TestCommand cleanupCommand : cleanup()) { var cleanupRequest = new TestRequest( "CleanupRequest[%s]: %s".formatted(i++, cleanupCommand.commandName()), cleanupCommand, testPlan.target(), testEnvironment, TestAssertion.forSuccess(testPlan,cleanupCommand)); - nodes.add(cleanupRequest.testNodes()); + nodes.add(cleanupRequest.testNodes(cleanupUriBuilder.clone())); } return nodes; diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/TestUri.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/TestUri.java new file mode 100644 index 0000000000..ffd68873ed --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/TestUri.java @@ -0,0 +1,159 @@ +package io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec; + +import io.stargate.sgv2.jsonapi.service.operation.builder.LiteralTerm; + +import java.net.URI; +import java.util.*; +import java.util.regex.Pattern; +import java.util.stream.Stream; + +import static java.util.stream.Collectors.joining; +import static java.util.stream.Collectors.toMap; + +public record TestUri( + Scheme scheme, + List segments) { + + public static TestUri.Builder builder(Scheme scheme){ + return new TestUri.Builder(scheme); + } + + public Segment leafType(){ + return segments.getLast().segment; + } + + public URI uri() { + var path = segments.stream() + .map(SegmentValue::toString) + .collect(joining("/")); + return URI.create(scheme.name() + "://" + path); + } + + public static TestUri parse(URI uri) { + + var builder = builder(Scheme.valueOf(uri.getScheme().toLowerCase())); + SegmentValue.parse(uri) + .forEach(builder::addSegment); + return builder.build(); + } + + public enum Scheme { + TESTPLAN, + TESTRUN; + + public String pathName() { + return name().toLowerCase(); + } + } + + public enum Segment { + TARGET(null), + LIFECYCLE(null), // used in multiple places + WORKFLOW(TARGET), + JOB(WORKFLOW), + SUITE(JOB), + ENV(SUITE), + STAGE(ENV), + REQUEST(STAGE), + COMMAND(REQUEST), + ASSERTION_CONTAINER(REQUEST), + ASSERTION(ASSERTION_CONTAINER); + + private final Segment parent; + + Segment(Segment parent) { + this.parent = parent; + } + + public Segment parent() { + return parent; + } + + public boolean isParentValid(Segment segment) { + // XXX TODO: needs work + return true; + //return (parent == null) || (parent == segment) || (segment == LIFECYCLE) ; + } + + public String pathName() { + return name().toLowerCase(); + } + } + + public record SegmentValue(Segment segment, String value) { + + private static final Pattern INVALID_CHARS = Pattern.compile("[^a-zA-Z0-9\\-_.~]"); + + public SegmentValue{ + Objects.requireNonNull(segment, "segment must not be null"); + Objects.requireNonNull(value, "value must not be null"); + } + + public static Stream parse(URI uri) { + return Arrays.stream(uri.getPath().split("/")) + .map(SegmentValue::parse); + } + + public static SegmentValue parse(String segmentKeyValue) { + var parts = segmentKeyValue.split("=", 2); + if (parts.length != 2) { + throw new IllegalArgumentException("Invalid segment, expected key=value format: " + segmentKeyValue); + } + try { + return new SegmentValue(Segment.valueOf(parts[0].toUpperCase()), parts[1]); + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException("Unknown segment key: " + parts[0]); + } + } + + @Override + public String toString() { + return segment.name() + "=" + INVALID_CHARS.matcher(value).replaceAll("_"); + } + } + + + public static class Builder { + + private final Scheme scheme; + private final List segmentValues; + + protected Builder(Scheme scheme) { + this(scheme, new ArrayList<>()); + } + + private Builder(Scheme scheme, List segmentValues) { + this.scheme = Objects.requireNonNull(scheme, "scheme must not be null"); + this.segmentValues = Objects.requireNonNull(segmentValues, "segmentValues must not be null"); + } + + public Builder addSegment(SegmentValue segmentValue) { + segmentValues.add(segmentValue); + return this; + } + + public Builder addSegment(Segment segment, String value) { + segmentValues.add(new SegmentValue(segment, value)); + return this; + } + + public Builder clone(){ + return new Builder(scheme, segmentValues); + } + + public TestUri build() { + for (int i = 0; i < segmentValues.size(); i++) { + var current = segmentValues.get(i); + var previous = i == 0 ? null : segmentValues.get(i - 1).segment(); + if (!current.segment().isParentValid(previous)) { + throw new IllegalArgumentException( + "Invalid segment order. segment=%s expected parent=%s but previous=%s" + .formatted(current.segment(), current.segment().parent(), previous)); + + } + } + return new TestUri(scheme, List.copyOf(segmentValues)); + } + } + +} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/Workflow.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/Workflow.java index c9b9d3da24..8732fd0610 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/Workflow.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/Workflow.java @@ -18,17 +18,19 @@ public Stream activeJobs(){ } - public DynamicContainer testNode(TestPlan testPlan) { + public DynamicContainer testNode(TestPlan testPlan, TestUri.Builder uriBuilder) { + uriBuilder.addSegment(TestUri.Segment.WORKFLOW, meta().name()); var desc = "Workflow: %s ".formatted( meta.name()); var jobNodes = activeJobs() - .map(job -> job.testNode(testPlan)); + .map(job -> job.testNode(testPlan, uriBuilder.clone())); return dynamicContainer( desc, - testPlan.addLifecycle(this, jobNodes)); + uriBuilder.build().uri(), + testPlan.addLifecycle(uriBuilder.clone(), this, jobNodes)); } } diff --git a/src/test/resources/integration-tests/vectorize/vectorize-workflow.json b/src/test/resources/integration-tests/vectorize/vectorize-workflow.json index 5cbfce0258..e08ff4c5e3 100644 --- a/src/test/resources/integration-tests/vectorize/vectorize-workflow.json +++ b/src/test/resources/integration-tests/vectorize/vectorize-workflow.json @@ -7,7 +7,7 @@ { "meta": { "name": "open-ai-vectorize", - "tags": [] + "tags": ["disable"] }, "fromEnvironment": { "x-embedding-api-key": "x_embedding_api_key", @@ -32,7 +32,6 @@ "meta": { "name": "voyageAI-vectorize", "tags": [ - "disabled" ] }, "fromEnvironment": { From 4c9c2399d52f6c1dae8bb8262b73743e64c99d0f Mon Sep 17 00:00:00 2001 From: Aaron Morton Date: Wed, 11 Mar 2026 10:44:26 +1300 Subject: [PATCH 14/89] WIP - refactor --- .../v1/vectorize/EnvironmentalRequest.java | 6 -- .../v1/vectorize/IntegrationJobRunner.java | 50 ---------------- .../jsonapi/api/v1/vectorize/RunnerBase.java | 15 ----- .../api/v1/vectorize/TargetConfiguration.java | 4 -- .../api/v1/vectorize/TestCaseResult.java | 17 ------ .../jsonapi/api/v1/vectorize/TestPlan.java | 26 ++++---- .../api/v1/vectorize/TestRequestRunner.java | 28 --------- .../api/v1/vectorize/TestSuiteRunner.java | 51 ---------------- .../api/v1/vectorize/VectorizeAstra.java | 2 +- .../api/v1/vectorize/VectorizeUnit.java | 60 ------------------- .../assertions/AssertionFactory.java | 2 +- .../assertions/AssertionFactoryRegistry.java | 6 -- .../assertions/AssertionMatcher.java | 2 +- .../vectorize/assertions/BodyAssertion.java | 2 +- .../DescribableAssertionMatcher.java | 2 +- .../v1/vectorize/assertions/Documents.java | 2 +- .../api/v1/vectorize/assertions/Http.java | 2 +- .../api/v1/vectorize/assertions/Response.java | 6 +- .../assertions/SingleTestAssertion.java | 46 ++++---------- .../api/v1/vectorize/assertions/Status.java | 2 +- .../v1/vectorize/assertions/Templated.java | 3 +- .../vectorize/assertions/TestAssertion.java | 15 ++--- .../assertions/TestAssertionContainer.java | 8 +-- .../lifecycle/TestPlanLifecycle.java | 18 +++--- .../vectorize/{ => messaging}/APIRequest.java | 9 ++- .../{ => messaging}/APIResponse.java | 2 +- .../{backends => targets}/AstraBackend.java | 4 +- .../{backends => targets}/Backend.java | 4 +- .../CassandraBackend.java | 18 +++--- .../vectorize/{ => targets}/Connection.java | 2 +- .../v1/vectorize/{ => targets}/Target.java | 35 ++++++----- .../testrun/DynamicTestExecutable.java | 60 +++++++++++++++++++ .../TestRunEnv.java} | 24 ++++---- .../TestRunRequest.java} | 29 ++++----- .../TestRunResponse.java} | 9 +-- .../{testspec => testrun}/TestUri.java | 7 +-- .../api/v1/vectorize/{ => testspec}/Job.java | 33 +++++----- .../api/v1/vectorize/testspec/SpecFiles.java | 7 ++- .../testspec/TargetConfiguration.java | 6 ++ .../TargetsSpec.java} | 17 ++++-- .../v1/vectorize/{ => testspec}/TestCase.java | 12 ++-- .../vectorize/{ => testspec}/TestCommand.java | 5 +- .../api/v1/vectorize/testspec/TestSpec.java | 2 +- .../v1/vectorize/testspec/TestSpecKind.java | 6 +- .../{TestSuite.java => TestSuiteSpec.java} | 17 +++--- .../{Workflow.java => WorkflowSpec.java} | 19 +++--- .../integration-tests/targets/targets.json | 6 +- .../vectorize/vectorize-workflow.json | 3 +- 48 files changed, 271 insertions(+), 440 deletions(-) delete mode 100644 src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/EnvironmentalRequest.java delete mode 100644 src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/IntegrationJobRunner.java delete mode 100644 src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/RunnerBase.java delete mode 100644 src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TargetConfiguration.java delete mode 100644 src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestCaseResult.java delete mode 100644 src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestRequestRunner.java delete mode 100644 src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestSuiteRunner.java delete mode 100644 src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/VectorizeUnit.java rename src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/{ => messaging}/APIRequest.java (87%) rename src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/{ => messaging}/APIResponse.java (73%) rename src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/{backends => targets}/AstraBackend.java (58%) rename src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/{backends => targets}/Backend.java (61%) rename src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/{backends => targets}/CassandraBackend.java (78%) rename src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/{ => targets}/Connection.java (82%) rename src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/{ => targets}/Target.java (65%) create mode 100644 src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testrun/DynamicTestExecutable.java rename src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/{TestEnvironment.java => testrun/TestRunEnv.java} (87%) rename src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/{TestRequest.java => testrun/TestRunRequest.java} (61%) rename src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/{TestResponse.java => testrun/TestRunResponse.java} (75%) rename src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/{testspec => testrun}/TestUri.java (95%) rename src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/{ => testspec}/Job.java (69%) create mode 100644 src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/TargetConfiguration.java rename src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/{TargetConfigurationss.java => testspec/TargetsSpec.java} (75%) rename src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/{ => testspec}/TestCase.java (55%) rename src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/{ => testspec}/TestCommand.java (95%) rename src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/{TestSuite.java => TestSuiteSpec.java} (80%) rename src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/{Workflow.java => WorkflowSpec.java} (58%) diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/EnvironmentalRequest.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/EnvironmentalRequest.java deleted file mode 100644 index 82587e992b..0000000000 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/EnvironmentalRequest.java +++ /dev/null @@ -1,6 +0,0 @@ -package io.stargate.sgv2.jsonapi.api.v1.vectorize; - -public interface EnvironmentalRequest { - - -} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/IntegrationJobRunner.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/IntegrationJobRunner.java deleted file mode 100644 index 12551cdd5a..0000000000 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/IntegrationJobRunner.java +++ /dev/null @@ -1,50 +0,0 @@ -package io.stargate.sgv2.jsonapi.api.v1.vectorize; - -import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.SpecFiles; - -public class IntegrationJobRunner { - - private final Target target; - private final SpecFiles itCollection; - private final Job job; - - public IntegrationJobRunner(Target target, SpecFiles itCollection, Job job) { - this.target = target; - this.itCollection = itCollection; - this.job = job; - } - -// public void run() { -// -// target.jobStarting(job); -// var allEnvs = job.allEnvironments(); -// -// List allTests = new ArrayList<>(); -// job.tests() -// .forEach( -// testName -> { -// allTests.addAll(itCollection.testByName(testName)); -// }); -// -// try{ -// -// for (TestSuite test : allTests) { -// for (TestEnvironment env : allEnvs) { -// -// try{ -// target.testStarting(test, env); -// var testRunner = new IntegrationTestRunner( itCollection, target, test, env); -// testRunner.run(); -// } -// finally{ -// target.testFinished(test, env); -// } -// -// } -// } -// } -// finally { -// target.jobFinished(job); -// } -// } -} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/RunnerBase.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/RunnerBase.java deleted file mode 100644 index ceb8576e45..0000000000 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/RunnerBase.java +++ /dev/null @@ -1,15 +0,0 @@ -package io.stargate.sgv2.jsonapi.api.v1.vectorize; - -import org.junit.jupiter.api.DynamicNode; -import org.junit.jupiter.api.DynamicTest; -import org.junit.jupiter.api.function.Executable; - -import java.util.Collection; -import java.util.List; - -import static org.junit.jupiter.api.DynamicTest.dynamicTest; - -public abstract class RunnerBase implements Executable { - - -} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TargetConfiguration.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TargetConfiguration.java deleted file mode 100644 index 58874c620d..0000000000 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TargetConfiguration.java +++ /dev/null @@ -1,4 +0,0 @@ -package io.stargate.sgv2.jsonapi.api.v1.vectorize; - -public record TargetConfiguration(String name, String backend, Connection connection) { -} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestCaseResult.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestCaseResult.java deleted file mode 100644 index eccc7429db..0000000000 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestCaseResult.java +++ /dev/null @@ -1,17 +0,0 @@ -package io.stargate.sgv2.jsonapi.api.v1.vectorize; - -import io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions.AssertionMatcher; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.TestSuite; - -public record TestCaseResult( - TestSuite integrationTest, - TestCase testCase, // Nullable - TestResponse testResponse, - AssertionError error, - AssertionMatcher failedAssertion -) { - - public boolean failed(){ - return error != null; - } -} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestPlan.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestPlan.java index 0b88e28a6e..1476fdf3d4 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestPlan.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestPlan.java @@ -1,7 +1,10 @@ package io.stargate.sgv2.jsonapi.api.v1.vectorize; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.targets.Target; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestRunEnv; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.Job; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestUri; import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.*; -import org.eclipse.aether.util.artifact.OverlayArtifactTypeRegistry; import org.junit.jupiter.api.DynamicContainer; import org.junit.jupiter.api.DynamicNode; @@ -12,26 +15,29 @@ import java.util.stream.Stream; import static org.junit.jupiter.api.DynamicContainer.dynamicContainer; -import static org.junit.jupiter.api.DynamicTest.dynamicTest; -public record TestPlan(Target target, SpecFiles specFiles, Set workflows){ +public record TestPlan(Target target, SpecFiles specFiles, Set workflows, boolean ignoreDisabled){ public static TestPlan create(String targetName, List workflows){ - var targetConfigs = TargetConfigurationss.loadAll("integration-tests/targets/targets.json"); + return create(targetName, workflows, true); + } + + public static TestPlan create(String targetName, List workflows, boolean ignoreDisabled){ + var targetConfigs = TargetsSpec.loadAll("integration-tests/targets/targets.json"); var target = new Target(targetConfigs.configuration(targetName)); var specFiles = SpecFiles.loadAll(List.of("integration-tests/vectorize", "integration-tests/assertions/assertion-templates.json")); - return new TestPlan(target, specFiles, Set.copyOf(workflows)); + return new TestPlan(target, specFiles, Set.copyOf(workflows), ignoreDisabled); } - public Stream selectedWorkflows(){ + public Stream selectedWorkflows(){ return specFiles.byKind(TestSpecKind.WORKFLOW) .filter(specFile -> workflows.isEmpty() || workflows.contains(specFile.spec().meta().name()) ) - .map(testSpec -> (Workflow) testSpec.spec()); + .map(testSpec -> (WorkflowSpec) testSpec.spec()); } public Stream testNode() { @@ -50,7 +56,7 @@ public Stream testNode() { desc, uriBuilder.build().uri(), selectedWorkflows() - .map(workflow -> workflow.testNode(this, uriBuilder.clone())) + .map(workflow -> workflow.testNode(this, uriBuilder.clone(), ignoreDisabled)) ) ); } @@ -75,7 +81,7 @@ private static Stream lifecycleNodes(TestUri.Builder uriBuilder, St return streamIfPresent(containerIfPresent(uriBuilder, namePrefix, meta, targetDynamicNode)); } - public Stream addLifecycle(TestUri.Builder uriBuilder, Workflow workflow, Stream dynamicNodes){ + public Stream addLifecycle(TestUri.Builder uriBuilder, WorkflowSpec workflow, Stream dynamicNodes){ var beforeUriBuilder = uriBuilder.clone().addSegment(TestUri.Segment.LIFECYCLE, "before-workflow"); var afterUriBuilder = uriBuilder.clone().addSegment(TestUri.Segment.LIFECYCLE, "after-workflow"); @@ -99,7 +105,7 @@ public Stream addLifecycle(TestUri.Builder uriBuilder, Jo ); } - public Stream addLifecycle(TestUri.Builder uriBuilder, TestSuite testSuite, TestEnvironment environment, Stream dynamicNodes){ + public Stream addLifecycle(TestUri.Builder uriBuilder, TestSuiteSpec testSuite, TestRunEnv environment, Stream dynamicNodes){ var beforeUriBuilder = uriBuilder.clone().addSegment(TestUri.Segment.LIFECYCLE, "before-test-suite"); var afterUriBuilder = uriBuilder.clone().addSegment(TestUri.Segment.LIFECYCLE, "after-test-suite"); diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestRequestRunner.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestRequestRunner.java deleted file mode 100644 index 294c2c11d0..0000000000 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestRequestRunner.java +++ /dev/null @@ -1,28 +0,0 @@ -package io.stargate.sgv2.jsonapi.api.v1.vectorize; - -import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.TestSuite; - -import java.util.Objects; - -import static org.junit.jupiter.api.DynamicTest.dynamicTest; - -public class TestRequestRunner extends RunnerBase{ - - private final TestRequest testRequest; - private final TestSuite testSuite; - private final TestCase testCase; - - public TestRequestRunner(TestRequest testRequest, TestSuite testSuite, TestCase testCase ) { - this.testRequest = Objects.requireNonNull(testRequest, "testRequest must not be null"); - - this.testSuite = testSuite; - this.testCase = testCase; - } - - @Override - public void execute() throws Throwable { - -// testRequest.execute().validate(testSuite, testCase); - } - -} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestSuiteRunner.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestSuiteRunner.java deleted file mode 100644 index b697b4e04c..0000000000 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestSuiteRunner.java +++ /dev/null @@ -1,51 +0,0 @@ -package io.stargate.sgv2.jsonapi.api.v1.vectorize; - -import com.fasterxml.jackson.databind.ObjectMapper; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.TestSuite; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import static io.restassured.RestAssured.given; - -public class TestSuiteRunner extends RunnerBase { - private static final Logger LOGGER = LoggerFactory.getLogger(TestSuiteRunner.class); - - - private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); - - private final TestPlan testPlan; - private final TestSuite testSuite; - private final TestEnvironment testEnvironment; - - public TestSuiteRunner( - TestPlan testPlan, TestSuite testSuite, TestEnvironment testEnvironment) { - this.testPlan = testPlan; - this.testSuite = testSuite; - this.testEnvironment = testEnvironment; - } - - - - @Override - public void execute() throws Throwable { - - - -// for (TestCase testCase : testSuite.tests()) { -// var testRequest = new TestRequest(testCase.command(), testPlan.target(), testEnvironment, TestAssertion.buildAssertions(testCase)); -// var testResponse = testRequest.execute(); -// var testCaseResult = testResponse.validate(testSuite, testCase); -// -// if (testCaseResult.failed()){ -// LOGGER.warn("TestCase FAILED: test.name={}, testCase.name={}, failedAssertion={}, error={}", -// testCaseResult.integrationTest().meta().name(),testCaseResult.testCase().name(), testCaseResult.failedAssertion(), String.valueOf(testCaseResult.error())); -// } -// else{ -// LOGGER.info("TestCase PASSED: test.name={}, testCase.name={}", -// testCaseResult.integrationTest().meta().name(),testCaseResult.testCase().name()); -// } -// } - - } - -} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/VectorizeAstra.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/VectorizeAstra.java index effb9182aa..a2a2c61cc4 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/VectorizeAstra.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/VectorizeAstra.java @@ -17,7 +17,7 @@ public class VectorizeAstra { @TestFactory Stream jobs() { - var testPlan = TestPlan.create("astra-dev", List.of("all-vectorize-workflow")); + var testPlan = TestPlan.create("astra-dev", List.of("all-vectorize-workflow"), false); return testPlan.testNode(); diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/VectorizeUnit.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/VectorizeUnit.java deleted file mode 100644 index 782e37bc7b..0000000000 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/VectorizeUnit.java +++ /dev/null @@ -1,60 +0,0 @@ -package io.stargate.sgv2.jsonapi.api.v1.vectorize; - -import org.eclipse.aether.util.artifact.OverlayArtifactTypeRegistry; -import org.junit.jupiter.api.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.List; - -public class VectorizeUnit { - private static final Logger LOGGER = LoggerFactory.getLogger(VectorizeUnit.class); - - @Test - public void doTest() { - -// var targets = TargetConfigurationss.loadAll("integration-tests/targets/targets.json"); -// var integrationTarget = new Target(targets.configuration("local")); -// -// var itCollection = ITCollection.loadAll("integration-tests/vectorize"); -// -// var workflow = itCollection.workflowFirstByName("all-vectorize-workflow"); - -// var testPlan = TestPlan.create("local", List.of("all-vectorize-workflow")); -// -// -// // SEQUENTIAL -// integrationTarget.workflowStarting(workflow); -// try { -// for (var job : jobs) { -// LOGGER.info("Starting job {}", job.meta()); -// new IntegrationJobRunner(integrationTarget, itCollection, job).run(); -// } -// } finally { -// integrationTarget.workflowFinished(workflow); -// } - - // Parallel -// int maxConcurrentJobs = 2; -// integrationTarget.workflowStarting(workflow); -// try { -// var executor = java.util.concurrent.Executors.newFixedThreadPool(maxConcurrentJobs); -// try { -// var futures = jobs.stream() -// .map(job -> java.util.concurrent.CompletableFuture.runAsync(() -> { -// LOGGER.info("Starting job {}", job.meta()); -// new IntegrationJobRunner(integrationTarget, itCollection, job).run(); -// }, executor)) -// .toList(); -// -// // wait for all, fail if any failed -// futures.forEach(java.util.concurrent.CompletableFuture::join); -// } finally { -// executor.shutdown(); -// } -// } finally { -// integrationTarget.workflowFinished(workflow); -// } - - } -} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/AssertionFactory.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/AssertionFactory.java index 08ce531266..5bff302b56 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/AssertionFactory.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/AssertionFactory.java @@ -1,7 +1,7 @@ package io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions; import com.fasterxml.jackson.databind.JsonNode; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.TestCommand; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.TestCommand; import io.stargate.sgv2.jsonapi.api.v1.vectorize.TestPlan; import java.lang.reflect.InvocationTargetException; diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/AssertionFactoryRegistry.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/AssertionFactoryRegistry.java index fb51179491..cc86703088 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/AssertionFactoryRegistry.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/AssertionFactoryRegistry.java @@ -1,11 +1,5 @@ package io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions; -import com.fasterxml.jackson.databind.JsonNode; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.TestCommand; - -import java.lang.reflect.Method; -import java.lang.reflect.Modifier; -import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/AssertionMatcher.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/AssertionMatcher.java index 70d4225013..60c911e337 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/AssertionMatcher.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/AssertionMatcher.java @@ -1,6 +1,6 @@ package io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.APIResponse; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.messaging.APIResponse; /** * Contract for running an assertion on the response from the API. diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/BodyAssertion.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/BodyAssertion.java index 6416d294f3..5393ea3df9 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/BodyAssertion.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/BodyAssertion.java @@ -1,6 +1,6 @@ package io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.APIResponse; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.messaging.APIResponse; import org.hamcrest.Matcher; import org.hamcrest.StringDescription; diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/DescribableAssertionMatcher.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/DescribableAssertionMatcher.java index 9e001ac0c6..2b3e3942a1 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/DescribableAssertionMatcher.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/DescribableAssertionMatcher.java @@ -1,6 +1,6 @@ package io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.APIResponse; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.messaging.APIResponse; import org.jspecify.annotations.NonNull; public record DescribableAssertionMatcher( diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Documents.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Documents.java index 3deb8135af..fc5bdf5dbd 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Documents.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Documents.java @@ -1,7 +1,7 @@ package io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions; import com.fasterxml.jackson.databind.JsonNode; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.TestCommand; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.TestCommand; import static net.javacrumbs.jsonunit.JsonMatchers.jsonEquals; import static org.hamcrest.Matchers.hasSize; diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Http.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Http.java index 387aa05b39..5958d9b4ee 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Http.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Http.java @@ -1,7 +1,7 @@ package io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions; import com.fasterxml.jackson.databind.JsonNode; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.TestCommand; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.TestCommand; import org.apache.http.HttpStatus; import static io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions.DescribableAssertionMatcher.described; diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Response.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Response.java index f085e91877..65d0283b73 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Response.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Response.java @@ -1,11 +1,7 @@ package io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions; import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.NullNode; -import com.fasterxml.jackson.databind.node.ObjectNode; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.TestCommand; - -import java.util.List; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.TestCommand; import static io.stargate.sgv2.jsonapi.api.v1.ResponseAssertions.*; import static org.hamcrest.Matchers.hasSize; diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/SingleTestAssertion.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/SingleTestAssertion.java index 387164c560..88773fdd81 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/SingleTestAssertion.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/SingleTestAssertion.java @@ -1,15 +1,11 @@ package io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions; import com.fasterxml.jackson.databind.JsonNode; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.TestResponse; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.TestUri; -import io.stargate.sgv2.jsonapi.service.schema.tables.ApiSupportDef; -import org.hamcrest.SelfDescribing; -import org.hamcrest.StringDescription; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestRunResponse; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.DynamicTestExecutable; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestUri; import org.junit.jupiter.api.DynamicNode; -import java.lang.reflect.Executable; -import java.util.Objects; import java.util.concurrent.atomic.AtomicReference; import static org.junit.jupiter.api.DynamicTest.dynamicTest; @@ -21,7 +17,7 @@ public record SingleTestAssertion( AssertionMatcher matcher ) implements TestAssertion { - public void run(TestResponse testResponse) { + public void run(TestRunResponse testResponse) { try { matcher.match(testResponse.apiResponse()); @@ -35,41 +31,25 @@ public void run(TestResponse testResponse) { } @Override - public DynamicNode testNodes(TestUri.Builder uriBuilder, AtomicReference testResponse) { + public DynamicNode testNodes(TestUri.Builder uriBuilder, AtomicReference testResponse) { - uriBuilder.addSegment(TestUri.Segment.ASSERTION, name()); - var original = (matcher instanceof Describable d) ? + var matcherDesc = (matcher instanceof Describable d) ? d.describe() : - null; + ""; - var truncated = ( original != null && original.length() > 60) ? - original.substring(0, 57) + "..." - : - original; - - var testDesc = truncated == null ? - name() - : - "%s [%s]".formatted(name(), truncated); - - // if we truncated the description of the test, we then want to pipe to std out when running - // because it will not be full in the test tree - var stdoutMessage = (original != null && !Objects.equals(truncated, original)) ? - "%s [%s]".formatted(name(), original) - : - null; - - return dynamicTest(testDesc, () -> { + var executable = new DynamicTestExecutable( + "%s [%s]".formatted(name(), matcherDesc), + uriBuilder.addSegment(TestUri.Segment.ASSERTION, name()), + () -> { var resp = testResponse.get(); if (resp == null) { throw new IllegalStateException("Response is null"); } - if (stdoutMessage != null) { - System.out.printf(stdoutMessage); - } run(resp); } ); + + return executable.testNode(); } } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Status.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Status.java index 4ae2703a89..722607068a 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Status.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Status.java @@ -1,7 +1,7 @@ package io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions; import com.fasterxml.jackson.databind.JsonNode; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.TestCommand; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.TestCommand; import static net.javacrumbs.jsonunit.JsonMatchers.jsonEquals; diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Templated.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Templated.java index 4c14df897c..cc86644ab7 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Templated.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Templated.java @@ -2,9 +2,8 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.TestCommand; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.TestCommand; import io.stargate.sgv2.jsonapi.api.v1.vectorize.TestPlan; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.AssertionTemplateSpec; import java.util.List; diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/TestAssertion.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/TestAssertion.java index 6b9c5ccb36..d212c54cf9 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/TestAssertion.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/TestAssertion.java @@ -1,19 +1,16 @@ package io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions; import com.fasterxml.jackson.databind.JsonNode; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.TestCase; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.TestCommand; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.TestCase; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.TestCommand; import io.stargate.sgv2.jsonapi.api.v1.vectorize.TestPlan; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.TestResponse; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestRunResponse; import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.AssertionTemplateSpec; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.SpecFiles; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.TestUri; -import org.assertj.core.api.AssertFactory; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestUri; import org.junit.jupiter.api.DynamicNode; import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Stream; @@ -23,9 +20,9 @@ public interface TestAssertion { JsonNode args(); - void run(TestResponse testResponse); + void run(TestRunResponse testResponse); - DynamicNode testNodes(TestUri.Builder uriBuilder, AtomicReference testResponse); + DynamicNode testNodes(TestUri.Builder uriBuilder, AtomicReference testResponse); static List forSuccess(TestPlan testPlan, TestCommand testCommand) { diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/TestAssertionContainer.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/TestAssertionContainer.java index 1af420a1b9..81bf82d9c9 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/TestAssertionContainer.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/TestAssertionContainer.java @@ -1,8 +1,8 @@ package io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions; import com.fasterxml.jackson.databind.JsonNode; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.TestResponse; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.TestUri; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestRunResponse; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestUri; import org.junit.jupiter.api.DynamicNode; import java.util.List; @@ -18,7 +18,7 @@ public record TestAssertionContainer( ) implements TestAssertion { @Override - public void run(TestResponse testResponse) { + public void run(TestRunResponse testResponse) { for (TestAssertion assertion : assertions) { try{ assertion.run(testResponse); @@ -37,7 +37,7 @@ public void run(TestResponse testResponse) { } @Override - public DynamicNode testNodes(TestUri.Builder uriBuilder, AtomicReference testResponse) { + public DynamicNode testNodes(TestUri.Builder uriBuilder, AtomicReference testResponse) { uriBuilder.addSegment(TestUri.Segment.ASSERTION, name()); var childs = assertions.stream() diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/lifecycle/TestPlanLifecycle.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/lifecycle/TestPlanLifecycle.java index 31eb1dee36..90420cf48d 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/lifecycle/TestPlanLifecycle.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/lifecycle/TestPlanLifecycle.java @@ -1,21 +1,21 @@ package io.stargate.sgv2.jsonapi.api.v1.vectorize.lifecycle; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.Job; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.TestEnvironment; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.Job; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestRunEnv; import io.stargate.sgv2.jsonapi.api.v1.vectorize.TestPlan; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.TestSuite; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.TestUri; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.Workflow; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.TestSuiteSpec; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestUri; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.WorkflowSpec; import org.junit.jupiter.api.DynamicNode; import java.util.Optional; public interface TestPlanLifecycle { - default Optional beforeWorkflow(TestPlan testPlan, TestUri.Builder uriBuilder, Workflow workflow){ + default Optional beforeWorkflow(TestPlan testPlan, TestUri.Builder uriBuilder, WorkflowSpec workflow){ return Optional.empty(); } - default Optional afterWorkflow(TestPlan testPlan,TestUri.Builder uriBuilder, Workflow workflow){ + default Optional afterWorkflow(TestPlan testPlan,TestUri.Builder uriBuilder, WorkflowSpec workflow){ return Optional.empty(); } @@ -26,10 +26,10 @@ default Optional afterJob(TestPlan testPlan, TestUri.Builder uriBu return Optional.empty(); } - default Optional beforeTestSuite(TestPlan testPlan, TestUri.Builder uriBuilder,TestSuite test, TestEnvironment env){ + default Optional beforeTestSuite(TestPlan testPlan, TestUri.Builder uriBuilder, TestSuiteSpec test, TestRunEnv env){ return Optional.empty(); } - default Optional afterTestSuite(TestPlan testPlan, TestUri.Builder uriBuilder,TestSuite test, TestEnvironment env){ + default Optional afterTestSuite(TestPlan testPlan, TestUri.Builder uriBuilder, TestSuiteSpec test, TestRunEnv env){ return Optional.empty(); } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/APIRequest.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/messaging/APIRequest.java similarity index 87% rename from src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/APIRequest.java rename to src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/messaging/APIRequest.java index e57f2768d1..00db656092 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/APIRequest.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/messaging/APIRequest.java @@ -1,4 +1,4 @@ -package io.stargate.sgv2.jsonapi.api.v1.vectorize; +package io.stargate.sgv2.jsonapi.api.v1.vectorize.messaging; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; @@ -8,6 +8,9 @@ import io.restassured.response.ValidatableResponse; import io.restassured.specification.RequestSpecification; import io.stargate.sgv2.jsonapi.api.model.command.CommandTarget; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.TestCommand; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestRunEnv; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.targets.Connection; import io.stargate.sgv2.jsonapi.config.constants.HttpConstants; import java.util.Map; @@ -24,10 +27,10 @@ public class APIRequest { private static String DB_PATH = "/"; private final Connection connection; - private final TestEnvironment integrationEnv; + private final TestRunEnv integrationEnv; private final ObjectNode request; - public APIRequest(Connection connection, TestEnvironment integrationEnv, ObjectNode request ) { + public APIRequest(Connection connection, TestRunEnv integrationEnv, ObjectNode request ) { this.connection = connection; this.integrationEnv = integrationEnv; diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/APIResponse.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/messaging/APIResponse.java similarity index 73% rename from src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/APIResponse.java rename to src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/messaging/APIResponse.java index 05120514be..0e407326fa 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/APIResponse.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/messaging/APIResponse.java @@ -1,4 +1,4 @@ -package io.stargate.sgv2.jsonapi.api.v1.vectorize; +package io.stargate.sgv2.jsonapi.api.v1.vectorize.messaging; import io.restassured.response.ValidatableResponse; diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/backends/AstraBackend.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/targets/AstraBackend.java similarity index 58% rename from src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/backends/AstraBackend.java rename to src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/targets/AstraBackend.java index 528fadf7bf..7c2e58f046 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/backends/AstraBackend.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/targets/AstraBackend.java @@ -1,6 +1,6 @@ -package io.stargate.sgv2.jsonapi.api.v1.vectorize.backends; +package io.stargate.sgv2.jsonapi.api.v1.vectorize.targets; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.Job; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.Job; public class AstraBackend extends Backend { diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/backends/Backend.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/targets/Backend.java similarity index 61% rename from src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/backends/Backend.java rename to src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/targets/Backend.java index 95acf200eb..027bf7bcaa 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/backends/Backend.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/targets/Backend.java @@ -1,6 +1,6 @@ -package io.stargate.sgv2.jsonapi.api.v1.vectorize.backends; +package io.stargate.sgv2.jsonapi.api.v1.vectorize.targets; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.Job; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.Job; import io.stargate.sgv2.jsonapi.api.v1.vectorize.lifecycle.TestPlanLifecycle; public abstract class Backend implements TestPlanLifecycle { diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/backends/CassandraBackend.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/targets/CassandraBackend.java similarity index 78% rename from src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/backends/CassandraBackend.java rename to src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/targets/CassandraBackend.java index ac2585d4c2..1f7767e3ae 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/backends/CassandraBackend.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/targets/CassandraBackend.java @@ -1,18 +1,18 @@ -package io.stargate.sgv2.jsonapi.api.v1.vectorize.backends; +package io.stargate.sgv2.jsonapi.api.v1.vectorize.targets; import com.fasterxml.jackson.databind.ObjectMapper; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.Job; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.TestCommand; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.Job; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.TestCommand; import io.stargate.sgv2.jsonapi.api.v1.vectorize.TestPlan; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.TestRequest; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestRunRequest; import io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions.TestAssertion; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.TestUri; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestUri; import org.apache.commons.lang3.RandomStringUtils; import org.junit.jupiter.api.DynamicNode; import java.util.Optional; -import static io.stargate.sgv2.jsonapi.api.v1.vectorize.TestEnvironment.toSafeSchemaIdentifier; +import static io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestRunEnv.toSafeSchemaIdentifier; /** * Cassandra:Y2Fzc2FuZHJh:Y2Fzc2FuZHJh @@ -42,7 +42,7 @@ public Optional beforeJob(TestPlan testPlan, TestUri.Builder uriBui """); var env = job.withoutMatrix(testPlan); - var setupRequest = new TestRequest( + var setupRequest = new TestRunRequest( env.substitutor().replace("createKeyspace: ${KEYSPACE_NAME}"), command, testPlan.target(), env, TestAssertion.forSuccess( testPlan, command)); @@ -50,7 +50,7 @@ public Optional beforeJob(TestPlan testPlan, TestUri.Builder uriBui } @Override - public Optional afterJob(TestPlan testPlan, TestUri.Builder uriBuilder,Job job) { + public Optional afterJob(TestPlan testPlan, TestUri.Builder uriBuilder, Job job) { var command = TestCommand.fromJson( """ { @@ -61,7 +61,7 @@ public Optional afterJob(TestPlan testPlan, TestUri.Builder uriBuil """); var env = job.withoutMatrix(testPlan); - var setupRequest = new TestRequest( + var setupRequest = new TestRunRequest( env.substitutor().replace("dropKeyspace: ${KEYSPACE_NAME}"), command, testPlan.target(), job.withoutMatrix(testPlan), TestAssertion.forSuccess(testPlan,command)); diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/Connection.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/targets/Connection.java similarity index 82% rename from src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/Connection.java rename to src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/targets/Connection.java index f21099f4fa..16e6d01fdc 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/Connection.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/targets/Connection.java @@ -1,4 +1,4 @@ -package io.stargate.sgv2.jsonapi.api.v1.vectorize; +package io.stargate.sgv2.jsonapi.api.v1.vectorize.targets; public record Connection( String domain, diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/Target.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/targets/Target.java similarity index 65% rename from src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/Target.java rename to src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/targets/Target.java index d7460153cb..67e845fee0 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/Target.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/targets/Target.java @@ -1,12 +1,15 @@ -package io.stargate.sgv2.jsonapi.api.v1.vectorize; +package io.stargate.sgv2.jsonapi.api.v1.vectorize.targets; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.backends.AstraBackend; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.backends.Backend; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.backends.CassandraBackend; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.*; import io.stargate.sgv2.jsonapi.api.v1.vectorize.lifecycle.TestPlanLifecycle; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.TestSuite; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.TestUri; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.Workflow; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.messaging.APIRequest; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestRunEnv; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.Job; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.TestCommand; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.TargetConfiguration; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.TestSuiteSpec; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestUri; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.WorkflowSpec; import org.junit.jupiter.api.DynamicNode; import java.util.HashMap; @@ -16,11 +19,11 @@ public class Target implements TestPlanLifecycle { private final TargetConfiguration targetConfiguration; private final Backend backend; - private final TestEnvironment env; + private final TestRunEnv env; public Target(TargetConfiguration targetConfiguration) { this.targetConfiguration = targetConfiguration; - this.env = new TestEnvironment(new HashMap<>()); + this.env = new TestRunEnv(new HashMap<>()); this.backend = switch (targetConfiguration.backend()) { case "cassandra" -> new CassandraBackend(); @@ -41,34 +44,34 @@ public void updateJobForTarget(Job job){ } - public APIRequest apiRequest(TestCommand testCommand, TestEnvironment env){ + public APIRequest apiRequest(TestCommand testCommand, TestRunEnv env){ return new APIRequest(targetConfiguration.connection(), env, testCommand.withEnvironment(env)); } @Override - public Optional beforeWorkflow(TestPlan testPlan, TestUri.Builder uriBuilder, Workflow workflow){ + public Optional beforeWorkflow(TestPlan testPlan, TestUri.Builder uriBuilder, WorkflowSpec workflow){ return backend.beforeWorkflow(testPlan, uriBuilder,workflow); } @Override - public Optional afterWorkflow(TestPlan testPlan, TestUri.Builder uriBuilder,Workflow workflow){ + public Optional afterWorkflow(TestPlan testPlan, TestUri.Builder uriBuilder, WorkflowSpec workflow){ return backend.afterWorkflow(testPlan, uriBuilder,workflow); } @Override - public Optional beforeJob(TestPlan testPlan, TestUri.Builder uriBuilder,Job job){ + public Optional beforeJob(TestPlan testPlan, TestUri.Builder uriBuilder, Job job){ return backend.beforeJob(testPlan, uriBuilder,job); } @Override - public Optional afterJob(TestPlan testPlan,TestUri.Builder uriBuilder,Job job){ + public Optional afterJob(TestPlan testPlan, TestUri.Builder uriBuilder, Job job){ return backend.afterJob(testPlan,uriBuilder, job); } @Override - public Optional beforeTestSuite(TestPlan testPlan, TestUri.Builder uriBuilder,TestSuite test, TestEnvironment env){ + public Optional beforeTestSuite(TestPlan testPlan, TestUri.Builder uriBuilder, TestSuiteSpec test, TestRunEnv env){ return backend.beforeTestSuite(testPlan,uriBuilder, test, env); } @Override - public Optional afterTestSuite(TestPlan testPlan, TestUri.Builder uriBuilder,TestSuite test, TestEnvironment env){ + public Optional afterTestSuite(TestPlan testPlan, TestUri.Builder uriBuilder, TestSuiteSpec test, TestRunEnv env){ return backend.afterTestSuite(testPlan, uriBuilder,test, env); } } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testrun/DynamicTestExecutable.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testrun/DynamicTestExecutable.java new file mode 100644 index 0000000000..d09b28820e --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testrun/DynamicTestExecutable.java @@ -0,0 +1,60 @@ +package io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun; + +import org.junit.jupiter.api.DynamicTest; +import org.junit.jupiter.api.function.Executable; + +import static org.junit.jupiter.api.DynamicTest.dynamicTest; + +public class DynamicTestExecutable implements Executable { + + private final String description; + private final TestUri testUri; + private final Executable executable; + + private final String trimmedDisplayName; + private final boolean isTrimmed; + + public DynamicTestExecutable(String description, TestUri.Builder testUri, Executable executable) { + this(description, testUri.build(), executable); + } + + @SuppressWarnings("StringEquality") + public DynamicTestExecutable(String description, TestUri testUri, Executable executable) { + this.description = description; + this.testUri = testUri; + this.executable = executable; + + var truncated = ( description != null && description.length() > 60) ? + description.substring(0, 57) + "..." + : + description; + + this.trimmedDisplayName = truncated; + this.isTrimmed = truncated != description; + } + + public String trimmedDisplayName(){ + return trimmedDisplayName; + } + + public DynamicTest testNode(){ + return dynamicTest(trimmedDisplayName, testUri.uri(), this); + } + + @Override + public void execute() throws Throwable { + beforeExecute(); + executable.execute(); + afterExecute(); + } + + private void beforeExecute(){ + if (isTrimmed) { + System.out.printf(description + "\n"); + } + } + + private void afterExecute(){ + System.out.printf("Executed - " + testUri.uri().toString() + "\n"); + } +} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestEnvironment.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testrun/TestRunEnv.java similarity index 87% rename from src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestEnvironment.java rename to src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testrun/TestRunEnv.java index 327f6dbc7f..d45adf0b8a 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestEnvironment.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testrun/TestRunEnv.java @@ -1,7 +1,7 @@ -package io.stargate.sgv2.jsonapi.api.v1.vectorize; +package io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.TestSuite; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.TestUri; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.TestPlan; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.TestSuiteSpec; import org.apache.commons.text.StringSubstitutor; import org.apache.commons.text.lookup.StringLookupFactory; import org.junit.jupiter.api.DynamicContainer; @@ -15,24 +15,24 @@ import static org.junit.jupiter.api.DynamicTest.dynamicTest; -public class TestEnvironment { +public class TestRunEnv { - private static final Logger LOGGER = LoggerFactory.getLogger(TestEnvironment.class); + private static final Logger LOGGER = LoggerFactory.getLogger(TestRunEnv.class); private static final Pattern PATTERN_NOT_WORD_CHARS = Pattern.compile("\\W+"); private static final Set SCHEMA_IDENTIFIER = Set.of("KEYSPACE_NAME", "COLLECTION_NAME"); private final Map vars = new HashMap<>(); - public TestEnvironment(){ + public TestRunEnv(){ this(new HashMap<>()); } - public TestEnvironment(Map vars) { + public TestRunEnv(Map vars) { this.vars.putAll(vars); } - public DynamicContainer testNode(TestPlan testPlan, TestUri.Builder uriBuilder, TestSuite testSuite) { + public DynamicContainer testNode(TestPlan testPlan, TestUri.Builder uriBuilder, TestSuiteSpec testSuite) { var d = description(); uriBuilder.addSegment(TestUri.Segment.ENV, d); @@ -64,15 +64,15 @@ private String description(){ .toString(); } - private TestEnvironment(TestEnvironment other){ + private TestRunEnv(TestRunEnv other){ this.vars.putAll(other.vars); } - public TestEnvironment clone(){ - return new TestEnvironment(this); + public TestRunEnv clone(){ + return new TestRunEnv(this); } - public TestEnvironment put(TestEnvironment other){ + public TestRunEnv put(TestRunEnv other){ this.vars.putAll(other.vars); return this; } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestRequest.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testrun/TestRunRequest.java similarity index 61% rename from src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestRequest.java rename to src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testrun/TestRunRequest.java index 8233b640af..2c362efd32 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestRequest.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testrun/TestRunRequest.java @@ -1,11 +1,11 @@ -package io.stargate.sgv2.jsonapi.api.v1.vectorize; +package io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun; import io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions.TestAssertion; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.TestUri; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.targets.Target; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.TestCommand; import org.junit.jupiter.api.DynamicContainer; import org.junit.jupiter.api.DynamicNode; -import java.util.ArrayList; import java.util.List; import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Stream; @@ -13,17 +13,17 @@ import static org.junit.jupiter.api.DynamicContainer.dynamicContainer; import static org.junit.jupiter.api.DynamicTest.dynamicTest; -public record TestRequest( +public record TestRunRequest( String name, TestCommand testCommand, Target target, - TestEnvironment testEnvironment, + TestRunEnv testEnvironment, List testAssertions) { - public TestResponse execute() { + public TestRunResponse execute() { var apiRequest = target.apiRequest(testCommand, testEnvironment); - return new TestResponse(this, apiRequest, apiRequest.execute()); + return new TestRunResponse(this, apiRequest, apiRequest.execute()); } public DynamicContainer testNodes(TestUri.Builder uriBuilder) { @@ -32,14 +32,15 @@ public DynamicContainer testNodes(TestUri.Builder uriBuilder) { Stream.Builder nodesBuilder = Stream.builder();; - AtomicReference atomicResponse = new AtomicReference<>(); + AtomicReference atomicResponse = new AtomicReference<>(); // Execute the request, and set so the assertions can pull the response after. - var commandUriBuilder = uriBuilder.clone(); - commandUriBuilder.addSegment(TestUri.Segment.COMMAND, testCommand.commandName().getApiName()); - nodesBuilder.add(dynamicTest("Command: " + testCommand.commandName().getApiName(), - commandUriBuilder.build().uri(), - () -> atomicResponse.set(execute()))); + var commandExecutable = new DynamicTestExecutable( + "Command: " + testCommand.commandName().getApiName(), + uriBuilder.clone().addSegment(TestUri.Segment.COMMAND, testCommand.commandName().getApiName()), + () -> atomicResponse.set(execute()) + ); + nodesBuilder.add(commandExecutable.testNode()); // tests for each assertion var assertionsUriBuilder = uriBuilder.clone().addSegment(TestUri.Segment.ASSERTION_CONTAINER, "assertions"); @@ -49,7 +50,7 @@ public DynamicContainer testNodes(TestUri.Builder uriBuilder) { // if we have assertion tests, put them in a container if (!assertionTests.isEmpty()) { - nodesBuilder.add(dynamicContainer("Assertions", assertionTests)); + nodesBuilder.add(dynamicContainer("Assertions",assertionsUriBuilder.build().uri(), assertionTests.stream())); } return dynamicContainer("Request: " + name, uriBuilder.build().uri(), nodesBuilder.build()); diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestResponse.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testrun/TestRunResponse.java similarity index 75% rename from src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestResponse.java rename to src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testrun/TestRunResponse.java index a80e796e89..df39678b70 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestResponse.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testrun/TestRunResponse.java @@ -1,9 +1,10 @@ -package io.stargate.sgv2.jsonapi.api.v1.vectorize; +package io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions.AssertionMatcher; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.messaging.APIRequest; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.messaging.APIResponse; -public record TestResponse( - TestRequest testRequest, +public record TestRunResponse( + TestRunRequest testRequest, APIRequest apiRequest, APIResponse apiResponse ) { diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/TestUri.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testrun/TestUri.java similarity index 95% rename from src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/TestUri.java rename to src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testrun/TestUri.java index ffd68873ed..5764078afa 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/TestUri.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testrun/TestUri.java @@ -1,6 +1,4 @@ -package io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec; - -import io.stargate.sgv2.jsonapi.service.operation.builder.LiteralTerm; +package io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun; import java.net.URI; import java.util.*; @@ -38,7 +36,6 @@ public static TestUri parse(URI uri) { } public enum Scheme { - TESTPLAN, TESTRUN; public String pathName() { @@ -138,7 +135,7 @@ public Builder addSegment(Segment segment, String value) { } public Builder clone(){ - return new Builder(scheme, segmentValues); + return new Builder(scheme, new ArrayList<>(segmentValues)); } public TestUri build() { diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/Job.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/Job.java similarity index 69% rename from src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/Job.java rename to src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/Job.java index 2a7b7b92c2..cbc8280b46 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/Job.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/Job.java @@ -1,9 +1,8 @@ -package io.stargate.sgv2.jsonapi.api.v1.vectorize; +package io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.TestSpecKind; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.TestSpecMeta; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.TestSuite; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.TestUri; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestRunEnv; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.TestPlan; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestUri; import org.junit.jupiter.api.DynamicContainer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -44,19 +43,19 @@ public DynamicContainer testNode(TestPlan testPlan, TestUri.Builder uriBuilder) testPlan.addLifecycle(uriBuilder.clone(),this, testSuiteNodes)); } - public Stream testSuites(TestPlan testPlan) { - Stream.Builder allTests = Stream.builder(); + public Stream testSuites(TestPlan testPlan) { + Stream.Builder allTests = Stream.builder(); tests() .forEach( testName -> { - testPlan.specFiles().byNameAsType(TestSuite.class, testName).forEach(allTests) ; + testPlan.specFiles().byNameAsType(TestSuiteSpec.class, testName).forEach(allTests) ; }); return allTests.build(); } - public TestEnvironment withoutMatrix(TestPlan testPlan) { + public TestRunEnv withoutMatrix(TestPlan testPlan) { - var fromEnv = new TestEnvironment(); + var fromEnv = new TestRunEnv(); for (Map.Entry entry : fromEnvironment.entrySet()) { var value = System.getenv(entry.getValue()); @@ -66,13 +65,13 @@ public TestEnvironment withoutMatrix(TestPlan testPlan) { fromEnv.put(entry.getKey(), value); } - var fromVariables = new TestEnvironment(variables); + var fromVariables = new TestRunEnv(variables); return fromEnv.clone().put(fromVariables); } - public List allEnvironments(TestPlan testPlan) { + public List allEnvironments(TestPlan testPlan) { - var fromEnv = new TestEnvironment(); + var fromEnv = new TestRunEnv(); for (Map.Entry entry : fromEnvironment.entrySet()) { var value = System.getenv(entry.getValue()); @@ -82,18 +81,18 @@ public List allEnvironments(TestPlan testPlan) { fromEnv.put(entry.getKey(), value); } - var fromVariables = new TestEnvironment(variables); + var fromVariables = new TestRunEnv(variables); // TODO: handle more matrix - List fromMatrix = new ArrayList<>(); + List fromMatrix = new ArrayList<>(); matrix.get("MODEL").forEach( model -> { - var env = new TestEnvironment(); + var env = new TestRunEnv(); env.put("MODEL", model); fromMatrix.add(env); }); - List allEnvs = new ArrayList<>(); + List allEnvs = new ArrayList<>(); for (var matrixEnv : fromMatrix) { var completeEnv = fromEnv diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/SpecFiles.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/SpecFiles.java index a526b34914..bf0e4eac9b 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/SpecFiles.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/SpecFiles.java @@ -27,7 +27,7 @@ private SpecFiles(List specFiles) { this.specFiles = specFiles; for (SpecFile file : specFiles) { - if (file.spec() instanceof TestSuite it){ + if (file.spec() instanceof TestSuiteSpec it){ it.expand(this); } } @@ -92,8 +92,9 @@ private static SpecFile loadOne(Path path) { var element = switch (TestSpecKind.valueOf(kindNode.asText().toUpperCase())) { case ASSERTION_TEMPLATE -> MAPPER.treeToValue(root, AssertionTemplateSpec.class); - case TEST_SUITE -> MAPPER.treeToValue(root, TestSuite.class); - case WORKFLOW -> MAPPER.treeToValue(root, Workflow.class); + case TARGETS -> MAPPER.treeToValue(root, TargetsSpec.class); + case TEST_SUITE -> MAPPER.treeToValue(root, TestSuiteSpec.class); + case WORKFLOW -> MAPPER.treeToValue(root, WorkflowSpec.class); }; return new SpecFile(file, element, root); diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/TargetConfiguration.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/TargetConfiguration.java new file mode 100644 index 0000000000..92491b097a --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/TargetConfiguration.java @@ -0,0 +1,6 @@ +package io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec; + +import io.stargate.sgv2.jsonapi.api.v1.vectorize.targets.Connection; + +public record TargetConfiguration(String name, String backend, Connection connection) { +} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TargetConfigurationss.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/TargetsSpec.java similarity index 75% rename from src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TargetConfigurationss.java rename to src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/TargetsSpec.java index 2fe7182b16..abf6ea45db 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TargetConfigurationss.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/TargetsSpec.java @@ -1,5 +1,6 @@ -package io.stargate.sgv2.jsonapi.api.v1.vectorize; +package io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec; +import com.fasterxml.jackson.databind.MapperFeature; import com.fasterxml.jackson.databind.ObjectMapper; import java.io.IOException; @@ -11,12 +12,16 @@ import java.util.List; import java.util.Set; -public record TargetConfigurationss(List targets) { +public record TargetsSpec( + TestSpecMeta meta, + List targets) + implements TestSpec { - private static final ObjectMapper MAPPER = new ObjectMapper(); + private static final ObjectMapper MAPPER = new ObjectMapper() + .configure(MapperFeature.ACCEPT_CASE_INSENSITIVE_ENUMS, true); - public TargetConfigurationss { + public TargetsSpec { Set seen = new HashSet(); for (TargetConfiguration target : targets) { if (seen.contains(target.name())) { @@ -30,11 +35,11 @@ public TargetConfiguration configuration(String name) { return targets.stream().filter(target -> target.name().equals(name)).findFirst().orElseThrow(() -> new IllegalArgumentException("target name not found: " + name)); } - static TargetConfigurationss loadAll(String path) { + public static TargetsSpec loadAll(String path) { final Path dir = resourceDir(path); try { - return MAPPER.readValue(dir.toFile(), TargetConfigurationss.class); + return MAPPER.readValue(dir.toFile(), TargetsSpec.class); } catch (IOException e) { throw new RuntimeException(e); } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestCase.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/TestCase.java similarity index 55% rename from src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestCase.java rename to src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/TestCase.java index 6e03d965ab..670ab9e73d 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestCase.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/TestCase.java @@ -1,10 +1,12 @@ -package io.stargate.sgv2.jsonapi.api.v1.vectorize; +package io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.databind.node.ObjectNode; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions.AssertionMatcher; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestRunEnv; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.TestPlan; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestRunRequest; import io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions.TestAssertion; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.TestUri; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestUri; import org.junit.jupiter.api.DynamicContainer; public record TestCase( @@ -16,9 +18,9 @@ public record TestCase( String include) { - public DynamicContainer testNodesForEnvironment(TestPlan testPlan, TestUri.Builder uriBuilder, TestEnvironment testEnvironment) { + public DynamicContainer testNodesForEnvironment(TestPlan testPlan, TestUri.Builder uriBuilder, TestRunEnv testEnvironment) { - var testRequest = new TestRequest( + var testRequest = new TestRunRequest( "TestCase: name=%s".formatted(name, command.commandName()), command(), testPlan.target(), testEnvironment, TestAssertion.buildAssertions(testPlan, this)); diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestCommand.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/TestCommand.java similarity index 95% rename from src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestCommand.java rename to src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/TestCommand.java index 255994aed0..08632e41c1 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestCommand.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/TestCommand.java @@ -1,4 +1,4 @@ -package io.stargate.sgv2.jsonapi.api.v1.vectorize; +package io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.core.JsonProcessingException; @@ -8,6 +8,7 @@ import com.fasterxml.jackson.databind.node.TextNode; import io.stargate.sgv2.jsonapi.api.model.command.CommandName; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestRunEnv; import org.apache.commons.text.StringSubstitutor; public class TestCommand { @@ -84,7 +85,7 @@ private static String commandNameString(ObjectNode request) { return name; } - public ObjectNode withEnvironment(TestEnvironment env) { + public ObjectNode withEnvironment(TestRunEnv env) { ObjectNode updated = request.deepCopy(); walk(updated, env.substitutor()); return updated; diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/TestSpec.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/TestSpec.java index f1871db67b..86460164ff 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/TestSpec.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/TestSpec.java @@ -1,6 +1,6 @@ package io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec; -public sealed interface TestSpec permits Workflow, TestSuite, AssertionTemplateSpec { +public sealed interface TestSpec permits AssertionTemplateSpec, TargetsSpec, TestSuiteSpec, WorkflowSpec { TestSpecMeta meta(); diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/TestSpecKind.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/TestSpecKind.java index f1a8a28171..4f708bea98 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/TestSpecKind.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/TestSpecKind.java @@ -2,13 +2,15 @@ public enum TestSpecKind { ASSERTION_TEMPLATE, + TARGETS, TEST_SUITE, WORKFLOW; public static TestSpecKind fromType(Class clazz) { if (clazz == AssertionTemplateSpec.class) { return ASSERTION_TEMPLATE; } - if (clazz == TestSuite.class) { return TEST_SUITE; } - if (clazz == Workflow.class) { return WORKFLOW; } + if (clazz == TargetsSpec.class) { return TARGETS; } + if (clazz == TestSuiteSpec.class) { return TEST_SUITE; } + if (clazz == WorkflowSpec.class) { return WORKFLOW; } throw new IllegalArgumentException("Unknown TestSpec type: " + clazz); } } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/TestSuite.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/TestSuiteSpec.java similarity index 80% rename from src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/TestSuite.java rename to src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/TestSuiteSpec.java index b8e0da0c30..0aca6ed97e 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/TestSuite.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/TestSuiteSpec.java @@ -2,6 +2,9 @@ import io.stargate.sgv2.jsonapi.api.v1.vectorize.*; import io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions.TestAssertion; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestRunRequest; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestRunEnv; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestUri; import org.junit.jupiter.api.DynamicContainer; import org.junit.jupiter.api.DynamicNode; @@ -11,11 +14,11 @@ import static org.junit.jupiter.api.DynamicContainer.dynamicContainer; -public record TestSuite(TestSpecMeta meta, List setup, List tests, List cleanup) +public record TestSuiteSpec(TestSpecMeta meta, List setup, List tests, List cleanup) implements TestSpec { - public DynamicContainer testNode(TestPlan testPlan, TestUri.Builder uriBuilder, List allEnvs) { + public DynamicContainer testNode(TestPlan testPlan, TestUri.Builder uriBuilder, List allEnvs) { uriBuilder.addSegment(TestUri.Segment.SUITE, meta().name()); @@ -30,7 +33,7 @@ public DynamicContainer testNode(TestPlan testPlan, TestUri.Builder uriBuilder, ); } - public Collection testNodesForEnvironment(TestPlan testPlan, TestUri.Builder uriBuilder, TestEnvironment testEnvironment) { + public Collection testNodesForEnvironment(TestPlan testPlan, TestUri.Builder uriBuilder, TestRunEnv testEnvironment) { List nodes = new ArrayList<>(); @@ -38,7 +41,7 @@ public Collection testNodesForEnvironment(TestPlan testPl var setupUriBuilder = uriBuilder.clone().addSegment(TestUri.Segment.STAGE, "setup"); for (TestCommand setupCommand : setup()) { - var setupRequest = new TestRequest( + var setupRequest = new TestRunRequest( "SetupRequest[%s]: %s".formatted(i++, setupCommand.commandName()), setupCommand, testPlan.target(), testEnvironment, TestAssertion.forSuccess(testPlan, setupCommand)); @@ -52,7 +55,7 @@ public Collection testNodesForEnvironment(TestPlan testPl var cleanupUriBuilder = uriBuilder.clone().addSegment(TestUri.Segment.STAGE, "cleanup"); for (TestCommand cleanupCommand : cleanup()) { - var cleanupRequest = new TestRequest( + var cleanupRequest = new TestRunRequest( "CleanupRequest[%s]: %s".formatted(i++, cleanupCommand.commandName()), cleanupCommand, testPlan.target(), testEnvironment, TestAssertion.forSuccess(testPlan,cleanupCommand)); nodes.add(cleanupRequest.testNodes(cleanupUriBuilder.clone())); @@ -67,7 +70,7 @@ public void expand(SpecFiles specFiles) { List expandedSetup = new ArrayList<>(); for (TestCommand command : setup) { if (command.includeFrom() != null) { - var includedTest = specFiles.byNameAsType(TestSuite.class, command.includeFrom()) + var includedTest = specFiles.byNameAsType(TestSuiteSpec.class, command.includeFrom()) .findFirst() .orElseThrow(() -> new IllegalStateException("Included TestSuite Setup not found. parent=%s, included=%s".formatted(meta().name(), command.includeFrom()))); @@ -82,7 +85,7 @@ public void expand(SpecFiles specFiles) { List expandedTests = new ArrayList<>(); for (TestCase testCase : tests) { if (testCase.include() != null) { - var includedTest = specFiles.byNameAsType(TestSuite.class, testCase.include()) + var includedTest = specFiles.byNameAsType(TestSuiteSpec.class, testCase.include()) .findFirst() .orElseThrow(() -> new IllegalStateException("Included TestSuite TestCase not found. parent=%s, included=%s".formatted(meta().name(), testCase.include()))); diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/Workflow.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/WorkflowSpec.java similarity index 58% rename from src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/Workflow.java rename to src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/WorkflowSpec.java index 8732fd0610..4d41144a39 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/Workflow.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/WorkflowSpec.java @@ -1,7 +1,7 @@ package io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.Job; import io.stargate.sgv2.jsonapi.api.v1.vectorize.TestPlan; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestUri; import org.junit.jupiter.api.DynamicContainer; import java.util.List; @@ -9,22 +9,25 @@ import static org.junit.jupiter.api.DynamicContainer.dynamicContainer; -public record Workflow(TestSpecMeta meta, List jobs) implements TestSpec { +public record WorkflowSpec(TestSpecMeta meta, List jobs) implements TestSpec { - public Stream activeJobs(){ - return jobs().stream() - .filter(job -> !job.meta().tags().contains("disabled")); + public DynamicContainer testNode(TestPlan testPlan, TestUri.Builder uriBuilder){ + return testNode(testPlan, uriBuilder, true); } - - public DynamicContainer testNode(TestPlan testPlan, TestUri.Builder uriBuilder) { + public DynamicContainer testNode(TestPlan testPlan, TestUri.Builder uriBuilder, boolean ignoreDisabled) { uriBuilder.addSegment(TestUri.Segment.WORKFLOW, meta().name()); var desc = "Workflow: %s ".formatted( meta.name()); - var jobNodes = activeJobs() + var testNodeJobs = ignoreDisabled ? + jobs().stream() + .filter(job -> !job.meta().tags().contains("disabled")) + : + jobs().stream(); + var jobNodes = testNodeJobs .map(job -> job.testNode(testPlan, uriBuilder.clone())); return dynamicContainer( diff --git a/src/test/resources/integration-tests/targets/targets.json b/src/test/resources/integration-tests/targets/targets.json index fc65150a81..67c7d54535 100644 --- a/src/test/resources/integration-tests/targets/targets.json +++ b/src/test/resources/integration-tests/targets/targets.json @@ -1,4 +1,8 @@ { + "meta": { + "name": "targets", + "kind": "targets" + }, "targets": [ { "name": "local", @@ -22,7 +26,7 @@ "name": "astra-dev", "backend": "astra", "connection": { - "domain": "https://830c0231-8c79-4212-b148-8c4bd467c917-us-west-2.apps.astra-dev.datastax.com", + "domain": "https://2637a6ae-d489-498a-a740-69362af254ef-us-west-2.apps.astra-dev.datastax.com", "port": 443, "basePath": "/api/json/v1" } diff --git a/src/test/resources/integration-tests/vectorize/vectorize-workflow.json b/src/test/resources/integration-tests/vectorize/vectorize-workflow.json index e08ff4c5e3..a20b56e789 100644 --- a/src/test/resources/integration-tests/vectorize/vectorize-workflow.json +++ b/src/test/resources/integration-tests/vectorize/vectorize-workflow.json @@ -7,7 +7,7 @@ { "meta": { "name": "open-ai-vectorize", - "tags": ["disable"] + "tags": ["disabled"] }, "fromEnvironment": { "x-embedding-api-key": "x_embedding_api_key", @@ -59,7 +59,6 @@ "meta": { "name": "jinaAI-vectorize", "tags": [ - "disabled" ] }, "fromEnvironment": { From 2549bb9f2d41a136f22d05076ae6bc600fb61549 Mon Sep 17 00:00:00 2001 From: Aaron Morton Date: Wed, 8 Apr 2026 10:35:00 +1200 Subject: [PATCH 15/89] WIP --- .../jsonapi/api/v1/vectorize/TestPlan.java | 50 ++++++++++++++++++- .../v1/vectorize/VectorizeTestFactory.java | 7 ++- .../api/v1/vectorize/testspec/Job.java | 12 +---- .../v1/vectorize/testspec/TestEnvAccess.java | 21 ++++++++ 4 files changed, 76 insertions(+), 14 deletions(-) create mode 100644 src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/TestEnvAccess.java diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestPlan.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestPlan.java index 1476fdf3d4..e99562db8a 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestPlan.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestPlan.java @@ -1,5 +1,7 @@ package io.stargate.sgv2.jsonapi.api.v1.vectorize; +import com.fasterxml.jackson.databind.MapperFeature; +import com.fasterxml.jackson.databind.ObjectMapper; import io.stargate.sgv2.jsonapi.api.v1.vectorize.targets.Target; import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestRunEnv; import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.Job; @@ -8,7 +10,10 @@ import org.junit.jupiter.api.DynamicContainer; import org.junit.jupiter.api.DynamicNode; +import java.io.IOException; +import java.nio.file.Path; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.function.Supplier; @@ -18,17 +23,36 @@ public record TestPlan(Target target, SpecFiles specFiles, Set workflows, boolean ignoreDisabled){ + private static final ObjectMapper MAPPER = new ObjectMapper() + .configure(MapperFeature.ACCEPT_CASE_INSENSITIVE_ENUMS, true); + + public static TestPlanContext fromFile(String path) { + + TestPlanFile planFile; + try { + planFile = MAPPER.readValue(Path.of(path).toFile(), TestPlanFile.class); + } catch (IOException e) { + throw new RuntimeException(e); + } + + var testPlan = create(planFile.target(), planFile.workflows(), planFile.ignoreDisabled()); + return new TestPlanContext(testPlan, planFile); + } public static TestPlan create(String targetName, List workflows){ return create(targetName, workflows, true); } - public static TestPlan create(String targetName, List workflows, boolean ignoreDisabled){ + public static TestPlan create(String targetName, List workflows, Boolean ignoreDisabled){ var targetConfigs = TargetsSpec.loadAll("integration-tests/targets/targets.json"); var target = new Target(targetConfigs.configuration(targetName)); var specFiles = SpecFiles.loadAll(List.of("integration-tests/vectorize", "integration-tests/assertions/assertion-templates.json")); - return new TestPlan(target, specFiles, Set.copyOf(workflows), ignoreDisabled); + return new TestPlan( + target, + specFiles, + workflows == null ? Set.of() : Set.copyOf(workflows), + ignoreDisabled == null || ignoreDisabled); } public Stream selectedWorkflows(){ @@ -116,4 +140,26 @@ public Stream addLifecycle(TestUri.Builder uriBuilder, Te lifecycleNodes( afterUriBuilder, "After TestSuite", testSuite.meta(), () -> target.afterTestSuite(this, afterUriBuilder,testSuite, environment))) ); } + + public record TestPlanFile( + String target, + List workflows, + Boolean ignoreDisabled, + Map envVars + ) { } + + public record TestPlanContext( + TestPlan testPlan, + TestPlanFile testPlanFile + ) implements AutoCloseable { + + public TestPlanContext{ + testPlanFile.envVars.forEach(System::setProperty); + } + + @Override + public void close() { + testPlanFile.envVars.keySet().forEach(System::clearProperty); + } + } } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/VectorizeTestFactory.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/VectorizeTestFactory.java index 19219119a1..c2def7951a 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/VectorizeTestFactory.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/VectorizeTestFactory.java @@ -12,12 +12,15 @@ import java.util.stream.Stream; public class VectorizeTestFactory { + private static final Logger LOGGER = LoggerFactory.getLogger(VectorizeTestFactory.class); + @TestFactory Stream jobs() { - var testPlan = TestPlan.create("local", List.of("all-vectorize-workflow")); +// var testPlan = TestPlan.create("local", List.of("all-vectorize-workflow")); + var testContext = TestPlan.fromFile("/Users/aaron.morton/code/stargate/jsonapi/.env/test-plans/dev.yaml"); - return testPlan.testNode(); + return testContext.testPlan().testNode(); } } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/Job.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/Job.java index cbc8280b46..6269d52322 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/Job.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/Job.java @@ -58,11 +58,7 @@ public TestRunEnv withoutMatrix(TestPlan testPlan) { var fromEnv = new TestRunEnv(); for (Map.Entry entry : fromEnvironment.entrySet()) { - var value = System.getenv(entry.getValue()); - if (value== null) { - throw new RuntimeException("Environment variable " + entry.getValue() + " is undefined"); - } - fromEnv.put(entry.getKey(), value); + fromEnv.put(entry.getKey(), TestEnvAccess.getEnvVar(entry.getKey())); } var fromVariables = new TestRunEnv(variables); @@ -74,11 +70,7 @@ public List allEnvironments(TestPlan testPlan) { var fromEnv = new TestRunEnv(); for (Map.Entry entry : fromEnvironment.entrySet()) { - var value = System.getenv(entry.getValue()); - if (value== null) { - throw new RuntimeException("Environment variable " + entry.getValue() + " is undefined"); - } - fromEnv.put(entry.getKey(), value); + fromEnv.put(entry.getKey(), TestEnvAccess.getEnvVar(entry.getKey())); } var fromVariables = new TestRunEnv(variables); diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/TestEnvAccess.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/TestEnvAccess.java new file mode 100644 index 0000000000..b247b11a5d --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/TestEnvAccess.java @@ -0,0 +1,21 @@ +package io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec; + +public abstract class TestEnvAccess { + + public static void putEnvVar(String key, String value) { + System.getProperties().put(key, value); + } + + public static String getEnvVar(String varName) { + + var value = System.getProperty(varName); + if (value != null){ + return value; + } + value = System.getenv(varName); + if (value== null) { + throw new RuntimeException("Environment variable not found in System Properties or Environment. varName=" + varName); + } + return value; + } +} From cb246f8ffcc2dc0e487903d79bf4f901726be5af Mon Sep 17 00:00:00 2001 From: Tatu Saloranta Date: Thu, 9 Apr 2026 16:47:23 -0700 Subject: [PATCH 16/89] Fix #2447: add Maven plug-in for tree-based console output --- pom.xml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/pom.xml b/pom.xml index a6c6afd5d7..913afa9ecb 100644 --- a/pom.xml +++ b/pom.xml @@ -356,6 +356,13 @@ maven-surefire-plugin ${surefire-plugin.version} + + + me.fabriciorby + maven-surefire-junit5-tree-reporter + 1.5.1 + + @{argLine} -Xmx4g -javaagent:${settings.localRepository}/org/mockito/mockito-core/${mockito.version}/mockito-core-${mockito.version}.jar + + plain + + true + + org.jboss.logmanager.LogManager ${maven.home} From af073210e309f45b1643c133036eac610da1ab25 Mon Sep 17 00:00:00 2001 From: Tatu Saloranta Date: Thu, 9 Apr 2026 16:59:25 -0700 Subject: [PATCH 17/89] Downgrde surefire plugin version --- pom.xml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 913afa9ecb..c05ddb1869 100644 --- a/pom.xml +++ b/pom.xml @@ -54,7 +54,11 @@ false ${skipTests} - 3.5.4 + + 3.5.3 false From c80b574f572db233f3c75ac452c6204ab84594f1 Mon Sep 17 00:00:00 2001 From: Tatu Saloranta Date: Thu, 9 Apr 2026 17:03:40 -0700 Subject: [PATCH 18/89] theme -> UNICODE --- pom.xml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index c05ddb1869..7f71ff0e73 100644 --- a/pom.xml +++ b/pom.xml @@ -379,7 +379,9 @@ true - + + UNICODE + org.jboss.logmanager.LogManager ${maven.home} From 97fb5100d6aa56e8b00c4c2f59707a299c9a706b Mon Sep 17 00:00:00 2001 From: Tatu Saloranta Date: Thu, 9 Apr 2026 17:33:11 -0700 Subject: [PATCH 19/89] Try to retain color codes for CI --- .github/workflows/continuous-integration.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/continuous-integration.yaml b/.github/workflows/continuous-integration.yaml index 1089b6159d..9b8a906861 100644 --- a/.github/workflows/continuous-integration.yaml +++ b/.github/workflows/continuous-integration.yaml @@ -94,7 +94,7 @@ jobs: - name: Build & Test run: | - ./mvnw -B -ntp clean test + ./mvnw -B -ntp -Dstyle.color=always clean test # Publish baseline coverage for main branch (for future PR comparisons) - name: Publish baseline coverage @@ -382,7 +382,7 @@ jobs: # -Dquarkus.package.write-transformed-bytecode-to-build-output=true is required for JaCoCo IT coverage # Note: Container image build is disabled for coverage (JaCoCo only works with JAR artifacts) run: | - ./mvnw -B -ntp clean verify -DskipUnitTests -DRERANKING_CONFIG_RESOURCE=test-reranking-providers-config.yaml -DEMBEDDING_CONFIG_RESOURCE=test-embedding-providers-config.yaml -Drun-create-index-parallel=true -Pjacoco-it -Dquarkus.package.write-transformed-bytecode-to-build-output=true ${{ matrix.profile }} + ./mvnw -B -ntp clean verify -Dstyle.color=always -DskipUnitTests -DRERANKING_CONFIG_RESOURCE=test-reranking-providers-config.yaml -DEMBEDDING_CONFIG_RESOURCE=test-embedding-providers-config.yaml -Drun-create-index-parallel=true -Pjacoco-it -Dquarkus.package.write-transformed-bytecode-to-build-output=true ${{ matrix.profile }} # Publish baseline IT coverage for main branch (for future PR comparisons) - name: Publish baseline IT coverage From 9f29d2de484f2eebf9ad912b3d209bc84c85793c Mon Sep 17 00:00:00 2001 From: Aaron Morton Date: Fri, 10 Apr 2026 12:42:24 +1200 Subject: [PATCH 20/89] WIP --- .../jsonapi/api/v1/vectorize/TestPlan.java | 45 ++++- .../v1/vectorize/VectorizeTestFactory.java | 2 +- .../api/v1/vectorize/testspec/Job.java | 2 +- .../assertions/assertion-templates.json | 4 + .../findEmbeddingProviders-workflow.json | 30 +++ .../vectorize/findEmbeddingProviders.json | 19 ++ ...ow.json => vectorize-header-workflow.json} | 13 +- .../vectorize/vectorize-shared-auth.json | 40 ++++ .../vectorize/vectorize-shared-workflow.json | 188 ++++++++++++++++++ 9 files changed, 326 insertions(+), 17 deletions(-) create mode 100644 src/test/resources/integration-tests/vectorize/findEmbeddingProviders-workflow.json create mode 100644 src/test/resources/integration-tests/vectorize/findEmbeddingProviders.json rename src/test/resources/integration-tests/vectorize/{vectorize-workflow.json => vectorize-header-workflow.json} (96%) create mode 100644 src/test/resources/integration-tests/vectorize/vectorize-shared-auth.json create mode 100644 src/test/resources/integration-tests/vectorize/vectorize-shared-workflow.json diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestPlan.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestPlan.java index e99562db8a..65c0e15659 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestPlan.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestPlan.java @@ -1,7 +1,10 @@ package io.stargate.sgv2.jsonapi.api.v1.vectorize; +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.core.StreamReadFeature; import com.fasterxml.jackson.databind.MapperFeature; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; import io.stargate.sgv2.jsonapi.api.v1.vectorize.targets.Target; import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestRunEnv; import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.Job; @@ -23,19 +26,37 @@ public record TestPlan(Target target, SpecFiles specFiles, Set workflows, boolean ignoreDisabled){ - private static final ObjectMapper MAPPER = new ObjectMapper() + private static final ObjectMapper JSON_MAPPER = new ObjectMapper(JsonFactory.builder() + .enable(StreamReadFeature.INCLUDE_SOURCE_IN_LOCATION) + .build() + ) .configure(MapperFeature.ACCEPT_CASE_INSENSITIVE_ENUMS, true); + private static final ObjectMapper YAML_MAPPER = new ObjectMapper( + YAMLFactory.builder() + .enable(StreamReadFeature.INCLUDE_SOURCE_IN_LOCATION) + .build() + ) + .configure(MapperFeature.ACCEPT_CASE_INSENSITIVE_ENUMS, true); + public static TestPlanContext fromFile(String path) { TestPlanFile planFile; try { - planFile = MAPPER.readValue(Path.of(path).toFile(), TestPlanFile.class); + planFile = YAML_MAPPER.readValue(Path.of(path).toFile(), TestPlanFile.class); } catch (IOException e) { throw new RuntimeException(e); } - var testPlan = create(planFile.target(), planFile.workflows(), planFile.ignoreDisabled()); + if (planFile.targetName() != null && planFile.customTarget!= null){ + throw new RuntimeException("Both targetName and customTarget set, use only one. testPlanFile=" + path); + } + + var testPlan = planFile.customTarget() != null ? + create(planFile.customTarget(), planFile.workflows, planFile.ignoreDisabled) + : + create(planFile.targetName(), planFile.workflows(), planFile.ignoreDisabled()); + return new TestPlanContext(testPlan, planFile); } public static TestPlan create(String targetName, List workflows){ @@ -44,15 +65,19 @@ public static TestPlan create(String targetName, List workflows){ public static TestPlan create(String targetName, List workflows, Boolean ignoreDisabled){ var targetConfigs = TargetsSpec.loadAll("integration-tests/targets/targets.json"); - var target = new Target(targetConfigs.configuration(targetName)); + return create(targetConfigs.configuration(targetName), workflows, ignoreDisabled); + } + + public static TestPlan create(TargetConfiguration targetConfiguration, List workflows, Boolean ignoreDisabled){ + var target = new Target(targetConfiguration); var specFiles = SpecFiles.loadAll(List.of("integration-tests/vectorize", "integration-tests/assertions/assertion-templates.json")); return new TestPlan( - target, - specFiles, - workflows == null ? Set.of() : Set.copyOf(workflows), - ignoreDisabled == null || ignoreDisabled); + target, + specFiles, + workflows == null ? Set.of() : Set.copyOf(workflows), + ignoreDisabled == null || ignoreDisabled); } public Stream selectedWorkflows(){ @@ -142,7 +167,9 @@ public Stream addLifecycle(TestUri.Builder uriBuilder, Te } public record TestPlanFile( - String target, + String name, + String targetName, + TargetConfiguration customTarget, List workflows, Boolean ignoreDisabled, Map envVars diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/VectorizeTestFactory.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/VectorizeTestFactory.java index c2def7951a..015781b86e 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/VectorizeTestFactory.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/VectorizeTestFactory.java @@ -19,7 +19,7 @@ public class VectorizeTestFactory { Stream jobs() { // var testPlan = TestPlan.create("local", List.of("all-vectorize-workflow")); - var testContext = TestPlan.fromFile("/Users/aaron.morton/code/stargate/jsonapi/.env/test-plans/dev.yaml"); + var testContext = TestPlan.fromFile("/Users/amorton/code/stargate/jsonapi/.env/test-plans/test-plan-dev-smoketest-aws-us-east-1.yaml"); return testContext.testPlan().testNode(); } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/Job.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/Job.java index 6269d52322..cb10e7c22a 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/Job.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/Job.java @@ -70,7 +70,7 @@ public List allEnvironments(TestPlan testPlan) { var fromEnv = new TestRunEnv(); for (Map.Entry entry : fromEnvironment.entrySet()) { - fromEnv.put(entry.getKey(), TestEnvAccess.getEnvVar(entry.getKey())); + fromEnv.put(entry.getKey(), TestEnvAccess.getEnvVar(entry.getValue())); } var fromVariables = new TestRunEnv(variables); diff --git a/src/test/resources/integration-tests/assertions/assertion-templates.json b/src/test/resources/integration-tests/assertions/assertion-templates.json index 76bcea748e..f2961c8a87 100644 --- a/src/test/resources/integration-tests/assertions/assertion-templates.json +++ b/src/test/resources/integration-tests/assertions/assertion-templates.json @@ -37,6 +37,10 @@ "http.success": null, "response.isWriteSuccess": null }, + "findEmbeddingProviders": { + "http.success": null, + "response.isDDLSuccess": null + }, "findOneAndDelete": { "http.success": null, "response.isFindAndSuccess": null diff --git a/src/test/resources/integration-tests/vectorize/findEmbeddingProviders-workflow.json b/src/test/resources/integration-tests/vectorize/findEmbeddingProviders-workflow.json new file mode 100644 index 0000000000..e71581803b --- /dev/null +++ b/src/test/resources/integration-tests/vectorize/findEmbeddingProviders-workflow.json @@ -0,0 +1,30 @@ +{ + "meta": { + "name": "findEmbeddingProviders-workflow", + "kind": "workflow" + }, + "jobs": [ + { + "meta": { + "name": "findEmbeddingProviders", + "tags": [ + ] + }, + "fromEnvironment": { + "Token": "Token" + }, + "variables": { + }, + "matrix": { + "filterModelStatus": [ + "SUPPORTED", + "DEPRECATED", + "END_OF_LIFE" + ] + }, + "tests": [ + "findEmbeddingProviders" + ] + } + ] +} \ No newline at end of file diff --git a/src/test/resources/integration-tests/vectorize/findEmbeddingProviders.json b/src/test/resources/integration-tests/vectorize/findEmbeddingProviders.json new file mode 100644 index 0000000000..b8b2cdf0f0 --- /dev/null +++ b/src/test/resources/integration-tests/vectorize/findEmbeddingProviders.json @@ -0,0 +1,19 @@ +{ + "meta": { + "name": "findEmbeddingProviders", + "kind" : "test_suite" + }, + "setup": [ + ], + "tests": [ + { + "name": "findEmbeddingProviders", + "command": { + "findEmbeddingProviders": {} + }, + "asserts": { + "Templated.isSuccess": null + } + } + ] +} \ No newline at end of file diff --git a/src/test/resources/integration-tests/vectorize/vectorize-workflow.json b/src/test/resources/integration-tests/vectorize/vectorize-header-workflow.json similarity index 96% rename from src/test/resources/integration-tests/vectorize/vectorize-workflow.json rename to src/test/resources/integration-tests/vectorize/vectorize-header-workflow.json index a20b56e789..d1cb43fde4 100644 --- a/src/test/resources/integration-tests/vectorize/vectorize-workflow.json +++ b/src/test/resources/integration-tests/vectorize/vectorize-header-workflow.json @@ -1,16 +1,18 @@ { "meta": { - "name": "all-vectorize-workflow", + "name": "vectorize-header-workflow", "kind": "workflow" }, "jobs": [ { "meta": { "name": "open-ai-vectorize", - "tags": ["disabled"] + "tags": [ + "disabled" + ] }, "fromEnvironment": { - "x-embedding-api-key": "x_embedding_api_key", + "x-embedding-api-key": "openAi_KEY", "Token": "Token" }, "variables": { @@ -31,7 +33,7 @@ { "meta": { "name": "voyageAI-vectorize", - "tags": [ + "tags": ["disabled" ] }, "fromEnvironment": { @@ -58,7 +60,7 @@ { "meta": { "name": "jinaAI-vectorize", - "tags": [ + "tags": ["disabled" ] }, "fromEnvironment": { @@ -163,7 +165,6 @@ "meta": { "name": "nvidia-vectorize", "tags": [ - "disabled" ] }, "fromEnvironment": { diff --git a/src/test/resources/integration-tests/vectorize/vectorize-shared-auth.json b/src/test/resources/integration-tests/vectorize/vectorize-shared-auth.json new file mode 100644 index 0000000000..eda94d81c9 --- /dev/null +++ b/src/test/resources/integration-tests/vectorize/vectorize-shared-auth.json @@ -0,0 +1,40 @@ +{ + "meta": { + "name": "vectorize-shared-auth", + "kind": "test_suite" + }, + "setup": [ + { + "createCollection": { + "name": "${COLLECTION_NAME}", + "options": { + "vector": { + "metric": "cosine", + "service": { + "provider": "${PROVIDER}", + "modelName": "${MODEL}", + "authentication":{ + "providerKey": "${CREDENTIAL}.providerKey" + } + } + } + } + } + }, + { + "$include": "vectorize-base" + } + ], + "tests": [ + { + "$include": "vectorize-base" + } + ], + "cleanup": [ + { + "deleteCollection": { + "name": "${COLLECTION_NAME}" + } + } + ] +} \ No newline at end of file diff --git a/src/test/resources/integration-tests/vectorize/vectorize-shared-workflow.json b/src/test/resources/integration-tests/vectorize/vectorize-shared-workflow.json new file mode 100644 index 0000000000..bc57265f78 --- /dev/null +++ b/src/test/resources/integration-tests/vectorize/vectorize-shared-workflow.json @@ -0,0 +1,188 @@ +{ + "meta": { + "name": "vectorize-shared-workflow", + "kind": "workflow" + }, + "jobs": [ + { + "meta": { + "name": "open-ai-vectorize", + "tags": [] + }, + "fromEnvironment": { + "x-embedding-api-key": "openAi_KEY", + "Token": "Token" + }, + "variables": { + "PROVIDER": "openai", + "COLLECTION_NAME": "${PROVIDER}-${MODEL}", + "CREDENTIAL" : "kent-open-ai-key" + }, + "matrix": { + "MODEL": [ + "text-embedding-3-small", + "text-embedding-3-large", + "text-embedding-ada-002" + ] + }, + "tests": [ + "vectorize-shared-auth" + ] + }, + { + "meta": { + "name": "voyageAI-vectorize", + "tags": ["disabled" + ] + }, + "fromEnvironment": { + "x-embedding-api-key": "voyageAI_KEY", + "Token": "Token" + }, + "variables": { + "PROVIDER": "voyageAI", + "COLLECTION_NAME": "${PROVIDER}-${MODEL}" + }, + "matrix": { + "MODEL": [ + "voyage-large-2-instruct", + "voyage-law-2", + "voyage-code-2", + "voyage-large-2", + "voyage-2" + ] + }, + "tests": [ + "vectorize-header-auth" + ] + }, + { + "meta": { + "name": "jinaAI-vectorize", + "tags": ["disabled" + ] + }, + "fromEnvironment": { + "x-embedding-api-key": "jinaAI_KEY", + "Token": "Token" + }, + "variables": { + "PROVIDER": "jinaAI", + "COLLECTION_NAME": "${PROVIDER}-${MODEL}" + }, + "matrix": { + "MODEL": [ + "jina-embeddings-v2-base-en", + "jina-embeddings-v2-base-de", + "jina-embeddings-v2-base-es", + "jina-embeddings-v2-base-code", + "jina-embeddings-v2-base-zh" + ] + }, + "tests": [ + "vectorize-header-auth" + ] + }, + { + "meta": { + "name": "huggingface-non-dedicated-vectorize", + "tags": [ + "disabled" + ] + }, + "fromEnvironment": { + "x-embedding-api-key": "huggingface_KEY", + "Token": "Token" + }, + "variables": { + "PROVIDER": "huggingface", + "COLLECTION_NAME": "${PROVIDER}-${MODEL}" + }, + "matrix": { + "MODEL": [ + "sentence-transformers/all-MiniLM-L6-v2", + "intfloat/multilingual-e5-large", + "intfloat/multilingual-e5-large-instruct", + "BAAI/bge-small-en-v1.5", + "BAAI/bge-base-en-v1.5", + "BAAI/bge-large-en-v1.5" + ] + }, + "tests": [ + "vectorize-header-auth" + ] + }, + { + "meta": { + "name": "mistral-vectorize", + "tags": [ + "disabled" + ] + }, + "fromEnvironment": { + "x-embedding-api-key": "mistral_KEY", + "Token": "Token" + }, + "variables": { + "PROVIDER": "mistral", + "COLLECTION_NAME": "${PROVIDER}-${MODEL}" + }, + "matrix": { + "MODEL": [ + "mistral-embed" + ] + }, + "tests": [ + "vectorize-header-auth" + ] + }, + { + "meta": { + "name": "upstageAI-vectorize", + "tags": [ + "disabled" + ] + }, + "fromEnvironment": { + "x-embedding-api-key": "upstageAI_KEY", + "Token": "Token" + }, + "variables": { + "PROVIDER": "upstageAI", + "COLLECTION_NAME": "${PROVIDER}-${MODEL}" + }, + "matrix": { + "MODEL": [ + "solar-embedding-1-large" + ] + }, + "tests": [ + "vectorize-header-auth" + ] + }, + { + "meta": { + "name": "nvidia-vectorize", + "tags": [ + "disabled" + ] + }, + "fromEnvironment": { + "x-embedding-api-key": "Token", + "Token": "Token" + }, + "variables": { + "PROVIDER": "nvidia", + "COLLECTION_NAME": "${PROVIDER}-${MODEL}" + }, + "matrix": { + "MODEL": [ + "NV-Embed-QA" + ] + }, + "tests": [ + "vectorize-header-auth" + ] + } + ] +} \ No newline at end of file From 5899b833103d6df2fe3ead6254fd5e94b0c4db23 Mon Sep 17 00:00:00 2001 From: Aaron Morton Date: Fri, 17 Apr 2026 09:59:36 +1200 Subject: [PATCH 21/89] testing GH workflow --- .github/workflows/test-bench-run.yaml | 99 ++++++ pom.xml | 12 + .../api/v1/vectorize/DynamicTreeListener.java | 311 ++++++++++++++++++ .../v1/vectorize/TestBenchConsoleWriter.java | 150 +++++++++ .../api/v1/vectorize/TestBenchTests.java | 39 +++ .../jsonapi/api/v1/vectorize/TestPlan.java | 227 ++++++++----- .../api/v1/vectorize/VectorizeAstra.java | 48 ++- .../jsonapi/api/v1/vectorize/VectorizeIT.java | 6 +- .../v1/vectorize/VectorizeTestFactory.java | 26 -- .../assertions/AssertionFactory.java | 25 +- .../assertions/AssertionFactoryRegistry.java | 16 +- .../assertions/AssertionMatcher.java | 5 +- .../vectorize/assertions/AssertionName.java | 16 +- .../vectorize/assertions/BodyAssertion.java | 13 +- .../DescribableAssertionMatcher.java | 10 +- .../v1/vectorize/assertions/Documents.java | 14 +- .../api/v1/vectorize/assertions/Http.java | 10 +- .../api/v1/vectorize/assertions/Response.java | 39 +-- .../assertions/SingleTestAssertion.java | 49 ++- .../api/v1/vectorize/assertions/Status.java | 5 +- .../v1/vectorize/assertions/Templated.java | 17 +- .../vectorize/assertions/TestAssertion.java | 69 ++-- .../assertions/TestAssertionContainer.java | 45 ++- .../lifecycle/TestPlanLifecycle.java | 29 +- .../v1/vectorize/messaging/APIRequest.java | 59 ++-- .../v1/vectorize/messaging/APIResponse.java | 4 +- .../api/v1/vectorize/targets/Backend.java | 7 +- .../vectorize/targets/CassandraBackend.java | 55 ++-- .../api/v1/vectorize/targets/Connection.java | 6 +- .../api/v1/vectorize/targets/Target.java | 63 ++-- .../testrun/DynamicTestExecutable.java | 30 +- .../api/v1/vectorize/testrun/TestRunEnv.java | 71 ++-- .../v1/vectorize/testrun/TestRunRequest.java | 41 ++- .../v1/vectorize/testrun/TestRunResponse.java | 53 ++- .../api/v1/vectorize/testrun/TestUri.java | 60 ++-- .../testspec/AssertionTemplateSpec.java | 13 +- .../api/v1/vectorize/testspec/Job.java | 58 ++-- .../api/v1/vectorize/testspec/SpecFile.java | 10 +- .../api/v1/vectorize/testspec/SpecFiles.java | 83 +++-- .../testspec/TargetConfiguration.java | 3 +- .../v1/vectorize/testspec/TargetsSpec.java | 22 +- .../api/v1/vectorize/testspec/TestCase.java | 25 +- .../v1/vectorize/testspec/TestCommand.java | 15 +- .../v1/vectorize/testspec/TestEnvAccess.java | 7 +- .../api/v1/vectorize/testspec/TestSpec.java | 3 +- .../v1/vectorize/testspec/TestSpecKind.java | 16 +- .../v1/vectorize/testspec/TestSpecMeta.java | 7 +- .../v1/vectorize/testspec/TestSuiteSpec.java | 76 +++-- .../v1/vectorize/testspec/WorkflowSpec.java | 34 +- ...it.platform.launcher.TestExecutionListener | 1 + src/test/resources/application.yaml | 9 + .../test-plans/vectorize-astra-test-plan.yaml | 12 + .../vectorize/vectorize-header-workflow.json | 4 +- .../vectorize/vectorize-shared-workflow.json | 24 -- 54 files changed, 1385 insertions(+), 766 deletions(-) create mode 100644 .github/workflows/test-bench-run.yaml create mode 100644 src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/DynamicTreeListener.java create mode 100644 src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestBenchConsoleWriter.java create mode 100644 src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestBenchTests.java delete mode 100644 src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/VectorizeTestFactory.java create mode 100644 src/test/resources/META-INF/services/org.junit.platform.launcher.TestExecutionListener create mode 100644 src/test/resources/integration-tests/test-plans/vectorize-astra-test-plan.yaml diff --git a/.github/workflows/test-bench-run.yaml b/.github/workflows/test-bench-run.yaml new file mode 100644 index 0000000000..2563420ac2 --- /dev/null +++ b/.github/workflows/test-bench-run.yaml @@ -0,0 +1,99 @@ +name: API Test Bench Run + +on: + workflow_dispatch: + inputs: + environment: + description: 'Environment to test against' + required: true + type: environment + pull_request: + +# needed when a workflow wants to use OIDC (OpenID Connect) to authenticate to cloud +permissions: + id-token: write + contents: write + packages: write + pull-requests: write + +# global env vars, available in all jobs and steps +env: + MAVEN_OPTS: '-Xmx4g' + DS_ARTIFACTORY_USERNAME: ${{ secrets.DS_ARTIFACTORY_USERNAME }} + DS_ARTIFACTORY_PASSWORD: ${{ secrets.DS_ARTIFACTORY_PASSWORD }} + +jobs: + setup: + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.set-matrix.outputs.TEST_TARGETS }} + steps: + - id: set-matrix + run: echo "TEST_TARGETS=${{ secrets.TEST_TARGETS }}" >> $GITHUB_OUTPUT + + # runs unit tests + build: + name: Unit tests + runs-on: ubuntu-latest + + # max run time 12 minutes + timeout-minutes: 12 + + strategy: + matrix: + include: ${{ fromJson(needs.setup.outputs.matrix) }} + + steps: + - uses: actions/checkout@v6 + + - name: Set up JDK 21 + uses: actions/setup-java@v5 + with: + distribution: 'temurin' + java-version: '21' + cache: maven + + - name: Setup Maven + run: | + mkdir -p ~/.m2 + cat < ~/.m2/settings.xml + + + + stargate-central + ${DS_ARTIFACTORY_USERNAME} + ${DS_ARTIFACTORY_PASSWORD} + + + stargate-snapshots + ${DS_ARTIFACTORY_USERNAME} + ${DS_ARTIFACTORY_PASSWORD} + + + artifactory + ${DS_ARTIFACTORY_USERNAME} + ${DS_ARTIFACTORY_PASSWORD} + + + artifactory-snapshots + ${DS_ARTIFACTORY_USERNAME} + ${DS_ARTIFACTORY_PASSWORD} + + + artifactory-releases + ${DS_ARTIFACTORY_USERNAME} + ${DS_ARTIFACTORY_PASSWORD} + + + + EOF + - name: Build & Test + env: + TEST_PLAN_FILE: classpath:integration-tests/test-plans/vectorize-astra-test-plan.yaml + Token: ${{ matrix.token }} + TARGET_NAME: ${{ matrix.name }} + ENDPOINT: ${{ matrix.endpoint }} + openAi_KEY: ${{ secrets.OPEN_AI_KEY }} + + run: | + ./mvnw -B -ntp verify -Dfmt.skip -DskipUnitTests -Dit.test=TestBenchTests \ No newline at end of file diff --git a/pom.xml b/pom.xml index 205677c88f..bd037882c4 100644 --- a/pom.xml +++ b/pom.xml @@ -350,6 +350,18 @@ quarkus-mcp-server-test test + + org.apache.maven.plugins + maven-surefire-plugin + ${surefire-plugin.version} + test + + + me.fabriciorby + maven-surefire-junit5-tree-reporter + 1.5.1 + test + diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/DynamicTreeListener.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/DynamicTreeListener.java new file mode 100644 index 0000000000..ce3a192183 --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/DynamicTreeListener.java @@ -0,0 +1,311 @@ +package io.stargate.sgv2.jsonapi.api.v1.vectorize; + +import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestUri; +import java.util.*; +import java.util.concurrent.ConcurrentHashMap; +import org.apache.maven.plugin.surefire.report.*; +import org.junit.platform.engine.TestExecutionResult; +import org.junit.platform.engine.UniqueId; +import org.junit.platform.engine.support.descriptor.UriSource; +import org.junit.platform.launcher.TestExecutionListener; +import org.junit.platform.launcher.TestIdentifier; +import org.junit.platform.launcher.TestPlan; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class DynamicTreeListener implements TestExecutionListener { + + private static final Logger LOGGER = LoggerFactory.getLogger(DynamicTreeListener.class); + + private TestTracker rootTracker; + private final Map testTrackers = new ConcurrentHashMap<>(); + + private final Map startTimes = new ConcurrentHashMap<>(); + private final Map containerStats = new ConcurrentHashMap<>(); + + private final TestBenchConsoleWriter writer = new TestBenchConsoleWriter(); + + @Override + public void testPlanExecutionStarted(TestPlan testPlan) {} + + @Override + public void testPlanExecutionFinished(TestPlan testPlan) { + writer.printTestPlanCompleted(rootTracker); + } + + @Override + public void dynamicTestRegistered(TestIdentifier testIdentifier) {} + + @Override + public void executionStarted(TestIdentifier id) { + if (!isTestBenchNode(id)) { + return; + } + // System.out.println("XXX " + id.getDisplayName()); + var tracker = getCreateTestTracker(id); + if (tracker == null) { + return; + } + // System.out.println("YYY " + tracker.runUri); + // System.out.println("ZZZ " + tracker.runUri.leafType()); + writer.printTestStarted(tracker); + } + + @Override + public void executionFinished(TestIdentifier id, TestExecutionResult result) { + var tracker = getCreateTestTracker(id); + if (tracker == null) { + return; + } + + tracker.executionFinished(result); + } + + @Override + public void executionSkipped(TestIdentifier id, String reason) { + var tracker = getCreateTestTracker(id); + if (tracker == null) { + return; + } + + tracker.executionSkipped(); + } + + private static boolean isTestBenchNode(TestIdentifier testIdentifier) { + return testIdentifier + .getUniqueId() + .startsWith("[engine:junit-jupiter]/[class:io.stargate.sgv2.jsonapi."); + } + + // private String buildClassName(TestIdentifier id) { + // List parts = new ArrayList<>(); + // Optional current = Optional.of(id); + // while (current.isPresent()) { + // TestIdentifier node = current.get(); + // if (isEngineOrClass(node)) { + // break; + // } + // parts.add(0, node.getDisplayName()); + // current = testPlan.getParent(node); + // } + // return rootClassName(id) + (parts.isEmpty() ? "" : "$" + String.join("$", parts)); + // } + // + // private String rootClassName(TestIdentifier id) { + // Optional current = Optional.of(id); + // while (current.isPresent()) { + // TestIdentifier node = current.get(); + // if (isClassNode(node)) { + // return node.getDisplayName(); + // } + // current = testPlan.getParent(node); + // } + // return "Unknown"; + // } + // + // private SimpleReportEntry toReportEntry(TestIdentifier id, String methodName, String + // methodDisplay) { + // return new SimpleReportEntry( + // RunMode.NORMAL_RUN, + // System.currentTimeMillis(), + // buildClassName(id), + // id.getDisplayName(), + // methodName, + // methodDisplay + // ); + // } + // + // private boolean isEngineOrClass(TestIdentifier id) { + // return id.getUniqueId().startsWith("[engine:") || isClassNode(id); + // } + // + // private boolean isClassNode(TestIdentifier id) { + // return id.getUniqueId().contains("[class:"); + // } + // + // private long elapsed(TestIdentifier id) { + // return System.currentTimeMillis() - startTimes.getOrDefault(id.getUniqueId(), + // System.currentTimeMillis()); + // } + // + // private Optional nearestContainerStats(TestIdentifier id) { + // Optional current = testPlan.getParent(id); + // while (current.isPresent()) { + // if (containerStats.containsKey(current.get().getUniqueId())) { + // return Optional.of(containerStats.get(current.get().getUniqueId())); + // } + // current = testPlan.getParent(current.get()); + // } + // return Optional.empty(); + // } + + private TestTracker getCreateTestTracker(TestIdentifier testIdentifier) { + + var existingTracker = testTrackers.get(testIdentifier.getUniqueIdObject()); + if (existingTracker != null) { + return existingTracker; + } + + // if this is not a TESTRUN:// it is not a node we care about + var testUri = + testIdentifier + .getSource() + .map( + testSource -> { + if (testSource instanceof UriSource uriSource) { + return TestUri.parse(uriSource.getUri()).orElse(null); + } + return null; + }); + if (testUri.isEmpty()) { + return null; + } + + // The TARGET is the top level item, everything else should have a parent + var parentTracker = + testUri.get().leafType() != TestUri.Segment.TARGET + ? Objects.requireNonNull( + testTrackers.get(testIdentifier.getParentIdObject().get()), + "parentID not found for testIdentifier: " + testIdentifier.toString()) + : null; + + var tracker = new TestTracker(testIdentifier, testUri.get(), parentTracker); + + if (rootTracker == null) { + rootTracker = tracker; + } + testTrackers.put(tracker.identifier.getUniqueIdObject(), tracker); + return tracker; + } + + /** */ + public class TestTracker { + + private final TestIdentifier identifier; + private final TestUri runUri; + private final TestTracker parent; + private final int depth; + + private final List children = new ArrayList<>(); + + private TestExecutionResult.Status junitStatus; + private final TestContainerStats stats; + + public TestTracker(TestIdentifier identifier, TestUri runUri, TestTracker parent) { + this.identifier = identifier; + this.runUri = runUri; + this.parent = parent; + this.depth = parent == null ? 0 : parent.depth + 1; + + this.stats = identifier.isContainer() ? new TestContainerStats() : null; + if (parent != null) { + parent.children.add(this); + } + } + + public TestIdentifier identifier() { + return identifier; + } + + public TestUri runUri() { + return runUri; + } + + public TestTracker parent() { + return parent; + } + + public List children() { + return children; + } + + public int depth() { + return depth; + } + + public TestContainerStats stats() { + return stats; + } + + public void executionFinished(TestExecutionResult result) { + junitStatus = result.getStatus(); + if (stats != null) { + stats.testCompleted(result); + } + if (parent != null) { + parent.executionFinished(result); + } + } + + public void executionSkipped() { + if (stats != null) { + stats.testSkipped(); + } + if (parent != null) { + parent.executionSkipped(); + } + } + } + + /** Modeled on org.apache.maven.plugin.surefire.report.TestSetStats */ + public class TestContainerStats { + + private TestContainerStats parent; + + private final long startedAtMillis; + private long lastFinishedAtMillis; + + private int successful; + + private int aborted; + + private int failures; + + private int skipped; + + public TestContainerStats() { + this.startedAtMillis = System.currentTimeMillis(); + } + + public long elapsedMillis() { + return lastFinishedAtMillis == 0 + ? System.currentTimeMillis() - startedAtMillis + : lastFinishedAtMillis - startedAtMillis; + } + + public int successful() { + return successful; + } + + public int aborted() { + return aborted; + } + + public int failures() { + return failures; + } + + public int skipped() { + return skipped; + } + + public boolean noErrors() { + return aborted == 0 && failures == 0; + } + + public void testCompleted(TestExecutionResult result) { + lastFinishedAtMillis = System.currentTimeMillis(); + + switch (result.getStatus()) { + case FAILED -> failures++; + case ABORTED -> aborted++; + case SUCCESSFUL -> successful++; + } + } + + public void testSkipped() { + lastFinishedAtMillis = System.currentTimeMillis(); + skipped++; + } + } +} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestBenchConsoleWriter.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestBenchConsoleWriter.java new file mode 100644 index 0000000000..6b87fdee13 --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestBenchConsoleWriter.java @@ -0,0 +1,150 @@ +package io.stargate.sgv2.jsonapi.api.v1.vectorize; + +import static org.apache.maven.surefire.shared.utils.logging.MessageUtils.buffer; + +import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestUri; +import org.apache.maven.plugin.surefire.report.Theme; +import org.apache.maven.surefire.shared.utils.logging.MessageBuilder; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class TestBenchConsoleWriter { + private static final Logger LOGGER = LoggerFactory.getLogger(TestBenchConsoleWriter.class); + + private boolean firstLine = true; + + private final Theme theme; + + public TestBenchConsoleWriter() { + this.theme = Theme.EMOJI; + } + + /** + * We print the test has starting , so the output from the test comes after, we then output the + * summary later. + * + * @param tracker + */ + public void printTestStarted(DynamicTreeListener.TestTracker tracker) { + + var buffer = buffer(); + if (firstLine) { + + firstLine = false; + buffer + .newline() + .a(theme.dash().repeat(20)) + .newline() + .a("Running Test Bench, summary shown at completion...") + .newline() + .a(theme.dash().repeat(20)) + .newline(); + } + + // [INFO] ── io.stargate.sgv2.jsonapi.api.model.command.impl.InsertManyCommandTest - + // 1.316 s + // [INFO] ├─ ✔ noDocuments - 0.274 s + + // if there is no parent + if (tracker.parent() == null) { + buffer.a(theme.dash()).strong(tracker.identifier().getDisplayName()); + } else { + buffer + .a(theme.blank().repeat(tracker.depth() - 1)) + .a(theme.entry()) + .strong(tracker.identifier().getDisplayName()); + } + LOGGER.info(buffer.toString()); + } + + public void printTestPlanCompleted(DynamicTreeListener.TestTracker rootTracker) { + + var buffer = buffer(); + + buffer + .newline() + .a(theme.dash().repeat(20)) + .newline() + .a("Test Bench Summary") + .newline() + .a(theme.dash().repeat(20)) + .newline(); + + // [INFO] ── io.stargate.sgv2.jsonapi.api.model.command.impl.InsertManyCommandTest - + // 1.316 s + // [INFO] ├─ ✔ noDocuments - 0.274 s + + // if there is no parent + // if (tracker.parent() == null){ + // buffer.a(theme.dash()) + // .strong(tracker.identifier().getDisplayName()); + // } + // else { + // buffer.a(theme.blank().repeat(tracker.depth() -1)) + // .a(theme.entry()) + // .strong(tracker.identifier().getDisplayName()); + // } + writeCompletedSummary(buffer, rootTracker, true); + LOGGER.info(buffer.toString()); + } + + /** + * TestPlan: smoketest-aws-us-east-1 on astra workflows vectorize-header-workflow Workflow: + * vectorize-header-workflow Job: nvidia-vectorize TestSuite: vectorize-header-auth TestEnv: + * [MODEL=NV-Embed-QA, PROVIDER=nvidia] RESULTS.... TestEnv: [MODEL=nvidia/nv-embedqa-e5-v5, + * PROVIDER=nvidia] RESULTS... + * + * @param buffer + * @param tracker + * @param isRoot + */ + private void writeCompletedSummary( + MessageBuilder buffer, DynamicTreeListener.TestTracker tracker, boolean isRoot) { + + // the tree part + if (isRoot) { + buffer.a(theme.dash()); + } else { + buffer.a(theme.blank().repeat(tracker.depth() - 1)).a(theme.entry()); + } + + var timing = + tracker.stats() == null ? "" : " - %s s".formatted(tracker.stats().elapsedMillis() / 1000); + // name of the group + if (tracker.stats().noErrors()) { + buffer.a(theme.successful()); + } else { + buffer.a(theme.failed()); + } + + buffer.strong(tracker.identifier().getDisplayName()).a(timing).newline(); + + // If we have a TestEnv then we want to write out the summary of results for it, otherwise + // descend until we get one + if (tracker.runUri().leafType() == TestUri.Segment.ENV) { + + buffer + .a(theme.blank().repeat(tracker.depth())) + .a(theme.details()) + .a("Successful: " + tracker.stats().successful()) + .newline(); + buffer + .a(theme.blank().repeat(tracker.depth())) + .a(theme.details()) + .a("Failures: " + tracker.stats().failures()) + .newline(); + buffer + .a(theme.blank().repeat(tracker.depth())) + .a(theme.details()) + .a("Aborted: " + tracker.stats().aborted()) + .newline(); + buffer + .a(theme.blank().repeat(tracker.depth())) + .a(theme.details()) + .a("Skipped: " + tracker.stats().skipped()) + .newline(); + } else { + tracker.children().forEach(child -> writeCompletedSummary(buffer, child, false)); + } + } +} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestBenchTests.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestBenchTests.java new file mode 100644 index 0000000000..dddd0fb048 --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestBenchTests.java @@ -0,0 +1,39 @@ +package io.stargate.sgv2.jsonapi.api.v1.vectorize; + +import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.TargetsSpec; +import java.nio.file.Path; +import java.util.stream.Stream; +import org.junit.jupiter.api.DynamicContainer; +import org.junit.jupiter.api.TestFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class TestBenchTests { + + private static final Logger LOGGER = LoggerFactory.getLogger(TestBenchTests.class); + + @TestFactory + public Stream runTestPlanFile() { + + var rawPath = System.getenv("TEST_PLAN_FILE"); + LOGGER.info("runTestPlanFile() - getting TEST_PLAN_FILE from ENV, rawPath={}", rawPath); + + var path = + rawPath.startsWith("classpath:") + ? TargetsSpec.resourceDir(rawPath.substring("classpath:".length())) + : Path.of(rawPath); + + var testContext = TestPlan.fromFile(path); + + return testContext.testPlan().testNode(); + } + // + // public static void main(String[] args) { + // LauncherDiscoveryRequest request = LauncherDiscoveryRequestBuilder.request() + // .selectors(DiscoverySelectors.selectClass(TestBenchTests.class)) + // .build(); + // + // Launcher launcher = LauncherFactory.create(); + // launcher.execute(request, new DynamicTreeListener()); + // } +} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestPlan.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestPlan.java index 65c0e15659..5557626304 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestPlan.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestPlan.java @@ -1,5 +1,7 @@ package io.stargate.sgv2.jsonapi.api.v1.vectorize; +import static org.junit.jupiter.api.DynamicContainer.dynamicContainer; + import com.fasterxml.jackson.core.JsonFactory; import com.fasterxml.jackson.core.StreamReadFeature; import com.fasterxml.jackson.databind.MapperFeature; @@ -7,13 +9,11 @@ import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; import io.stargate.sgv2.jsonapi.api.v1.vectorize.targets.Target; import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestRunEnv; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.Job; import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestUri; import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.*; -import org.junit.jupiter.api.DynamicContainer; -import org.junit.jupiter.api.DynamicNode; - +import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.Job; import java.io.IOException; +import java.nio.file.Files; import java.nio.file.Path; import java.util.List; import java.util.Map; @@ -21,149 +21,205 @@ import java.util.Set; import java.util.function.Supplier; import java.util.stream.Stream; +import org.apache.commons.text.StringSubstitutor; +import org.junit.jupiter.api.DynamicContainer; +import org.junit.jupiter.api.DynamicNode; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -import static org.junit.jupiter.api.DynamicContainer.dynamicContainer; - -public record TestPlan(Target target, SpecFiles specFiles, Set workflows, boolean ignoreDisabled){ +public record TestPlan( + Target target, SpecFiles specFiles, Set workflows, boolean ignoreDisabled) { + private static final Logger LOGGER = LoggerFactory.getLogger(TestPlan.class); - private static final ObjectMapper JSON_MAPPER = new ObjectMapper(JsonFactory.builder() - .enable(StreamReadFeature.INCLUDE_SOURCE_IN_LOCATION) - .build() - ) - .configure(MapperFeature.ACCEPT_CASE_INSENSITIVE_ENUMS, true); + private static final ObjectMapper JSON_MAPPER = + new ObjectMapper( + JsonFactory.builder().enable(StreamReadFeature.INCLUDE_SOURCE_IN_LOCATION).build()) + .configure(MapperFeature.ACCEPT_CASE_INSENSITIVE_ENUMS, true); - private static final ObjectMapper YAML_MAPPER = new ObjectMapper( - YAMLFactory.builder() - .enable(StreamReadFeature.INCLUDE_SOURCE_IN_LOCATION) - .build() - ) + private static final ObjectMapper YAML_MAPPER = + new ObjectMapper( + YAMLFactory.builder().enable(StreamReadFeature.INCLUDE_SOURCE_IN_LOCATION).build()) .configure(MapperFeature.ACCEPT_CASE_INSENSITIVE_ENUMS, true); - public static TestPlanContext fromFile(String path) { + public static TestPlanContext fromFile(Path path) { + LOGGER.info("fromFile() - Loading test plan file, path={}", path); TestPlanFile planFile; try { - planFile = YAML_MAPPER.readValue(Path.of(path).toFile(), TestPlanFile.class); + var raw = Files.readString(path); + var substituted = new StringSubstitutor(System.getenv()).replace(raw); + planFile = YAML_MAPPER.readValue(substituted, TestPlanFile.class); } catch (IOException e) { throw new RuntimeException(e); } - if (planFile.targetName() != null && planFile.customTarget!= null){ - throw new RuntimeException("Both targetName and customTarget set, use only one. testPlanFile=" + path); + if (planFile.targetName() != null && planFile.customTarget != null) { + throw new RuntimeException( + "Both targetName and customTarget set, use only one. testPlanFile=" + path); } - var testPlan = planFile.customTarget() != null ? - create(planFile.customTarget(), planFile.workflows, planFile.ignoreDisabled) - : - create(planFile.targetName(), planFile.workflows(), planFile.ignoreDisabled()); + var testPlan = + planFile.customTarget() != null + ? create(planFile.customTarget(), planFile.workflows, planFile.ignoreDisabled) + : create(planFile.targetName(), planFile.workflows(), planFile.ignoreDisabled()); return new TestPlanContext(testPlan, planFile); } - public static TestPlan create(String targetName, List workflows){ + + public static TestPlan create(String targetName, List workflows) { return create(targetName, workflows, true); } - public static TestPlan create(String targetName, List workflows, Boolean ignoreDisabled){ + public static TestPlan create(String targetName, List workflows, Boolean ignoreDisabled) { var targetConfigs = TargetsSpec.loadAll("integration-tests/targets/targets.json"); return create(targetConfigs.configuration(targetName), workflows, ignoreDisabled); } - public static TestPlan create(TargetConfiguration targetConfiguration, List workflows, Boolean ignoreDisabled){ + public static TestPlan create( + TargetConfiguration targetConfiguration, List workflows, Boolean ignoreDisabled) { var target = new Target(targetConfiguration); - var specFiles = SpecFiles.loadAll(List.of("integration-tests/vectorize", "integration-tests/assertions/assertion-templates.json")); - - return new TestPlan( - target, - specFiles, - workflows == null ? Set.of() : Set.copyOf(workflows), - ignoreDisabled == null || ignoreDisabled); + var specFiles = + SpecFiles.loadAll( + List.of( + "integration-tests/vectorize", + "integration-tests/assertions/assertion-templates.json")); + + return new TestPlan( + target, + specFiles, + workflows == null ? Set.of() : Set.copyOf(workflows), + ignoreDisabled == null || ignoreDisabled); } - public Stream selectedWorkflows(){ + public Stream selectedWorkflows() { - return specFiles.byKind(TestSpecKind.WORKFLOW) - .filter(specFile -> - workflows.isEmpty() || workflows.contains(specFile.spec().meta().name()) - ) + return specFiles + .byKind(TestSpecKind.WORKFLOW) + .filter( + specFile -> workflows.isEmpty() || workflows.contains(specFile.spec().meta().name())) .map(testSpec -> (WorkflowSpec) testSpec.spec()); } public Stream testNode() { - var desc = "TestPlan: %s on %s workflows %s".formatted( - target.configuration().name(), - target.configuration().backend(), - workflows.isEmpty() ? "" : String.join(", ", workflows) - ); + var desc = + "TestPlan: %s on %s workflows %s" + .formatted( + target.configuration().name(), + target.configuration().backend(), + workflows.isEmpty() ? "" : String.join(", ", workflows)); - var uriBuilder = TestUri.builder(TestUri.Scheme.TESTRUN) - .addSegment(TestUri.Segment.TARGET, target.configuration().name()); + var uriBuilder = + TestUri.builder(TestUri.Scheme.DATAAPI) + .addSegment(TestUri.Segment.TARGET, target.configuration().name()); return Stream.of( dynamicContainer( desc, uriBuilder.build().uri(), selectedWorkflows() - .map(workflow -> workflow.testNode(this, uriBuilder.clone(), ignoreDisabled)) - ) - ); + .map(workflow -> workflow.testNode(this, uriBuilder.clone(), ignoreDisabled)))); } - public void updateJobForTarget(Job job){ + public void updateJobForTarget(Job job) { target.updateJobForTarget(job); } - private static Optional containerIfPresent(TestUri.Builder uriBuilder, String namePrefix, TestSpecMeta meta, Optional dynamicNode){ - return dynamicNode - .map( - node -> dynamicContainer(namePrefix + ": " + meta.name(), uriBuilder.build().uri(), Stream.of(node)) - ); + private static Optional containerIfPresent( + TestUri.Builder uriBuilder, + String namePrefix, + TestSpecMeta meta, + Optional dynamicNode) { + return dynamicNode.map( + node -> + dynamicContainer( + namePrefix + ": " + meta.name(), uriBuilder.build().uri(), Stream.of(node))); } - private static Stream streamIfPresent(Optional container){ + private static Stream streamIfPresent(Optional container) { return container.stream().flatMap(Stream::of); } - private static Stream lifecycleNodes(TestUri.Builder uriBuilder, String namePrefix, TestSpecMeta meta, Supplier> nodeSupplier){ + private static Stream lifecycleNodes( + TestUri.Builder uriBuilder, + String namePrefix, + TestSpecMeta meta, + Supplier> nodeSupplier) { var targetDynamicNode = nodeSupplier.get(); return streamIfPresent(containerIfPresent(uriBuilder, namePrefix, meta, targetDynamicNode)); } - public Stream addLifecycle(TestUri.Builder uriBuilder, WorkflowSpec workflow, Stream dynamicNodes){ + public Stream addLifecycle( + TestUri.Builder uriBuilder, + WorkflowSpec workflow, + Stream dynamicNodes) { - var beforeUriBuilder = uriBuilder.clone().addSegment(TestUri.Segment.LIFECYCLE, "before-workflow"); - var afterUriBuilder = uriBuilder.clone().addSegment(TestUri.Segment.LIFECYCLE, "after-workflow"); + var beforeUriBuilder = + uriBuilder.clone().addSegment(TestUri.Segment.LIFECYCLE, "before-workflow"); + var afterUriBuilder = + uriBuilder.clone().addSegment(TestUri.Segment.LIFECYCLE, "after-workflow"); return Stream.concat( - lifecycleNodes(beforeUriBuilder, "Before Workflow", workflow.meta(), () -> target.beforeWorkflow(this,beforeUriBuilder, workflow)), - Stream.concat(dynamicNodes, - lifecycleNodes(afterUriBuilder, "After Workflow", workflow.meta(), () -> target.afterWorkflow(this, afterUriBuilder, workflow))) - ); + lifecycleNodes( + beforeUriBuilder, + "Before Workflow", + workflow.meta(), + () -> target.beforeWorkflow(this, beforeUriBuilder, workflow)), + Stream.concat( + dynamicNodes, + lifecycleNodes( + afterUriBuilder, + "After Workflow", + workflow.meta(), + () -> target.afterWorkflow(this, afterUriBuilder, workflow)))); } - public Stream addLifecycle(TestUri.Builder uriBuilder, Job job, Stream dynamicNodes){ + public Stream addLifecycle( + TestUri.Builder uriBuilder, Job job, Stream dynamicNodes) { var beforeUriBuilder = uriBuilder.clone().addSegment(TestUri.Segment.LIFECYCLE, "before-job"); - var afterUriBuilder = uriBuilder.clone().addSegment(TestUri.Segment.LIFECYCLE, "after-job"); + var afterUriBuilder = uriBuilder.clone().addSegment(TestUri.Segment.LIFECYCLE, "after-job"); return Stream.concat( - lifecycleNodes(beforeUriBuilder, "Before Job", job.meta(), () -> target.beforeJob(this,beforeUriBuilder, job)), - Stream.concat(dynamicNodes, - lifecycleNodes(afterUriBuilder, "After Job", job.meta(), () -> target.afterJob(this,afterUriBuilder, job))) - ); + lifecycleNodes( + beforeUriBuilder, + "Before Job", + job.meta(), + () -> target.beforeJob(this, beforeUriBuilder, job)), + Stream.concat( + dynamicNodes, + lifecycleNodes( + afterUriBuilder, + "After Job", + job.meta(), + () -> target.afterJob(this, afterUriBuilder, job)))); } - public Stream addLifecycle(TestUri.Builder uriBuilder, TestSuiteSpec testSuite, TestRunEnv environment, Stream dynamicNodes){ + public Stream addLifecycle( + TestUri.Builder uriBuilder, + TestSuiteSpec testSuite, + TestRunEnv environment, + Stream dynamicNodes) { - var beforeUriBuilder = uriBuilder.clone().addSegment(TestUri.Segment.LIFECYCLE, "before-test-suite"); - var afterUriBuilder = uriBuilder.clone().addSegment(TestUri.Segment.LIFECYCLE, "after-test-suite"); + var beforeUriBuilder = + uriBuilder.clone().addSegment(TestUri.Segment.LIFECYCLE, "before-test-suite"); + var afterUriBuilder = + uriBuilder.clone().addSegment(TestUri.Segment.LIFECYCLE, "after-test-suite"); return Stream.concat( - lifecycleNodes(beforeUriBuilder, "Before TestSuite", testSuite.meta(), () -> target.beforeTestSuite(this, beforeUriBuilder,testSuite, environment)), - Stream.concat(dynamicNodes, - lifecycleNodes( afterUriBuilder, "After TestSuite", testSuite.meta(), () -> target.afterTestSuite(this, afterUriBuilder,testSuite, environment))) - ); + lifecycleNodes( + beforeUriBuilder, + "Before TestSuite", + testSuite.meta(), + () -> target.beforeTestSuite(this, beforeUriBuilder, testSuite, environment)), + Stream.concat( + dynamicNodes, + lifecycleNodes( + afterUriBuilder, + "After TestSuite", + testSuite.meta(), + () -> target.afterTestSuite(this, afterUriBuilder, testSuite, environment)))); } public record TestPlanFile( @@ -172,16 +228,15 @@ public record TestPlanFile( TargetConfiguration customTarget, List workflows, Boolean ignoreDisabled, - Map envVars - ) { } + Map envVars) {} - public record TestPlanContext( - TestPlan testPlan, - TestPlanFile testPlanFile - ) implements AutoCloseable { + public record TestPlanContext(TestPlan testPlan, TestPlanFile testPlanFile) + implements AutoCloseable { - public TestPlanContext{ - testPlanFile.envVars.forEach(System::setProperty); + public TestPlanContext { + if (testPlanFile.envVars != null) { + testPlanFile.envVars.forEach(System::setProperty); + } } @Override diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/VectorizeAstra.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/VectorizeAstra.java index a2a2c61cc4..94784d0c02 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/VectorizeAstra.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/VectorizeAstra.java @@ -1,17 +1,15 @@ package io.stargate.sgv2.jsonapi.api.v1.vectorize; +import java.util.List; +import java.util.stream.Stream; import org.junit.jupiter.api.DynamicContainer; import org.junit.jupiter.api.TestFactory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.List; -import java.util.stream.Stream; - public class VectorizeAstra { private static final Logger LOGGER = LoggerFactory.getLogger(VectorizeAstra.class); - private static final int TEST_PARALLELISM = 8; // ← set this @TestFactory @@ -21,27 +19,27 @@ Stream jobs() { return testPlan.testNode(); - // Parallel -// int maxConcurrentJobs = 2; -// integrationTarget.workflowStarting(workflow); -// try { -// var executor = java.util.concurrent.Executors.newFixedThreadPool(maxConcurrentJobs); -// try { -// var futures = jobs.stream() -// .map(job -> java.util.concurrent.CompletableFuture.runAsync(() -> { -// LOGGER.info("Starting job {}", job.meta()); -// new IntegrationJobRunner(integrationTarget, itCollection, job).run(); -// }, executor)) -// .toList(); -// -// // wait for all, fail if any failed -// futures.forEach(java.util.concurrent.CompletableFuture::join); -// } finally { -// executor.shutdown(); -// } -// } finally { -// integrationTarget.workflowFinished(workflow); -// } + // Parallel + // int maxConcurrentJobs = 2; + // integrationTarget.workflowStarting(workflow); + // try { + // var executor = java.util.concurrent.Executors.newFixedThreadPool(maxConcurrentJobs); + // try { + // var futures = jobs.stream() + // .map(job -> java.util.concurrent.CompletableFuture.runAsync(() -> { + // LOGGER.info("Starting job {}", job.meta()); + // new IntegrationJobRunner(integrationTarget, itCollection, job).run(); + // }, executor)) + // .toList(); + // + // // wait for all, fail if any failed + // futures.forEach(java.util.concurrent.CompletableFuture::join); + // } finally { + // executor.shutdown(); + // } + // } finally { + // integrationTarget.workflowFinished(workflow); + // } } } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/VectorizeIT.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/VectorizeIT.java index 5ae06c4758..1721e5787c 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/VectorizeIT.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/VectorizeIT.java @@ -4,12 +4,10 @@ import io.quarkus.test.junit.QuarkusIntegrationTest; import io.stargate.sgv2.jsonapi.api.v1.AbstractCollectionIntegrationTestBase; import io.stargate.sgv2.jsonapi.testresource.DseTestResource; -import org.junit.jupiter.api.DynamicContainer; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.TestFactory; - import java.util.List; import java.util.stream.Stream; +import org.junit.jupiter.api.DynamicContainer; +import org.junit.jupiter.api.TestFactory; @QuarkusIntegrationTest @WithTestResource(value = DseTestResource.class) diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/VectorizeTestFactory.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/VectorizeTestFactory.java deleted file mode 100644 index 015781b86e..0000000000 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/VectorizeTestFactory.java +++ /dev/null @@ -1,26 +0,0 @@ -package io.stargate.sgv2.jsonapi.api.v1.vectorize; - -import org.junit.jupiter.api.DynamicContainer; -import org.junit.jupiter.api.TestFactory; -import org.junit.jupiter.api.parallel.Execution; -import org.junit.jupiter.api.parallel.ExecutionMode; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.Collection; -import java.util.List; -import java.util.stream.Stream; - -public class VectorizeTestFactory { - - private static final Logger LOGGER = LoggerFactory.getLogger(VectorizeTestFactory.class); - - @TestFactory - Stream jobs() { - -// var testPlan = TestPlan.create("local", List.of("all-vectorize-workflow")); - var testContext = TestPlan.fromFile("/Users/amorton/code/stargate/jsonapi/.env/test-plans/test-plan-dev-smoketest-aws-us-east-1.yaml"); - - return testContext.testPlan().testNode(); - } -} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/AssertionFactory.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/AssertionFactory.java index 5bff302b56..f4ae01255c 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/AssertionFactory.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/AssertionFactory.java @@ -1,18 +1,14 @@ package io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions; import com.fasterxml.jackson.databind.JsonNode; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.TestCommand; import io.stargate.sgv2.jsonapi.api.v1.vectorize.TestPlan; - +import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.TestCommand; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.lang.reflect.Modifier; import java.util.List; -/** - - */ - +/** */ public sealed interface AssertionFactory { public static final AssertionFactoryRegistry REGISTRY = new AssertionFactoryRegistry(); @@ -24,7 +20,8 @@ non-sealed interface AssertionMatcherFactory extends AssertionFactory { @FunctionalInterface non-sealed interface TemplatedAssertionFactory extends AssertionFactory { - List create(TestPlan testPlan, JsonNode template, TestCommand testCommand, JsonNode args); + List create( + TestPlan testPlan, JsonNode template, TestCommand testCommand, JsonNode args); } static boolean isValidFactoryMethod(Method method) { @@ -45,9 +42,7 @@ static boolean isValidFactoryMethod(Method method) { if (method.getReturnType() == AssertionMatcher.class) { var p = method.getParameterTypes(); - return p.length == 2 - && p[0] == TestCommand.class - && p[1] == JsonNode.class; + return p.length == 2 && p[0] == TestCommand.class && p[1] == JsonNode.class; } return false; } @@ -66,10 +61,9 @@ protected WrappedMethod(Class clazz, Method method) { } static WrappedMethod of(Class clazz, Method method) { - return (method.getReturnType() == AssertionMatcher.class) ? - new WrappedAssertionMatcherFactory(clazz, method) - : - new WrappedTemplatedAssertionFactory(clazz, method); + return (method.getReturnType() == AssertionMatcher.class) + ? new WrappedAssertionMatcherFactory(clazz, method) + : new WrappedTemplatedAssertionFactory(clazz, method); } public Class clazz() { @@ -119,7 +113,8 @@ final class WrappedTemplatedAssertionFactory extends WrappedMethod } @Override - public List create(TestPlan testPlan, JsonNode template, TestCommand testCommand, JsonNode args) { + public List create( + TestPlan testPlan, JsonNode template, TestCommand testCommand, JsonNode args) { return invoke(testPlan, template, testCommand, args); } } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/AssertionFactoryRegistry.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/AssertionFactoryRegistry.java index cc86703088..e369202d35 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/AssertionFactoryRegistry.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/AssertionFactoryRegistry.java @@ -5,12 +5,12 @@ public class AssertionFactoryRegistry { - private final Map factoryMethods = new ConcurrentHashMap<>(); - + private final Map factoryMethods = + new ConcurrentHashMap<>(); public void register(Class cls) { for (var method : cls.getMethods()) { - if ( AssertionFactory.isValidFactoryMethod(method)) { + if (AssertionFactory.isValidFactoryMethod(method)) { var wrapped = AssertionFactory.WrappedMethod.of(cls, method); factoryMethods.put(wrapped.assertionName(), wrapped); } @@ -27,7 +27,9 @@ public AssertionFactory.WrappedMethod getWrapped(String fullKey) { factoryMethod = factoryMethods.get(normalisedName); if (factoryMethod == null) { - throw new IllegalArgumentException("Unknown assertion factory. (normalised)name: %s known=%s".formatted(normalisedName, factoryMethods.keySet())); + throw new IllegalArgumentException( + "Unknown assertion factory. (normalised)name: %s known=%s" + .formatted(normalisedName, factoryMethods.keySet())); } return factoryMethod; } @@ -38,8 +40,10 @@ private void loadClassFor(AssertionName normalisedName) { Class.forName(normalisedName.properClassName()); // class static initializer should call register() } catch (ClassNotFoundException e) { - throw new IllegalArgumentException("Unknown assertion factory. normalisedName=%s, properClassName()=%s".formatted(normalisedName, normalisedName.properClassName()), e); + throw new IllegalArgumentException( + "Unknown assertion factory. normalisedName=%s, properClassName()=%s" + .formatted(normalisedName, normalisedName.properClassName()), + e); } } - } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/AssertionMatcher.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/AssertionMatcher.java index 60c911e337..a00898cf93 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/AssertionMatcher.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/AssertionMatcher.java @@ -2,9 +2,7 @@ import io.stargate.sgv2.jsonapi.api.v1.vectorize.messaging.APIResponse; -/** - * Contract for running an assertion on the response from the API. - */ +/** Contract for running an assertion on the response from the API. */ @FunctionalInterface public interface AssertionMatcher { @@ -15,5 +13,4 @@ public interface AssertionMatcher { * @throws AssertionError if the match fails */ void match(APIResponse apiResponse); - } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/AssertionName.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/AssertionName.java index 96ec5b54b7..bf5ae37e13 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/AssertionName.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/AssertionName.java @@ -4,10 +4,9 @@ public record AssertionName(String typeName, String funcName) { - private static final String PACKAGE = - "io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions"; + private static final String PACKAGE = "io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions"; - public AssertionName{ + public AssertionName { typeName = typeName.toLowerCase(); funcName = funcName.toLowerCase(); } @@ -23,22 +22,19 @@ public static AssertionName from(String fullKey) { return new AssertionName(type, func); } - public static String properName(Class clazz, Method method ) { + public static String properName(Class clazz, Method method) { return clazz.getSimpleName() + '.' + method.getName(); } - public static String properName(Method method ) { + public static String properName(Method method) { return method.getName(); } - public String properClassName() { - return PACKAGE + "." - + Character.toUpperCase(typeName.charAt(0)) - + typeName.substring(1); + return PACKAGE + "." + Character.toUpperCase(typeName.charAt(0)) + typeName.substring(1); } public String normalisedKey() { - return typeName + "." + funcName; + return typeName + "." + funcName; } } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/BodyAssertion.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/BodyAssertion.java index 5393ea3df9..be069a1a42 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/BodyAssertion.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/BodyAssertion.java @@ -7,17 +7,14 @@ /** * Assertions that check the body of the response using a {@link Matcher} form hamcrest. * - *

- * Example: + *

Example: + * * 

  *   return new BodyAssertion("data.documents", hasSize(expectedCount));
  *
- *

*/ -public record BodyAssertion( - String bodyPath, - Matcher matcher -) implements Describable, AssertionMatcher { +public record BodyAssertion(String bodyPath, Matcher matcher) + implements Describable, AssertionMatcher { @Override public void match(APIResponse apiResponse) { @@ -32,4 +29,4 @@ public String describe() { // called should truncate if it wants to limit it return "body('%s') - %s".formatted(bodyPath(), describable.toString()); } -} \ No newline at end of file +} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/DescribableAssertionMatcher.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/DescribableAssertionMatcher.java index 2b3e3942a1..370569abc7 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/DescribableAssertionMatcher.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/DescribableAssertionMatcher.java @@ -3,13 +3,11 @@ import io.stargate.sgv2.jsonapi.api.v1.vectorize.messaging.APIResponse; import org.jspecify.annotations.NonNull; -public record DescribableAssertionMatcher( - String description, - AssertionMatcher matcher -) implements Describable, AssertionMatcher { +public record DescribableAssertionMatcher(String description, AssertionMatcher matcher) + implements Describable, AssertionMatcher { - - public static DescribableAssertionMatcher described(String description, AssertionMatcher matcher) { + public static DescribableAssertionMatcher described( + String description, AssertionMatcher matcher) { return new DescribableAssertionMatcher(description, matcher); } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Documents.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Documents.java index fc5bdf5dbd..7dd711bbd6 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Documents.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Documents.java @@ -1,17 +1,15 @@ package io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions; -import com.fasterxml.jackson.databind.JsonNode; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.TestCommand; - import static net.javacrumbs.jsonunit.JsonMatchers.jsonEquals; import static org.hamcrest.Matchers.hasSize; +import com.fasterxml.jackson.databind.JsonNode; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.TestCommand; + /** - * Assertions that check the `document` or `documents` in the `data` field - * of the API result. - *

- * See {@link TestAssertion} - *

+ * Assertions that check the `document` or `documents` in the `data` field of the API result. + * + *

See {@link TestAssertion} */ public class Documents { diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Http.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Http.java index 5958d9b4ee..c2efa27e09 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Http.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Http.java @@ -1,18 +1,20 @@ package io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions; +import static io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions.DescribableAssertionMatcher.described; + import com.fasterxml.jackson.databind.JsonNode; import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.TestCommand; import org.apache.http.HttpStatus; -import static io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions.DescribableAssertionMatcher.described; - public class Http { static { AssertionFactory.REGISTRY.register(Http.class); } - public static AssertionMatcher success(TestCommand testCommand, JsonNode args){ - return described("http status is groovy", apiResponse -> apiResponse.validatable().statusCode(HttpStatus.SC_OK)); + public static AssertionMatcher success(TestCommand testCommand, JsonNode args) { + return described( + "http status is groovy", + apiResponse -> apiResponse.validatable().statusCode(HttpStatus.SC_OK)); } } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Response.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Response.java index 65d0283b73..48e462124e 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Response.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Response.java @@ -1,17 +1,14 @@ package io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions; -import com.fasterxml.jackson.databind.JsonNode; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.TestCommand; - import static io.stargate.sgv2.jsonapi.api.v1.ResponseAssertions.*; -import static org.hamcrest.Matchers.hasSize; +import com.fasterxml.jackson.databind.JsonNode; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.TestCommand; /** * Assertions that check the structure of the API response, e.g. should it have a `data` field - *

- * See {@link TestAssertion} - *

+ * + *

See {@link TestAssertion} */ public class Response { @@ -19,22 +16,24 @@ public class Response { AssertionFactory.REGISTRY.register(Response.class); } - /** - * Checks the hTTP status AND the shape of the response doc - */ + /** Checks the hTTP status AND the shape of the response doc */ public static AssertionMatcher isSuccess(TestCommand testCommand, JsonNode args) { var commandName = testCommand.commandName(); - return switch (commandName.getCommandType()){ + return switch (commandName.getCommandType()) { case DDL -> isDDLSuccess(testCommand, args); - case DML -> - switch (commandName) { - case FIND_ONE, FIND -> Response.isFindSuccess(testCommand, args); - case FIND_ONE_AND_DELETE, FIND_ONE_AND_REPLACE, FIND_ONE_AND_UPDATE -> Response.isFindAndSuccess(testCommand, args); - case INSERT_ONE, INSERT_MANY -> Response.isWriteSuccess(testCommand, args); - default -> throw new IllegalStateException("No isSuccess mapping for command name: " + commandName); - }; - case ADMIN -> throw new IllegalStateException("No isSuccess mapping for command name: " + commandName); + case DML -> + switch (commandName) { + case FIND_ONE, FIND -> Response.isFindSuccess(testCommand, args); + case FIND_ONE_AND_DELETE, FIND_ONE_AND_REPLACE, FIND_ONE_AND_UPDATE -> + Response.isFindAndSuccess(testCommand, args); + case INSERT_ONE, INSERT_MANY -> Response.isWriteSuccess(testCommand, args); + default -> + throw new IllegalStateException( + "No isSuccess mapping for command name: " + commandName); + }; + case ADMIN -> + throw new IllegalStateException("No isSuccess mapping for command name: " + commandName); }; } @@ -46,7 +45,6 @@ public static AssertionMatcher isFindAndSuccess(TestCommand testCommand, JsonNod return new BodyAssertion("$", responseIsFindAndSuccess()); } - public static AssertionMatcher isWriteSuccess(TestCommand testCommand, JsonNode args) { return new BodyAssertion("$", responseIsWriteSuccess()); } @@ -54,5 +52,4 @@ public static AssertionMatcher isWriteSuccess(TestCommand testCommand, JsonNode public static AssertionMatcher isDDLSuccess(TestCommand testCommand, JsonNode args) { return new BodyAssertion("$", responseIsDDLSuccess()); } - } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/SingleTestAssertion.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/SingleTestAssertion.java index 88773fdd81..1704100cfb 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/SingleTestAssertion.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/SingleTestAssertion.java @@ -1,21 +1,14 @@ package io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions; import com.fasterxml.jackson.databind.JsonNode; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestRunResponse; import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.DynamicTestExecutable; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestRunResponse; import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestUri; -import org.junit.jupiter.api.DynamicNode; - import java.util.concurrent.atomic.AtomicReference; +import org.junit.jupiter.api.DynamicNode; -import static org.junit.jupiter.api.DynamicTest.dynamicTest; -import static org.junit.jupiter.api.DynamicTest.stream; - -public record SingleTestAssertion( - String name, - JsonNode args, - AssertionMatcher matcher -) implements TestAssertion { +public record SingleTestAssertion(String name, JsonNode args, AssertionMatcher matcher) + implements TestAssertion { public void run(TestRunResponse testResponse) { @@ -31,24 +24,22 @@ public void run(TestRunResponse testResponse) { } @Override - public DynamicNode testNodes(TestUri.Builder uriBuilder, AtomicReference testResponse) { - - var matcherDesc = (matcher instanceof Describable d) ? - d.describe() - : - ""; - - var executable = new DynamicTestExecutable( - "%s [%s]".formatted(name(), matcherDesc), - uriBuilder.addSegment(TestUri.Segment.ASSERTION, name()), - () -> { - var resp = testResponse.get(); - if (resp == null) { - throw new IllegalStateException("Response is null"); - } - run(resp); - } - ); + public DynamicNode testNodes( + TestUri.Builder uriBuilder, AtomicReference testResponse) { + + var matcherDesc = (matcher instanceof Describable d) ? d.describe() : ""; + + var executable = + new DynamicTestExecutable( + "%s [%s]".formatted(name(), matcherDesc), + uriBuilder.addSegment(TestUri.Segment.ASSERTION, name()), + () -> { + var resp = testResponse.get(); + if (resp == null) { + throw new IllegalStateException("Response is null"); + } + run(resp); + }); return executable.testNode(); } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Status.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Status.java index 722607068a..4797035d77 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Status.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Status.java @@ -1,17 +1,16 @@ package io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions; +import static net.javacrumbs.jsonunit.JsonMatchers.jsonEquals; + import com.fasterxml.jackson.databind.JsonNode; import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.TestCommand; -import static net.javacrumbs.jsonunit.JsonMatchers.jsonEquals; - public class Status { static { AssertionFactory.REGISTRY.register(Status.class); } - public static AssertionMatcher isExactly(TestCommand testCommand, JsonNode args) { return new BodyAssertion("status", jsonEquals(args)); } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Templated.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Templated.java index cc86644ab7..45da77c935 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Templated.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Templated.java @@ -2,9 +2,8 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.TestCommand; import io.stargate.sgv2.jsonapi.api.v1.vectorize.TestPlan; - +import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.TestCommand; import java.util.List; public class Templated { @@ -13,7 +12,8 @@ public class Templated { AssertionFactory.REGISTRY.register(Templated.class); } - public static List isSuccess(TestPlan testPlan, JsonNode template, TestCommand testCommand, JsonNode args){ + public static List isSuccess( + TestPlan testPlan, JsonNode template, TestCommand testCommand, JsonNode args) { var commandTemplate = template.get(testCommand.commandName().getApiName()); if (commandTemplate == null) { throw new IllegalArgumentException( @@ -23,12 +23,11 @@ public static List isSuccess(TestPlan testPlan, JsonNode template return runTemplate(testPlan, (ObjectNode) commandTemplate, testCommand, args); } - - private static List runTemplate(TestPlan testPlan, ObjectNode template, TestCommand testCommand, JsonNode args) { + private static List runTemplate( + TestPlan testPlan, ObjectNode template, TestCommand testCommand, JsonNode args) { return template.properties().stream() - .map(entry -> new TestAssertion.AssertionDefinition(entry.getKey(), entry.getValue())) - .map(def -> TestAssertion.buildAssertion(testPlan, testCommand, def)) - .toList(); + .map(entry -> new TestAssertion.AssertionDefinition(entry.getKey(), entry.getValue())) + .map(def -> TestAssertion.buildAssertion(testPlan, testCommand, def)) + .toList(); } - } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/TestAssertion.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/TestAssertion.java index d212c54cf9..e7a43ccf92 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/TestAssertion.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/TestAssertion.java @@ -1,18 +1,17 @@ package io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions; import com.fasterxml.jackson.databind.JsonNode; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.TestCase; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.TestCommand; import io.stargate.sgv2.jsonapi.api.v1.vectorize.TestPlan; import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestRunResponse; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.AssertionTemplateSpec; import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestUri; -import org.junit.jupiter.api.DynamicNode; - +import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.AssertionTemplateSpec; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.TestCase; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.TestCommand; import java.util.List; import java.util.Map; import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Stream; +import org.junit.jupiter.api.DynamicNode; public interface TestAssertion { @@ -24,36 +23,33 @@ public interface TestAssertion { DynamicNode testNodes(TestUri.Builder uriBuilder, AtomicReference testResponse); - static List forSuccess(TestPlan testPlan, TestCommand testCommand) { - var builder = Stream.builder() - .add(new AssertionDefinition("Templated.isSuccess", null)); + var builder = + Stream.builder() + .add(new AssertionDefinition("Templated.isSuccess", null)); return buildAssertions(testPlan, testCommand, builder.build()); } static List buildAssertions(TestPlan testPlan, TestCase testCase) { - var defs = testCase.asserts().properties().stream() - .map(AssertionDefinition::create); + var defs = testCase.asserts().properties().stream().map(AssertionDefinition::create); return buildAssertions(testPlan, testCase.command(), defs); } - static List buildAssertions(TestPlan testPlan, TestCommand testCommand, Stream defs) { + static List buildAssertions( + TestPlan testPlan, TestCommand testCommand, Stream defs) { - return defs.map( - def -> buildAssertion(testPlan, testCommand, def) - ).toList(); + return defs.map(def -> buildAssertion(testPlan, testCommand, def)).toList(); } - static TestAssertion buildAssertion(TestPlan testPlan, TestCommand testCommand, AssertionDefinition def) { + static TestAssertion buildAssertion( + TestPlan testPlan, TestCommand testCommand, AssertionDefinition def) { return def.addFactory(AssertionFactory.REGISTRY).build(testPlan, testCommand); } - /** - *

- */ + /** */ record AssertionDefinition(String name, JsonNode args) { static AssertionDefinition create(Map.Entry def) { @@ -64,34 +60,39 @@ AssertionDefWithFactory addFactory(AssertionFactoryRegistry registry) { var factory = registry.getWrapped(name()); if (factory == null) { - throw new IllegalStateException("Unknown Assertion Factory name=" + name()); + throw new IllegalStateException("Unknown Assertion Factory name=" + name()); } return new AssertionDefWithFactory(factory, args); } } - /** - *

- */ - record AssertionDefWithFactory(AssertionFactory.WrappedMethod method, JsonNode args){ + /** */ + record AssertionDefWithFactory(AssertionFactory.WrappedMethod method, JsonNode args) { TestAssertion build(TestPlan testPlan, TestCommand testCommand) { return switch (method) { case AssertionFactory.WrappedAssertionMatcherFactory factory -> - new SingleTestAssertion(method.properName(), args(), factory.create(testCommand, args())); - - case AssertionFactory.TemplatedAssertionFactory factory ->{ - - var template = testPlan.specFiles().byType(AssertionTemplateSpec.class) - .flatMap(assertTemplate -> assertTemplate.templateFor(method.properName()).stream()) - .findFirst() - .orElseThrow(() -> new IllegalStateException("Unknown Assertion Template name=" + method.properName())); - - yield new TestAssertionContainer(method.properName(), args(), factory.create(testPlan, template, testCommand, args())); + new SingleTestAssertion( + method.properName(), args(), factory.create(testCommand, args())); + + case AssertionFactory.TemplatedAssertionFactory factory -> { + var template = + testPlan + .specFiles() + .byType(AssertionTemplateSpec.class) + .flatMap( + assertTemplate -> assertTemplate.templateFor(method.properName()).stream()) + .findFirst() + .orElseThrow( + () -> + new IllegalStateException( + "Unknown Assertion Template name=" + method.properName())); + + yield new TestAssertionContainer( + method.properName(), args(), factory.create(testPlan, template, testCommand, args())); } }; } - } } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/TestAssertionContainer.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/TestAssertionContainer.java index 81bf82d9c9..6510245817 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/TestAssertionContainer.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/TestAssertionContainer.java @@ -1,48 +1,45 @@ package io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions; +import static org.junit.jupiter.api.DynamicContainer.dynamicContainer; + import com.fasterxml.jackson.databind.JsonNode; import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestRunResponse; import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestUri; -import org.junit.jupiter.api.DynamicNode; - import java.util.List; import java.util.concurrent.atomic.AtomicReference; +import org.junit.jupiter.api.DynamicNode; -import static org.junit.jupiter.api.DynamicContainer.dynamicContainer; -import static org.junit.jupiter.api.DynamicTest.dynamicTest; - -public record TestAssertionContainer( - String name, - JsonNode args, - List assertions -) implements TestAssertion { +public record TestAssertionContainer(String name, JsonNode args, List assertions) + implements TestAssertion { @Override public void run(TestRunResponse testResponse) { - for (TestAssertion assertion : assertions) { - try{ + for (TestAssertion assertion : assertions) { + try { assertion.run(testResponse); - } - catch (AssertionError e) { - System.out.printf("Failed Assertion Container: name=%s, args=%s\n\t Caused By: name=%s, args=%s", name, args, - assertion.name(), assertion.args()); + } catch (AssertionError e) { + System.out.printf( + "Failed Assertion Container: name=%s, args=%s\n\t Caused By: name=%s, args=%s", + name, args, assertion.name(), assertion.args()); throw e; - } - catch (Exception e) { - System.out.printf("Error In Assertion Container: name=%s, args=%s\n\t Caused By: name=%s, args=%s", name, args, - assertion.name(), assertion.args()); + } catch (Exception e) { + System.out.printf( + "Error In Assertion Container: name=%s, args=%s\n\t Caused By: name=%s, args=%s", + name, args, assertion.name(), assertion.args()); throw e; } } } @Override - public DynamicNode testNodes(TestUri.Builder uriBuilder, AtomicReference testResponse) { + public DynamicNode testNodes( + TestUri.Builder uriBuilder, AtomicReference testResponse) { uriBuilder.addSegment(TestUri.Segment.ASSERTION, name()); - var childs = assertions.stream() - .map(assertion -> assertion.testNodes(uriBuilder.clone(), testResponse)) - .toList(); + var childs = + assertions.stream() + .map(assertion -> assertion.testNodes(uriBuilder.clone(), testResponse)) + .toList(); return dynamicContainer(name, childs); } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/lifecycle/TestPlanLifecycle.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/lifecycle/TestPlanLifecycle.java index 90420cf48d..d79ba00192 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/lifecycle/TestPlanLifecycle.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/lifecycle/TestPlanLifecycle.java @@ -1,36 +1,41 @@ package io.stargate.sgv2.jsonapi.api.v1.vectorize.lifecycle; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.Job; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestRunEnv; import io.stargate.sgv2.jsonapi.api.v1.vectorize.TestPlan; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.TestSuiteSpec; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestRunEnv; import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestUri; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.Job; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.TestSuiteSpec; import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.WorkflowSpec; -import org.junit.jupiter.api.DynamicNode; - import java.util.Optional; +import org.junit.jupiter.api.DynamicNode; public interface TestPlanLifecycle { - default Optional beforeWorkflow(TestPlan testPlan, TestUri.Builder uriBuilder, WorkflowSpec workflow){ + default Optional beforeWorkflow( + TestPlan testPlan, TestUri.Builder uriBuilder, WorkflowSpec workflow) { return Optional.empty(); } - default Optional afterWorkflow(TestPlan testPlan,TestUri.Builder uriBuilder, WorkflowSpec workflow){ + + default Optional afterWorkflow( + TestPlan testPlan, TestUri.Builder uriBuilder, WorkflowSpec workflow) { return Optional.empty(); } - default Optional beforeJob(TestPlan testPlan,TestUri.Builder uriBuilder, Job job){ + default Optional beforeJob(TestPlan testPlan, TestUri.Builder uriBuilder, Job job) { return Optional.empty(); } - default Optional afterJob(TestPlan testPlan, TestUri.Builder uriBuilder, Job job){ + + default Optional afterJob(TestPlan testPlan, TestUri.Builder uriBuilder, Job job) { return Optional.empty(); } - default Optional beforeTestSuite(TestPlan testPlan, TestUri.Builder uriBuilder, TestSuiteSpec test, TestRunEnv env){ + default Optional beforeTestSuite( + TestPlan testPlan, TestUri.Builder uriBuilder, TestSuiteSpec test, TestRunEnv env) { return Optional.empty(); } - default Optional afterTestSuite(TestPlan testPlan, TestUri.Builder uriBuilder, TestSuiteSpec test, TestRunEnv env){ + + default Optional afterTestSuite( + TestPlan testPlan, TestUri.Builder uriBuilder, TestSuiteSpec test, TestRunEnv env) { return Optional.empty(); } - } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/messaging/APIRequest.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/messaging/APIRequest.java index 00db656092..400ed16dd2 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/messaging/APIRequest.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/messaging/APIRequest.java @@ -1,5 +1,7 @@ package io.stargate.sgv2.jsonapi.api.v1.vectorize.messaging; +import static io.restassured.RestAssured.given; + import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; @@ -8,42 +10,37 @@ import io.restassured.response.ValidatableResponse; import io.restassured.specification.RequestSpecification; import io.stargate.sgv2.jsonapi.api.model.command.CommandTarget; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.TestCommand; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestRunEnv; import io.stargate.sgv2.jsonapi.api.v1.vectorize.targets.Connection; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestRunEnv; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.TestCommand; import io.stargate.sgv2.jsonapi.config.constants.HttpConstants; - import java.util.Map; -import static io.restassured.RestAssured.given; - - public class APIRequest { public static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); - private static String COLLECTION_PATH = "/{keyspace}/{collection}"; - private static String KEYSPACE_PATH = "/{keyspace}"; - private static String DB_PATH = "/"; + private static String COLLECTION_PATH = "/{keyspace}/{collection}"; + private static String KEYSPACE_PATH = "/{keyspace}"; + private static String DB_PATH = "/"; private final Connection connection; private final TestRunEnv integrationEnv; private final ObjectNode request; - public APIRequest(Connection connection, TestRunEnv integrationEnv, ObjectNode request ) { + public APIRequest(Connection connection, TestRunEnv integrationEnv, ObjectNode request) { this.connection = connection; this.integrationEnv = integrationEnv; this.request = request; } - public APIResponse execute(){ + public APIResponse execute() { var requestSpec = requestSpec(); return new APIResponse(this, executeRequest(requestSpec)); } - private RequestSpecification requestSpec() { String requestString; @@ -53,30 +50,29 @@ private RequestSpecification requestSpec() { throw new RuntimeException(e); } - return jsonRequest() - .body(requestString).when(); + return jsonRequest().body(requestString).when(); } private ValidatableResponse executeRequest(RequestSpecification requestSpec) { var commandName = TestCommand.commandName(request); Response response; - if (commandName.getTargets().contains(CommandTarget.COLLECTION) || commandName.getTargets().contains(CommandTarget.TABLE)){ - response = requestSpec - .post(COLLECTION_PATH, integrationEnv.requiredValue("KEYSPACE_NAME"), integrationEnv.requiredValue("COLLECTION_NAME")); - } - else if (commandName.getTargets().contains(CommandTarget.KEYSPACE) ){ - response = requestSpec.post(KEYSPACE_PATH, integrationEnv.requiredValue("KEYSPACE_NAME")); - } - else if(commandName.getTargets().contains(CommandTarget.DATABASE)){ - response = requestSpec.post(DB_PATH); - } - else { + if (commandName.getTargets().contains(CommandTarget.COLLECTION) + || commandName.getTargets().contains(CommandTarget.TABLE)) { + response = + requestSpec.post( + COLLECTION_PATH, + integrationEnv.requiredValue("KEYSPACE_NAME"), + integrationEnv.requiredValue("COLLECTION_NAME")); + } else if (commandName.getTargets().contains(CommandTarget.KEYSPACE)) { + response = requestSpec.post(KEYSPACE_PATH, integrationEnv.requiredValue("KEYSPACE_NAME")); + } else if (commandName.getTargets().contains(CommandTarget.DATABASE)) { + response = requestSpec.post(DB_PATH); + } else { throw new IllegalArgumentException("Do not know how to execute command: " + commandName); } - return response - .then().log().all(); + return response.then().log().all(); } protected Map getHeaders() { @@ -88,16 +84,17 @@ protected Map getHeaders() { integrationEnv.requiredValue("x-embedding-api-key")); } - public RequestSpecification jsonRequest(){ + public RequestSpecification jsonRequest() { + // .log().uri() + // .log().body() return given() - .log().all() + .log() + .all() .baseUri(connection.domain()) .port(connection.port()) .basePath(connection.basePath()) .headers(getHeaders()) .contentType(ContentType.JSON); - } - } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/messaging/APIResponse.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/messaging/APIResponse.java index 0e407326fa..c05feaa974 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/messaging/APIResponse.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/messaging/APIResponse.java @@ -2,6 +2,4 @@ import io.restassured.response.ValidatableResponse; -public record APIResponse (APIRequest apiRequest, - ValidatableResponse validatable) { -} +public record APIResponse(APIRequest apiRequest, ValidatableResponse validatable) {} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/targets/Backend.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/targets/Backend.java index 027bf7bcaa..a2d40bb290 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/targets/Backend.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/targets/Backend.java @@ -1,12 +1,9 @@ package io.stargate.sgv2.jsonapi.api.v1.vectorize.targets; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.Job; import io.stargate.sgv2.jsonapi.api.v1.vectorize.lifecycle.TestPlanLifecycle; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.Job; public abstract class Backend implements TestPlanLifecycle { - public void updateJobForTarget(Job job){ - } - + public void updateJobForTarget(Job job) {} } - diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/targets/CassandraBackend.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/targets/CassandraBackend.java index 1f7767e3ae..eb80347f16 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/targets/CassandraBackend.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/targets/CassandraBackend.java @@ -1,22 +1,19 @@ package io.stargate.sgv2.jsonapi.api.v1.vectorize.targets; +import static io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestRunEnv.toSafeSchemaIdentifier; + import com.fasterxml.jackson.databind.ObjectMapper; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.Job; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.TestCommand; import io.stargate.sgv2.jsonapi.api.v1.vectorize.TestPlan; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestRunRequest; import io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions.TestAssertion; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestRunRequest; import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestUri; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.Job; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.TestCommand; +import java.util.Optional; import org.apache.commons.lang3.RandomStringUtils; import org.junit.jupiter.api.DynamicNode; -import java.util.Optional; - -import static io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestRunEnv.toSafeSchemaIdentifier; - -/** - * Cassandra:Y2Fzc2FuZHJh:Y2Fzc2FuZHJh - */ +/** Cassandra:Y2Fzc2FuZHJh:Y2Fzc2FuZHJh */ public class CassandraBackend extends Backend { private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); @@ -24,16 +21,21 @@ public class CassandraBackend extends Backend { @Override public void updateJobForTarget(Job job) { // max length for keyspace is 48 chars - var keyspaceName = toSafeSchemaIdentifier( - "job_" + job.meta().name().substring(0, Math.min(job.meta().name().length(), 27)) + "_" + RandomStringUtils.insecure().nextAlphanumeric(16)); + var keyspaceName = + toSafeSchemaIdentifier( + "job_" + + job.meta().name().substring(0, Math.min(job.meta().name().length(), 27)) + + "_" + + RandomStringUtils.insecure().nextAlphanumeric(16)); job.variables().put("KEYSPACE_NAME", keyspaceName); } @Override public Optional beforeJob(TestPlan testPlan, TestUri.Builder uriBuilder, Job job) { - var command = TestCommand.fromJson( - """ + var command = + TestCommand.fromJson( + """ { "createKeyspace": { "name": "${KEYSPACE_NAME}" @@ -42,17 +44,22 @@ public Optional beforeJob(TestPlan testPlan, TestUri.Builder uriBui """); var env = job.withoutMatrix(testPlan); - var setupRequest = new TestRunRequest( - env.substitutor().replace("createKeyspace: ${KEYSPACE_NAME}"), - command, testPlan.target(), env, TestAssertion.forSuccess( testPlan, command)); + var setupRequest = + new TestRunRequest( + env.substitutor().replace("createKeyspace: ${KEYSPACE_NAME}"), + command, + testPlan.target(), + env, + TestAssertion.forSuccess(testPlan, command)); return Optional.of(setupRequest.testNodes(uriBuilder)); } @Override public Optional afterJob(TestPlan testPlan, TestUri.Builder uriBuilder, Job job) { - var command = TestCommand.fromJson( - """ + var command = + TestCommand.fromJson( + """ { "dropKeyspace": { "name": "${KEYSPACE_NAME}" @@ -61,9 +68,13 @@ public Optional afterJob(TestPlan testPlan, TestUri.Builder uriBuil """); var env = job.withoutMatrix(testPlan); - var setupRequest = new TestRunRequest( - env.substitutor().replace("dropKeyspace: ${KEYSPACE_NAME}"), - command, testPlan.target(), job.withoutMatrix(testPlan), TestAssertion.forSuccess(testPlan,command)); + var setupRequest = + new TestRunRequest( + env.substitutor().replace("dropKeyspace: ${KEYSPACE_NAME}"), + command, + testPlan.target(), + job.withoutMatrix(testPlan), + TestAssertion.forSuccess(testPlan, command)); return Optional.of(setupRequest.testNodes(uriBuilder)); } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/targets/Connection.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/targets/Connection.java index 16e6d01fdc..29803a2cd8 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/targets/Connection.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/targets/Connection.java @@ -1,10 +1,6 @@ package io.stargate.sgv2.jsonapi.api.v1.vectorize.targets; -public record Connection( - String domain, - Integer port, - String basePath -) { +public record Connection(String domain, Integer port, String basePath) { public Connection { if (domain == null) { domain = "localhost"; diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/targets/Target.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/targets/Target.java index 67e845fee0..88dbc3c13a 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/targets/Target.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/targets/Target.java @@ -4,16 +4,15 @@ import io.stargate.sgv2.jsonapi.api.v1.vectorize.lifecycle.TestPlanLifecycle; import io.stargate.sgv2.jsonapi.api.v1.vectorize.messaging.APIRequest; import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestRunEnv; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestUri; import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.Job; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.TestCommand; import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.TargetConfiguration; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.TestCommand; import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.TestSuiteSpec; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestUri; import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.WorkflowSpec; -import org.junit.jupiter.api.DynamicNode; - import java.util.HashMap; import java.util.Optional; +import org.junit.jupiter.api.DynamicNode; public class Target implements TestPlanLifecycle { @@ -25,53 +24,63 @@ public Target(TargetConfiguration targetConfiguration) { this.targetConfiguration = targetConfiguration; this.env = new TestRunEnv(new HashMap<>()); - this.backend = switch (targetConfiguration.backend()) { - case "cassandra" -> new CassandraBackend(); - case "astra" -> new AstraBackend(); - default -> throw new IllegalArgumentException("Unknown backend: " + targetConfiguration.backend()); - }; + this.backend = + switch (targetConfiguration.backend()) { + case "cassandra" -> new CassandraBackend(); + case "astra" -> new AstraBackend(); + default -> + throw new IllegalArgumentException( + "Unknown backend: " + targetConfiguration.backend()); + }; } - public TargetConfiguration configuration(){ + public TargetConfiguration configuration() { return targetConfiguration; } - public Connection connection(){ + + public Connection connection() { return targetConfiguration.connection(); } - public void updateJobForTarget(Job job){ + public void updateJobForTarget(Job job) { backend.updateJobForTarget(job); } - - public APIRequest apiRequest(TestCommand testCommand, TestRunEnv env){ - return new APIRequest(targetConfiguration.connection(), env, testCommand.withEnvironment(env)); + public APIRequest apiRequest(TestCommand testCommand, TestRunEnv env) { + return new APIRequest(targetConfiguration.connection(), env, testCommand.withEnvironment(env)); } @Override - public Optional beforeWorkflow(TestPlan testPlan, TestUri.Builder uriBuilder, WorkflowSpec workflow){ - return backend.beforeWorkflow(testPlan, uriBuilder,workflow); + public Optional beforeWorkflow( + TestPlan testPlan, TestUri.Builder uriBuilder, WorkflowSpec workflow) { + return backend.beforeWorkflow(testPlan, uriBuilder, workflow); } + @Override - public Optional afterWorkflow(TestPlan testPlan, TestUri.Builder uriBuilder, WorkflowSpec workflow){ - return backend.afterWorkflow(testPlan, uriBuilder,workflow); + public Optional afterWorkflow( + TestPlan testPlan, TestUri.Builder uriBuilder, WorkflowSpec workflow) { + return backend.afterWorkflow(testPlan, uriBuilder, workflow); } @Override - public Optional beforeJob(TestPlan testPlan, TestUri.Builder uriBuilder, Job job){ - return backend.beforeJob(testPlan, uriBuilder,job); + public Optional beforeJob(TestPlan testPlan, TestUri.Builder uriBuilder, Job job) { + return backend.beforeJob(testPlan, uriBuilder, job); } + @Override - public Optional afterJob(TestPlan testPlan, TestUri.Builder uriBuilder, Job job){ - return backend.afterJob(testPlan,uriBuilder, job); + public Optional afterJob(TestPlan testPlan, TestUri.Builder uriBuilder, Job job) { + return backend.afterJob(testPlan, uriBuilder, job); } @Override - public Optional beforeTestSuite(TestPlan testPlan, TestUri.Builder uriBuilder, TestSuiteSpec test, TestRunEnv env){ - return backend.beforeTestSuite(testPlan,uriBuilder, test, env); + public Optional beforeTestSuite( + TestPlan testPlan, TestUri.Builder uriBuilder, TestSuiteSpec test, TestRunEnv env) { + return backend.beforeTestSuite(testPlan, uriBuilder, test, env); } + @Override - public Optional afterTestSuite(TestPlan testPlan, TestUri.Builder uriBuilder, TestSuiteSpec test, TestRunEnv env){ - return backend.afterTestSuite(testPlan, uriBuilder,test, env); + public Optional afterTestSuite( + TestPlan testPlan, TestUri.Builder uriBuilder, TestSuiteSpec test, TestRunEnv env) { + return backend.afterTestSuite(testPlan, uriBuilder, test, env); } } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testrun/DynamicTestExecutable.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testrun/DynamicTestExecutable.java index d09b28820e..9e688bd8c4 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testrun/DynamicTestExecutable.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testrun/DynamicTestExecutable.java @@ -1,11 +1,11 @@ package io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun; +import static org.junit.jupiter.api.DynamicTest.dynamicTest; + import org.junit.jupiter.api.DynamicTest; import org.junit.jupiter.api.function.Executable; -import static org.junit.jupiter.api.DynamicTest.dynamicTest; - -public class DynamicTestExecutable implements Executable { +public class DynamicTestExecutable implements Executable { private final String description; private final TestUri testUri; @@ -14,30 +14,30 @@ public class DynamicTestExecutable implements Executable { private final String trimmedDisplayName; private final boolean isTrimmed; - public DynamicTestExecutable(String description, TestUri.Builder testUri, Executable executable) { + public DynamicTestExecutable(String description, TestUri.Builder testUri, Executable executable) { this(description, testUri.build(), executable); } @SuppressWarnings("StringEquality") - public DynamicTestExecutable(String description, TestUri testUri, Executable executable) { + public DynamicTestExecutable(String description, TestUri testUri, Executable executable) { this.description = description; this.testUri = testUri; this.executable = executable; - var truncated = ( description != null && description.length() > 60) ? - description.substring(0, 57) + "..." - : - description; + var truncated = + (description != null && description.length() > 60) + ? description.substring(0, 57) + "..." + : description; this.trimmedDisplayName = truncated; this.isTrimmed = truncated != description; } - public String trimmedDisplayName(){ - return trimmedDisplayName; + public String trimmedDisplayName() { + return trimmedDisplayName; } - public DynamicTest testNode(){ + public DynamicTest testNode() { return dynamicTest(trimmedDisplayName, testUri.uri(), this); } @@ -48,13 +48,13 @@ public void execute() throws Throwable { afterExecute(); } - private void beforeExecute(){ + private void beforeExecute() { if (isTrimmed) { System.out.printf(description + "\n"); } } - private void afterExecute(){ - System.out.printf("Executed - " + testUri.uri().toString() + "\n"); + private void afterExecute() { + // System.out.printf("Executed - " + testUri.uri().toString() + "\n"); } } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testrun/TestRunEnv.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testrun/TestRunEnv.java index d45adf0b8a..6d035237c1 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testrun/TestRunEnv.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testrun/TestRunEnv.java @@ -2,19 +2,16 @@ import io.stargate.sgv2.jsonapi.api.v1.vectorize.TestPlan; import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.TestSuiteSpec; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; +import java.util.regex.Pattern; import org.apache.commons.text.StringSubstitutor; import org.apache.commons.text.lookup.StringLookupFactory; import org.junit.jupiter.api.DynamicContainer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.HashMap; -import java.util.Map; -import java.util.Set; -import java.util.regex.Pattern; - -import static org.junit.jupiter.api.DynamicTest.dynamicTest; - public class TestRunEnv { private static final Logger LOGGER = LoggerFactory.getLogger(TestRunEnv.class); @@ -24,7 +21,7 @@ public class TestRunEnv { private final Map vars = new HashMap<>(); - public TestRunEnv(){ + public TestRunEnv() { this(new HashMap<>()); } @@ -32,7 +29,8 @@ public TestRunEnv(Map vars) { this.vars.putAll(vars); } - public DynamicContainer testNode(TestPlan testPlan, TestUri.Builder uriBuilder, TestSuiteSpec testSuite) { + public DynamicContainer testNode( + TestPlan testPlan, TestUri.Builder uriBuilder, TestSuiteSpec testSuite) { var d = description(); uriBuilder.addSegment(TestUri.Segment.ENV, d); @@ -41,80 +39,83 @@ public DynamicContainer testNode(TestPlan testPlan, TestUri.Builder uriBuilder, var envNodes = testSuite.testNodesForEnvironment(testPlan, uriBuilder.clone(), this).stream(); return DynamicContainer.dynamicContainer( - desc, - testPlan.addLifecycle(uriBuilder.clone(), testSuite, this, envNodes)); + desc, + uriBuilder.build().uri(), + testPlan.addLifecycle(uriBuilder.clone(), testSuite, this, envNodes)); } - private String description(){ + private String description() { return vars.entrySet().stream() .filter( entry -> { var name = entry.getKey().toUpperCase(); - if (SCHEMA_IDENTIFIER.contains(name)){ + if (SCHEMA_IDENTIFIER.contains(name)) { return false; // schema names not usually interesting } - if (name.contains("KEY") || name.contains("API") || name.contains("TOKEN")){ + if (name.contains("KEY") || name.contains("API") || name.contains("TOKEN")) { return false; // assume security } return true; - } - ) + }) .sorted(Map.Entry.comparingByKey()) .toList() .toString(); } - private TestRunEnv(TestRunEnv other){ + private TestRunEnv(TestRunEnv other) { this.vars.putAll(other.vars); } - public TestRunEnv clone(){ + public TestRunEnv clone() { return new TestRunEnv(this); } - public TestRunEnv put(TestRunEnv other){ + public TestRunEnv put(TestRunEnv other) { this.vars.putAll(other.vars); return this; } - public void put(String key, String value){ + public void put(String key, String value) { this.vars.put(key, value); } - public String requiredValue(String name){ - if (vars.containsKey(name)){ + public String requiredValue(String name) { + if (vars.containsKey(name)) { return get(name); } - throw new RuntimeException(String.format("Required env var not found name:%s, defined: %s", name, String.join(", ", vars.keySet()))); + throw new RuntimeException( + String.format( + "Required env var not found name:%s, defined: %s", + name, String.join(", ", vars.keySet()))); } - public StringSubstitutor substitutor(){ + public StringSubstitutor substitutor() { - return new StringSubstitutor(StringLookupFactory.INSTANCE.functionStringLookup(this::get)).setEnableUndefinedVariableException(true); + return new StringSubstitutor(StringLookupFactory.INSTANCE.functionStringLookup(this::get)) + .setEnableUndefinedVariableException(true); } - private String get(String name){ + + private String get(String name) { var value = vars.get(name); - if (value == null){ + if (value == null) { return ""; } var substituted = substitutor().replace(value); - var cleaned = SCHEMA_IDENTIFIER.contains(name) ? - toSafeSchemaIdentifier(substituted) - : - substituted; + var cleaned = + SCHEMA_IDENTIFIER.contains(name) ? toSafeSchemaIdentifier(substituted) : substituted; return cleaned; } - - public static String toSafeSchemaIdentifier(String name){ + public static String toSafeSchemaIdentifier(String name) { var newValue = PATTERN_NOT_WORD_CHARS.matcher(name).replaceAll("_"); - if (newValue.length() > 48){ + if (newValue.length() > 48) { return newValue.substring(0, 47); -// throw new RuntimeException("Schema Identifier longer than 48 characters orginalName=%s, afterNormalisation==%s".formatted(name,newValue)); + // throw new RuntimeException("Schema Identifier longer than 48 characters + // orginalName=%s, afterNormalisation==%s".formatted(name,newValue)); } return newValue; } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testrun/TestRunRequest.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testrun/TestRunRequest.java index 2c362efd32..ec4b37c70c 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testrun/TestRunRequest.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testrun/TestRunRequest.java @@ -1,17 +1,15 @@ package io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun; +import static org.junit.jupiter.api.DynamicContainer.dynamicContainer; + import io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions.TestAssertion; import io.stargate.sgv2.jsonapi.api.v1.vectorize.targets.Target; import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.TestCommand; -import org.junit.jupiter.api.DynamicContainer; -import org.junit.jupiter.api.DynamicNode; - import java.util.List; import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Stream; - -import static org.junit.jupiter.api.DynamicContainer.dynamicContainer; -import static org.junit.jupiter.api.DynamicTest.dynamicTest; +import org.junit.jupiter.api.DynamicContainer; +import org.junit.jupiter.api.DynamicNode; public record TestRunRequest( String name, @@ -30,27 +28,36 @@ public DynamicContainer testNodes(TestUri.Builder uriBuilder) { uriBuilder.addSegment(TestUri.Segment.REQUEST, name()); - Stream.Builder nodesBuilder = Stream.builder();; + Stream.Builder nodesBuilder = Stream.builder(); + ; AtomicReference atomicResponse = new AtomicReference<>(); // Execute the request, and set so the assertions can pull the response after. - var commandExecutable = new DynamicTestExecutable( - "Command: " + testCommand.commandName().getApiName(), - uriBuilder.clone().addSegment(TestUri.Segment.COMMAND, testCommand.commandName().getApiName()), - () -> atomicResponse.set(execute()) - ); + var commandExecutable = + new DynamicTestExecutable( + "Command: " + testCommand.commandName().getApiName(), + uriBuilder + .clone() + .addSegment(TestUri.Segment.COMMAND, testCommand.commandName().getApiName()), + () -> atomicResponse.set(execute())); nodesBuilder.add(commandExecutable.testNode()); // tests for each assertion - var assertionsUriBuilder = uriBuilder.clone().addSegment(TestUri.Segment.ASSERTION_CONTAINER, "assertions"); - var assertionTests = testAssertions().stream().map( - testAssertion -> testAssertion.testNodes(assertionsUriBuilder.clone(), atomicResponse)) - .toList(); + var assertionsUriBuilder = + uriBuilder.clone().addSegment(TestUri.Segment.ASSERTION_CONTAINER, "assertions"); + var assertionTests = + testAssertions().stream() + .map( + testAssertion -> + testAssertion.testNodes(assertionsUriBuilder.clone(), atomicResponse)) + .toList(); // if we have assertion tests, put them in a container if (!assertionTests.isEmpty()) { - nodesBuilder.add(dynamicContainer("Assertions",assertionsUriBuilder.build().uri(), assertionTests.stream())); + nodesBuilder.add( + dynamicContainer( + "Assertions", assertionsUriBuilder.build().uri(), assertionTests.stream())); } return dynamicContainer("Request: " + name, uriBuilder.build().uri(), nodesBuilder.build()); diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testrun/TestRunResponse.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testrun/TestRunResponse.java index df39678b70..ea7e869fb9 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testrun/TestRunResponse.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testrun/TestRunResponse.java @@ -4,33 +4,30 @@ import io.stargate.sgv2.jsonapi.api.v1.vectorize.messaging.APIResponse; public record TestRunResponse( - TestRunRequest testRequest, - APIRequest apiRequest, - APIResponse apiResponse -) { + TestRunRequest testRequest, APIRequest apiRequest, APIResponse apiResponse) { -// public TestCaseResult validate(TestSuite integrationTest, TestCase testCase){ -// return validate(integrationTest, testCase, true); -// } -// -// public TestCaseResult validate(TestSuite testSuite, TestCase testCase, boolean throwOnError) { -// -// AssertionError assertionError = null; -// AssertionMatcher failedAssertion = null; -// for (var testAssertion : testRequest.testAssertions()){ -// try{ -// testAssertion.run(apiResponse); -// } -// catch(AssertionError e){ -// if (throwOnError){ -// throw e; -// } -// -// failedAssertion = testAssertion; -// assertionError = e; -// break; -// } -// } -// return new TestCaseResult(testSuite, testCase, this , assertionError, failedAssertion); -// } + // public TestCaseResult validate(TestSuite integrationTest, TestCase testCase){ + // return validate(integrationTest, testCase, true); + // } + // + // public TestCaseResult validate(TestSuite testSuite, TestCase testCase, boolean throwOnError) { + // + // AssertionError assertionError = null; + // AssertionMatcher failedAssertion = null; + // for (var testAssertion : testRequest.testAssertions()){ + // try{ + // testAssertion.run(apiResponse); + // } + // catch(AssertionError e){ + // if (throwOnError){ + // throw e; + // } + // + // failedAssertion = testAssertion; + // assertionError = e; + // break; + // } + // } + // return new TestCaseResult(testSuite, testCase, this , assertionError, failedAssertion); + // } } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testrun/TestUri.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testrun/TestUri.java index 5764078afa..a9973b2394 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testrun/TestUri.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testrun/TestUri.java @@ -1,48 +1,53 @@ package io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun; +import static java.util.stream.Collectors.joining; + import java.net.URI; import java.util.*; import java.util.regex.Pattern; import java.util.stream.Stream; -import static java.util.stream.Collectors.joining; -import static java.util.stream.Collectors.toMap; - -public record TestUri( - Scheme scheme, - List segments) { +public record TestUri(Scheme scheme, List segments) { - public static TestUri.Builder builder(Scheme scheme){ - return new TestUri.Builder(scheme); + public static TestUri.Builder builder(Scheme scheme) { + return new TestUri.Builder(scheme); } - public Segment leafType(){ + public Segment leafType() { return segments.getLast().segment; } public URI uri() { - var path = segments.stream() - .map(SegmentValue::toString) - .collect(joining("/")); - return URI.create(scheme.name() + "://" + path); + var path = segments.stream().map(SegmentValue::toString).collect(joining("/")); + + return URI.create(scheme.name() + "://" + AUTHORITY + "/" + path); } - public static TestUri parse(URI uri) { + public static Optional parse(URI uri) { + + Scheme scheme; + try { + scheme = Scheme.valueOf(uri.getScheme().toUpperCase()); + } catch (IllegalArgumentException e) { + return Optional.empty(); + } - var builder = builder(Scheme.valueOf(uri.getScheme().toLowerCase())); - SegmentValue.parse(uri) - .forEach(builder::addSegment); - return builder.build(); + var builder = builder(scheme); + SegmentValue.parse(uri).forEach(builder::addSegment); + return Optional.of(builder.build()); } public enum Scheme { - TESTRUN; + DATAAPI; public String pathName() { return name().toLowerCase(); } } + // aka the domain + public static final String AUTHORITY = "TESTRUN"; + public enum Segment { TARGET(null), LIFECYCLE(null), // used in multiple places @@ -69,7 +74,7 @@ public Segment parent() { public boolean isParentValid(Segment segment) { // XXX TODO: needs work return true; - //return (parent == null) || (parent == segment) || (segment == LIFECYCLE) ; + // return (parent == null) || (parent == segment) || (segment == LIFECYCLE) ; } public String pathName() { @@ -81,20 +86,22 @@ public record SegmentValue(Segment segment, String value) { private static final Pattern INVALID_CHARS = Pattern.compile("[^a-zA-Z0-9\\-_.~]"); - public SegmentValue{ + public SegmentValue { Objects.requireNonNull(segment, "segment must not be null"); Objects.requireNonNull(value, "value must not be null"); } public static Stream parse(URI uri) { - return Arrays.stream(uri.getPath().split("/")) - .map(SegmentValue::parse); + var path = uri.getPath().startsWith("/") ? uri.getPath().substring(1) : uri.getPath(); + + return Arrays.stream(path.split("/")).map(SegmentValue::parse); } public static SegmentValue parse(String segmentKeyValue) { var parts = segmentKeyValue.split("=", 2); if (parts.length != 2) { - throw new IllegalArgumentException("Invalid segment, expected key=value format: " + segmentKeyValue); + throw new IllegalArgumentException( + "Invalid segment, expected key=value format: " + segmentKeyValue); } try { return new SegmentValue(Segment.valueOf(parts[0].toUpperCase()), parts[1]); @@ -109,7 +116,6 @@ public String toString() { } } - public static class Builder { private final Scheme scheme; @@ -134,7 +140,7 @@ public Builder addSegment(Segment segment, String value) { return this; } - public Builder clone(){ + public Builder clone() { return new Builder(scheme, new ArrayList<>(segmentValues)); } @@ -146,11 +152,9 @@ public TestUri build() { throw new IllegalArgumentException( "Invalid segment order. segment=%s expected parent=%s but previous=%s" .formatted(current.segment(), current.segment().parent(), previous)); - } } return new TestUri(scheme, List.copyOf(segmentValues)); } } - } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/AssertionTemplateSpec.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/AssertionTemplateSpec.java index a67c679468..0d714e7280 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/AssertionTemplateSpec.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/AssertionTemplateSpec.java @@ -1,20 +1,13 @@ package io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec; import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.ObjectNode; -import io.quarkus.smallrye.health.runtime.SmallRyeIndividualHealthGroupHandler; -import io.stargate.sgv2.jsonapi.api.v1.util.scenarios.ThreeClusteringKeysTableScenario; -import io.stargate.sgv2.jsonapi.service.schema.collections.CqlColumnMatcher; - import java.util.Map; import java.util.Optional; -public record AssertionTemplateSpec( - TestSpecMeta meta, - Map templates -) implements TestSpec { +public record AssertionTemplateSpec(TestSpecMeta meta, Map templates) + implements TestSpec { - public Optional templateFor(String name){ + public Optional templateFor(String name) { return Optional.ofNullable(templates.get(name)); } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/Job.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/Job.java index cb10e7c22a..24d372195b 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/Job.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/Job.java @@ -1,21 +1,19 @@ package io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestRunEnv; +import static org.junit.jupiter.api.DynamicContainer.dynamicContainer; + import io.stargate.sgv2.jsonapi.api.v1.vectorize.TestPlan; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestRunEnv; import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestUri; -import org.junit.jupiter.api.DynamicContainer; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.stream.Stream; - -import static org.junit.jupiter.api.DynamicContainer.dynamicContainer; +import org.junit.jupiter.api.DynamicContainer; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public record Job( - TestSpecMeta meta, Map fromEnvironment, Map variables, @@ -24,35 +22,35 @@ public record Job( private static final Logger LOGGER = LoggerFactory.getLogger(Job.class); - public DynamicContainer testNode(TestPlan testPlan, TestUri.Builder uriBuilder) { uriBuilder.addSegment(TestUri.Segment.JOB, meta.name()); - var desc = "Job: %s ".formatted( - meta.name()); + var desc = "Job: %s ".formatted(meta.name()); testPlan.updateJobForTarget(this); var allEnvs = allEnvironments(testPlan); - var testSuiteNodes = testSuites(testPlan) - .map(testSuite -> testSuite.testNode(testPlan, uriBuilder.clone(), allEnvs)); - - return dynamicContainer( - desc, - uriBuilder.build().uri(), - testPlan.addLifecycle(uriBuilder.clone(),this, testSuiteNodes)); + var testSuiteNodes = + testSuites(testPlan) + .map(testSuite -> testSuite.testNode(testPlan, uriBuilder.clone(), allEnvs)); + + return dynamicContainer( + desc, + uriBuilder.build().uri(), + testPlan.addLifecycle(uriBuilder.clone(), this, testSuiteNodes)); } public Stream testSuites(TestPlan testPlan) { - Stream.Builder allTests = Stream.builder(); + Stream.Builder allTests = Stream.builder(); tests() .forEach( testName -> { - testPlan.specFiles().byNameAsType(TestSuiteSpec.class, testName).forEach(allTests) ; + testPlan.specFiles().byNameAsType(TestSuiteSpec.class, testName).forEach(allTests); }); return allTests.build(); } + public TestRunEnv withoutMatrix(TestPlan testPlan) { var fromEnv = new TestRunEnv(); @@ -65,6 +63,7 @@ public TestRunEnv withoutMatrix(TestPlan testPlan) { return fromEnv.clone().put(fromVariables); } + public List allEnvironments(TestPlan testPlan) { var fromEnv = new TestRunEnv(); @@ -77,20 +76,19 @@ public List allEnvironments(TestPlan testPlan) { // TODO: handle more matrix List fromMatrix = new ArrayList<>(); - matrix.get("MODEL").forEach( - model -> { - var env = new TestRunEnv(); - env.put("MODEL", model); - fromMatrix.add(env); - }); + matrix + .get("MODEL") + .forEach( + model -> { + var env = new TestRunEnv(); + env.put("MODEL", model); + fromMatrix.add(env); + }); List allEnvs = new ArrayList<>(); for (var matrixEnv : fromMatrix) { - var completeEnv = fromEnv - .clone() - .put(fromVariables) - .put(matrixEnv); + var completeEnv = fromEnv.clone().put(fromVariables).put(matrixEnv); allEnvs.add(completeEnv); } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/SpecFile.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/SpecFile.java index a3ddf724d7..f240bb6d01 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/SpecFile.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/SpecFile.java @@ -1,14 +1,10 @@ package io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec; import com.fasterxml.jackson.databind.JsonNode; -import org.jspecify.annotations.NonNull; - import java.io.File; +import org.jspecify.annotations.NonNull; -public record SpecFile( - File file, - TestSpec spec, - JsonNode root) { +public record SpecFile(File file, TestSpec spec, JsonNode root) { @Override public @NonNull String toString() { @@ -19,4 +15,4 @@ public record SpecFile( .append(spec.meta()) .toString(); } -} \ No newline at end of file +} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/SpecFiles.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/SpecFiles.java index bf0e4eac9b..27b7fff883 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/SpecFiles.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/SpecFiles.java @@ -2,7 +2,6 @@ import com.fasterxml.jackson.databind.MapperFeature; import com.fasterxml.jackson.databind.ObjectMapper; - import java.io.IOException; import java.io.UncheckedIOException; import java.net.URISyntaxException; @@ -13,13 +12,11 @@ import java.util.function.Predicate; import java.util.stream.Stream; -/** - * Collection of all the test spec files read for this execution. - */ +/** Collection of all the test spec files read for this execution. */ public class SpecFiles { - private static final ObjectMapper MAPPER = new ObjectMapper() - .configure(MapperFeature.ACCEPT_CASE_INSENSITIVE_ENUMS, true); + private static final ObjectMapper MAPPER = + new ObjectMapper().configure(MapperFeature.ACCEPT_CASE_INSENSITIVE_ENUMS, true); private final List specFiles; @@ -27,28 +24,24 @@ private SpecFiles(List specFiles) { this.specFiles = specFiles; for (SpecFile file : specFiles) { - if (file.spec() instanceof TestSuiteSpec it){ - it.expand(this); - } + if (file.spec() instanceof TestSuiteSpec it) { + it.expand(this); + } } } public static SpecFiles loadAll(List paths) { - var specFiles = resourceDirs(paths) - .flatMap(SpecFiles::loadAll) - .toList(); + var specFiles = resourceDirs(paths).flatMap(SpecFiles::loadAll).toList(); return new SpecFiles(specFiles); } public Stream byKind(TestSpecKind kind) { - return specFiles.stream() - .filter(itFile -> itFile.spec().meta().kind() == kind); + return specFiles.stream().filter(itFile -> itFile.spec().meta().kind() == kind); } public Stream byType(Class clazz) { - return byKind(TestSpecKind.fromType(clazz)) - .map(specFile -> specFile.spec().asSpecType(clazz)); + return byKind(TestSpecKind.fromType(clazz)).map(specFile -> specFile.spec().asSpecType(clazz)); } public Stream byName(TestSpecKind kind, String name) { @@ -60,22 +53,21 @@ public Stream byNameAsType(Class clazz, String name) .map(specFile -> specFile.spec().asSpecType(clazz)); } - private Stream match(TestSpecKind kind, Predicate predicate) { - return byKind(kind) - .filter(specFile -> predicate.test(specFile.spec())); + return byKind(kind).filter(specFile -> predicate.test(specFile.spec())); } private static Stream loadAll(Path path) { try (Stream pathStream = Files.walk(path)) { - return pathStream.filter(Files::isRegularFile) - .filter(SpecFiles::isJsonFile) - .map(SpecFiles::loadOne) - .toList() // force so the files are read before closing - .stream(); + return pathStream + .filter(Files::isRegularFile) + .filter(SpecFiles::isJsonFile) + .map(SpecFiles::loadOne) + .toList() // force so the files are read before closing + .stream(); } catch (IOException e) { - throw new UncheckedIOException("Failed reading test resources under: " + path , e); + throw new UncheckedIOException("Failed reading test resources under: " + path, e); } } @@ -92,10 +84,9 @@ private static SpecFile loadOne(Path path) { var element = switch (TestSpecKind.valueOf(kindNode.asText().toUpperCase())) { case ASSERTION_TEMPLATE -> MAPPER.treeToValue(root, AssertionTemplateSpec.class); - case TARGETS -> MAPPER.treeToValue(root, TargetsSpec.class); + case TARGETS -> MAPPER.treeToValue(root, TargetsSpec.class); case TEST_SUITE -> MAPPER.treeToValue(root, TestSuiteSpec.class); case WORKFLOW -> MAPPER.treeToValue(root, WorkflowSpec.class); - }; return new SpecFile(file, element, root); } catch (IOException e) { @@ -111,23 +102,25 @@ private static Stream resourceDirs(List paths) { var cl = Thread.currentThread().getContextClassLoader(); - return paths.stream().map( - path -> { - String normalized = path.startsWith("/") ? path.substring(1) : path; - - var url = cl.getResource(normalized); - if (url == null) { - throw new IllegalArgumentException("Test resource folder not found: " + path); - } - - try { - // Works for file: URLs; if you run tests from a jar, switch to getResourceAsStream-based - // walking. - return Paths.get(url.toURI()); - } catch (URISyntaxException e) { - throw new IllegalArgumentException("Bad resource URI for: " + path + " -> " + url, e); - } - } - ); + return paths.stream() + .map( + path -> { + String normalized = path.startsWith("/") ? path.substring(1) : path; + + var url = cl.getResource(normalized); + if (url == null) { + throw new IllegalArgumentException("Test resource folder not found: " + path); + } + + try { + // Works for file: URLs; if you run tests from a jar, switch to + // getResourceAsStream-based + // walking. + return Paths.get(url.toURI()); + } catch (URISyntaxException e) { + throw new IllegalArgumentException( + "Bad resource URI for: " + path + " -> " + url, e); + } + }); } } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/TargetConfiguration.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/TargetConfiguration.java index 92491b097a..e9974df284 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/TargetConfiguration.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/TargetConfiguration.java @@ -2,5 +2,4 @@ import io.stargate.sgv2.jsonapi.api.v1.vectorize.targets.Connection; -public record TargetConfiguration(String name, String backend, Connection connection) { -} +public record TargetConfiguration(String name, String backend, Connection connection) {} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/TargetsSpec.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/TargetsSpec.java index abf6ea45db..a0b8188ed9 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/TargetsSpec.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/TargetsSpec.java @@ -2,7 +2,6 @@ import com.fasterxml.jackson.databind.MapperFeature; import com.fasterxml.jackson.databind.ObjectMapper; - import java.io.IOException; import java.net.URISyntaxException; import java.net.URL; @@ -12,27 +11,27 @@ import java.util.List; import java.util.Set; -public record TargetsSpec( - TestSpecMeta meta, - List targets) +public record TargetsSpec(TestSpecMeta meta, List targets) implements TestSpec { - private static final ObjectMapper MAPPER = new ObjectMapper() - .configure(MapperFeature.ACCEPT_CASE_INSENSITIVE_ENUMS, true); - + private static final ObjectMapper MAPPER = + new ObjectMapper().configure(MapperFeature.ACCEPT_CASE_INSENSITIVE_ENUMS, true); public TargetsSpec { Set seen = new HashSet(); - for (TargetConfiguration target : targets) { + for (TargetConfiguration target : targets) { if (seen.contains(target.name())) { - throw new IllegalArgumentException("target name already exists: " + target.name() ); + throw new IllegalArgumentException("target name already exists: " + target.name()); } seen.add(target.name()); } } public TargetConfiguration configuration(String name) { - return targets.stream().filter(target -> target.name().equals(name)).findFirst().orElseThrow(() -> new IllegalArgumentException("target name not found: " + name)); + return targets.stream() + .filter(target -> target.name().equals(name)) + .findFirst() + .orElseThrow(() -> new IllegalArgumentException("target name not found: " + name)); } public static TargetsSpec loadAll(String path) { @@ -45,7 +44,7 @@ public static TargetsSpec loadAll(String path) { } } - private static Path resourceDir(String path) { + public static Path resourceDir(String path) { String normalized = path.startsWith("/") ? path.substring(1) : path; ClassLoader cl = Thread.currentThread().getContextClassLoader(); @@ -62,5 +61,4 @@ private static Path resourceDir(String path) { throw new IllegalArgumentException("Bad resource URI for: " + path + " -> " + url, e); } } - } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/TestCase.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/TestCase.java index 670ab9e73d..f7e5ff7d72 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/TestCase.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/TestCase.java @@ -2,29 +2,30 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.databind.node.ObjectNode; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestRunEnv; import io.stargate.sgv2.jsonapi.api.v1.vectorize.TestPlan; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestRunRequest; import io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions.TestAssertion; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestRunEnv; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestRunRequest; import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestUri; import org.junit.jupiter.api.DynamicContainer; public record TestCase( - String name, TestCommand command, ObjectNode asserts, - @JsonProperty("$include") - String include) { - + @JsonProperty("$include") String include) { - public DynamicContainer testNodesForEnvironment(TestPlan testPlan, TestUri.Builder uriBuilder, TestRunEnv testEnvironment) { + public DynamicContainer testNodesForEnvironment( + TestPlan testPlan, TestUri.Builder uriBuilder, TestRunEnv testEnvironment) { - var testRequest = new TestRunRequest( + var testRequest = + new TestRunRequest( "TestCase: name=%s".formatted(name, command.commandName()), - command(), testPlan.target(), testEnvironment, TestAssertion.buildAssertions(testPlan, this)); - - return testRequest.testNodes(uriBuilder); - } + command(), + testPlan.target(), + testEnvironment, + TestAssertion.buildAssertions(testPlan, this)); + return testRequest.testNodes(uriBuilder); + } } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/TestCommand.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/TestCommand.java index 08632e41c1..b8661633fe 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/TestCommand.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/TestCommand.java @@ -7,11 +7,10 @@ import com.fasterxml.jackson.databind.node.ObjectNode; import com.fasterxml.jackson.databind.node.TextNode; import io.stargate.sgv2.jsonapi.api.model.command.CommandName; - import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestRunEnv; import org.apache.commons.text.StringSubstitutor; -public class TestCommand { +public class TestCommand { private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); @@ -23,28 +22,29 @@ public class TestCommand { public TestCommand(ObjectNode request) { // if non null, that this is a point to find commands in the named test. - var includeField =request.get("$include"); + var includeField = request.get("$include"); this.includeFrom = includeField == null ? null : includeField.asText(); if (includeField != null) { this.request = null; this.commandName = null; - } - else { + } else { this.request = request; this.commandName = commandName(request); } } - private void checkIsInclude(){ + private void checkIsInclude() { if (includeFrom != null) { throw new IllegalStateException("TestCommand is defined to $include from: " + includeFrom); } } + public ObjectNode request() { checkIsInclude(); return request; } + public CommandName commandName() { checkIsInclude(); return commandName; @@ -57,7 +57,7 @@ public String includeFrom() { public static TestCommand fromJson(String json) { try { - return new TestCommand((ObjectNode) OBJECT_MAPPER.readTree(json)); + return new TestCommand((ObjectNode) OBJECT_MAPPER.readTree(json)); } catch (JsonProcessingException e) { throw new RuntimeException(e); } @@ -91,7 +91,6 @@ public ObjectNode withEnvironment(TestRunEnv env) { return updated; } - private static void walk(ObjectNode obj, StringSubstitutor subs) { obj.properties() .forEach( diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/TestEnvAccess.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/TestEnvAccess.java index b247b11a5d..8d1e174a09 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/TestEnvAccess.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/TestEnvAccess.java @@ -9,12 +9,13 @@ public static void putEnvVar(String key, String value) { public static String getEnvVar(String varName) { var value = System.getProperty(varName); - if (value != null){ + if (value != null) { return value; } value = System.getenv(varName); - if (value== null) { - throw new RuntimeException("Environment variable not found in System Properties or Environment. varName=" + varName); + if (value == null) { + throw new RuntimeException( + "Environment variable not found in System Properties or Environment. varName=" + varName); } return value; } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/TestSpec.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/TestSpec.java index 86460164ff..ff1ce62d4d 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/TestSpec.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/TestSpec.java @@ -1,6 +1,7 @@ package io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec; -public sealed interface TestSpec permits AssertionTemplateSpec, TargetsSpec, TestSuiteSpec, WorkflowSpec { +public sealed interface TestSpec + permits AssertionTemplateSpec, TargetsSpec, TestSuiteSpec, WorkflowSpec { TestSpecMeta meta(); diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/TestSpecKind.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/TestSpecKind.java index 4f708bea98..cae0d8d361 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/TestSpecKind.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/TestSpecKind.java @@ -7,10 +7,18 @@ public enum TestSpecKind { WORKFLOW; public static TestSpecKind fromType(Class clazz) { - if (clazz == AssertionTemplateSpec.class) { return ASSERTION_TEMPLATE; } - if (clazz == TargetsSpec.class) { return TARGETS; } - if (clazz == TestSuiteSpec.class) { return TEST_SUITE; } - if (clazz == WorkflowSpec.class) { return WORKFLOW; } + if (clazz == AssertionTemplateSpec.class) { + return ASSERTION_TEMPLATE; + } + if (clazz == TargetsSpec.class) { + return TARGETS; + } + if (clazz == TestSuiteSpec.class) { + return TEST_SUITE; + } + if (clazz == WorkflowSpec.class) { + return WORKFLOW; + } throw new IllegalArgumentException("Unknown TestSpec type: " + clazz); } } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/TestSpecMeta.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/TestSpecMeta.java index d42034dbe2..bebc224a36 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/TestSpecMeta.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/TestSpecMeta.java @@ -2,9 +2,4 @@ import java.util.List; -public record TestSpecMeta(String name, - TestSpecKind kind, - List tags) { - - -} +public record TestSpecMeta(String name, TestSpecKind kind, List tags) {} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/TestSuiteSpec.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/TestSuiteSpec.java index 0aca6ed97e..8289428808 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/TestSuiteSpec.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/TestSuiteSpec.java @@ -1,39 +1,37 @@ package io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec; +import static org.junit.jupiter.api.DynamicContainer.dynamicContainer; + import io.stargate.sgv2.jsonapi.api.v1.vectorize.*; import io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions.TestAssertion; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestRunRequest; import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestRunEnv; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestRunRequest; import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestUri; -import org.junit.jupiter.api.DynamicContainer; -import org.junit.jupiter.api.DynamicNode; - import java.util.ArrayList; import java.util.Collection; import java.util.List; +import org.junit.jupiter.api.DynamicContainer; +import org.junit.jupiter.api.DynamicNode; -import static org.junit.jupiter.api.DynamicContainer.dynamicContainer; - -public record TestSuiteSpec(TestSpecMeta meta, List setup, List tests, List cleanup) +public record TestSuiteSpec( + TestSpecMeta meta, List setup, List tests, List cleanup) implements TestSpec { - - public DynamicContainer testNode(TestPlan testPlan, TestUri.Builder uriBuilder, List allEnvs) { + public DynamicContainer testNode( + TestPlan testPlan, TestUri.Builder uriBuilder, List allEnvs) { uriBuilder.addSegment(TestUri.Segment.SUITE, meta().name()); - var desc = "TestSuite: %s ".formatted( - meta.name()); + var desc = "TestSuite: %s ".formatted(meta.name()); return dynamicContainer( desc, uriBuilder.build().uri(), - allEnvs.stream() - .map(testEnv -> testEnv.testNode(testPlan, uriBuilder.clone(), this)) - ); + allEnvs.stream().map(testEnv -> testEnv.testNode(testPlan, uriBuilder.clone(), this))); } - public Collection testNodesForEnvironment(TestPlan testPlan, TestUri.Builder uriBuilder, TestRunEnv testEnvironment) { + public Collection testNodesForEnvironment( + TestPlan testPlan, TestUri.Builder uriBuilder, TestRunEnv testEnvironment) { List nodes = new ArrayList<>(); @@ -41,28 +39,36 @@ public Collection testNodesForEnvironment(TestPlan testPl var setupUriBuilder = uriBuilder.clone().addSegment(TestUri.Segment.STAGE, "setup"); for (TestCommand setupCommand : setup()) { - var setupRequest = new TestRunRequest( - "SetupRequest[%s]: %s".formatted(i++, setupCommand.commandName()), - setupCommand, testPlan.target(), testEnvironment, TestAssertion.forSuccess(testPlan, setupCommand)); + var setupRequest = + new TestRunRequest( + "SetupRequest[%s]: %s".formatted(i++, setupCommand.commandName()), + setupCommand, + testPlan.target(), + testEnvironment, + TestAssertion.forSuccess(testPlan, setupCommand)); nodes.add(setupRequest.testNodes(setupUriBuilder.clone())); } var testUriBuilder = uriBuilder.clone().addSegment(TestUri.Segment.STAGE, "test"); for (var testCase : tests()) { - nodes.add(testCase.testNodesForEnvironment(testPlan, testUriBuilder.clone(), testEnvironment)); + nodes.add( + testCase.testNodesForEnvironment(testPlan, testUriBuilder.clone(), testEnvironment)); } var cleanupUriBuilder = uriBuilder.clone().addSegment(TestUri.Segment.STAGE, "cleanup"); for (TestCommand cleanupCommand : cleanup()) { - var cleanupRequest = new TestRunRequest( - "CleanupRequest[%s]: %s".formatted(i++, cleanupCommand.commandName()), - cleanupCommand, testPlan.target(), testEnvironment, TestAssertion.forSuccess(testPlan,cleanupCommand)); + var cleanupRequest = + new TestRunRequest( + "CleanupRequest[%s]: %s".formatted(i++, cleanupCommand.commandName()), + cleanupCommand, + testPlan.target(), + testEnvironment, + TestAssertion.forSuccess(testPlan, cleanupCommand)); nodes.add(cleanupRequest.testNodes(cleanupUriBuilder.clone())); } return nodes; - } public void expand(SpecFiles specFiles) { @@ -70,9 +76,15 @@ public void expand(SpecFiles specFiles) { List expandedSetup = new ArrayList<>(); for (TestCommand command : setup) { if (command.includeFrom() != null) { - var includedTest = specFiles.byNameAsType(TestSuiteSpec.class, command.includeFrom()) - .findFirst() - .orElseThrow(() -> new IllegalStateException("Included TestSuite Setup not found. parent=%s, included=%s".formatted(meta().name(), command.includeFrom()))); + var includedTest = + specFiles + .byNameAsType(TestSuiteSpec.class, command.includeFrom()) + .findFirst() + .orElseThrow( + () -> + new IllegalStateException( + "Included TestSuite Setup not found. parent=%s, included=%s" + .formatted(meta().name(), command.includeFrom()))); expandedSetup.addAll(includedTest.setup()); } else { @@ -85,9 +97,15 @@ public void expand(SpecFiles specFiles) { List expandedTests = new ArrayList<>(); for (TestCase testCase : tests) { if (testCase.include() != null) { - var includedTest = specFiles.byNameAsType(TestSuiteSpec.class, testCase.include()) - .findFirst() - .orElseThrow(() -> new IllegalStateException("Included TestSuite TestCase not found. parent=%s, included=%s".formatted(meta().name(), testCase.include()))); + var includedTest = + specFiles + .byNameAsType(TestSuiteSpec.class, testCase.include()) + .findFirst() + .orElseThrow( + () -> + new IllegalStateException( + "Included TestSuite TestCase not found. parent=%s, included=%s" + .formatted(meta().name(), testCase.include()))); expandedTests.addAll(includedTest.tests()); } else { diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/WorkflowSpec.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/WorkflowSpec.java index 4d41144a39..dfdb85a2ef 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/WorkflowSpec.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/WorkflowSpec.java @@ -1,39 +1,31 @@ package io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec; +import static org.junit.jupiter.api.DynamicContainer.dynamicContainer; + import io.stargate.sgv2.jsonapi.api.v1.vectorize.TestPlan; import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestUri; -import org.junit.jupiter.api.DynamicContainer; - import java.util.List; -import java.util.stream.Stream; - -import static org.junit.jupiter.api.DynamicContainer.dynamicContainer; +import org.junit.jupiter.api.DynamicContainer; public record WorkflowSpec(TestSpecMeta meta, List jobs) implements TestSpec { - - public DynamicContainer testNode(TestPlan testPlan, TestUri.Builder uriBuilder){ + public DynamicContainer testNode(TestPlan testPlan, TestUri.Builder uriBuilder) { return testNode(testPlan, uriBuilder, true); } - public DynamicContainer testNode(TestPlan testPlan, TestUri.Builder uriBuilder, boolean ignoreDisabled) { + public DynamicContainer testNode( + TestPlan testPlan, TestUri.Builder uriBuilder, boolean ignoreDisabled) { uriBuilder.addSegment(TestUri.Segment.WORKFLOW, meta().name()); - var desc = "Workflow: %s ".formatted( - meta.name()); + var desc = "Workflow: %s ".formatted(meta.name()); - var testNodeJobs = ignoreDisabled ? - jobs().stream() - .filter(job -> !job.meta().tags().contains("disabled")) - : - jobs().stream(); - var jobNodes = testNodeJobs - .map(job -> job.testNode(testPlan, uriBuilder.clone())); + var testNodeJobs = + ignoreDisabled + ? jobs().stream().filter(job -> !job.meta().tags().contains("disabled")) + : jobs().stream(); + var jobNodes = testNodeJobs.map(job -> job.testNode(testPlan, uriBuilder.clone())); return dynamicContainer( - desc, - uriBuilder.build().uri(), - testPlan.addLifecycle(uriBuilder.clone(), this, jobNodes)); + desc, uriBuilder.build().uri(), testPlan.addLifecycle(uriBuilder.clone(), this, jobNodes)); } - } diff --git a/src/test/resources/META-INF/services/org.junit.platform.launcher.TestExecutionListener b/src/test/resources/META-INF/services/org.junit.platform.launcher.TestExecutionListener new file mode 100644 index 0000000000..1dbce58eb5 --- /dev/null +++ b/src/test/resources/META-INF/services/org.junit.platform.launcher.TestExecutionListener @@ -0,0 +1 @@ +io.stargate.sgv2.jsonapi.api.v1.vectorize.DynamicTreeListener \ No newline at end of file diff --git a/src/test/resources/application.yaml b/src/test/resources/application.yaml index 3e963be935..074c9b1c2d 100644 --- a/src/test/resources/application.yaml +++ b/src/test/resources/application.yaml @@ -12,6 +12,10 @@ quarkus: log: console: format: "%-5p [%t] %d{yyyy-MM-dd HH:mm:ss,SSS} %F:%L - %m%n" + handler: + console: + PLAIN_CONSOLE: + format: "%m%n" category: # production log level for this category is DEBUG, way too noisy for tests 'io.stargate': @@ -22,3 +26,8 @@ quarkus: # Fine for JVM mode; if building native images, use index-dependency instead 'io.quarkus.deployment.steps.ReflectiveHierarchyStep': level: ERROR + 'io.stargate.sgv2.jsonapi.api.v1.vectorize.TestBenchConsoleWriter': + level: INFO + handlers: + - PLAIN_CONSOLE + use-parent-handlers: false diff --git a/src/test/resources/integration-tests/test-plans/vectorize-astra-test-plan.yaml b/src/test/resources/integration-tests/test-plans/vectorize-astra-test-plan.yaml new file mode 100644 index 0000000000..a02047b93f --- /dev/null +++ b/src/test/resources/integration-tests/test-plans/vectorize-astra-test-plan.yaml @@ -0,0 +1,12 @@ +name: Astra ENV +customTarget: + name: ${TARGET_NAME} + backend: astra + connection: + domain: ${ENDPOINT} + port: 443 + basePath: /api/json/v1 +workflows: + - vectorize-header-workflow + - vectorize-shared-workflow +ignoreDisabled: true \ No newline at end of file diff --git a/src/test/resources/integration-tests/vectorize/vectorize-header-workflow.json b/src/test/resources/integration-tests/vectorize/vectorize-header-workflow.json index d1cb43fde4..314c8a7e4c 100644 --- a/src/test/resources/integration-tests/vectorize/vectorize-header-workflow.json +++ b/src/test/resources/integration-tests/vectorize/vectorize-header-workflow.json @@ -8,7 +8,6 @@ "meta": { "name": "open-ai-vectorize", "tags": [ - "disabled" ] }, "fromEnvironment": { @@ -177,7 +176,8 @@ }, "matrix": { "MODEL": [ - "NV-Embed-QA" + "NV-Embed-QA", + "nvidia/nv-embedqa-e5-v5-XX" ] }, "tests": [ diff --git a/src/test/resources/integration-tests/vectorize/vectorize-shared-workflow.json b/src/test/resources/integration-tests/vectorize/vectorize-shared-workflow.json index bc57265f78..54a3444226 100644 --- a/src/test/resources/integration-tests/vectorize/vectorize-shared-workflow.json +++ b/src/test/resources/integration-tests/vectorize/vectorize-shared-workflow.json @@ -159,30 +159,6 @@ "tests": [ "vectorize-header-auth" ] - }, - { - "meta": { - "name": "nvidia-vectorize", - "tags": [ - "disabled" - ] - }, - "fromEnvironment": { - "x-embedding-api-key": "Token", - "Token": "Token" - }, - "variables": { - "PROVIDER": "nvidia", - "COLLECTION_NAME": "${PROVIDER}-${MODEL}" - }, - "matrix": { - "MODEL": [ - "NV-Embed-QA" - ] - }, - "tests": [ - "vectorize-header-auth" - ] } ] } \ No newline at end of file From 0e7ee9decc15655f4a597115eff635f38d071e5d Mon Sep 17 00:00:00 2001 From: Aaron Morton Date: Fri, 17 Apr 2026 10:01:40 +1200 Subject: [PATCH 22/89] remove on pull dispatch --- .github/workflows/test-bench-run.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/test-bench-run.yaml b/.github/workflows/test-bench-run.yaml index 2563420ac2..30fb362d59 100644 --- a/.github/workflows/test-bench-run.yaml +++ b/.github/workflows/test-bench-run.yaml @@ -7,7 +7,6 @@ on: description: 'Environment to test against' required: true type: environment - pull_request: # needed when a workflow wants to use OIDC (OpenID Connect) to authenticate to cloud permissions: From eebce0ad9cc5b2f84f7b7860dad2bcbdd333fe89 Mon Sep 17 00:00:00 2001 From: Aaron Morton Date: Fri, 17 Apr 2026 10:04:05 +1200 Subject: [PATCH 23/89] put pull_request back :) --- .github/workflows/test-bench-run.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test-bench-run.yaml b/.github/workflows/test-bench-run.yaml index 30fb362d59..2563420ac2 100644 --- a/.github/workflows/test-bench-run.yaml +++ b/.github/workflows/test-bench-run.yaml @@ -7,6 +7,7 @@ on: description: 'Environment to test against' required: true type: environment + pull_request: # needed when a workflow wants to use OIDC (OpenID Connect) to authenticate to cloud permissions: From c55db387dfc25c2db95b5f115b2a0d73615c3afa Mon Sep 17 00:00:00 2001 From: Aaron Morton Date: Fri, 17 Apr 2026 10:10:36 +1200 Subject: [PATCH 24/89] default env ? --- .github/workflows/test-bench-run.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/test-bench-run.yaml b/.github/workflows/test-bench-run.yaml index 2563420ac2..f5e991250f 100644 --- a/.github/workflows/test-bench-run.yaml +++ b/.github/workflows/test-bench-run.yaml @@ -43,6 +43,8 @@ jobs: matrix: include: ${{ fromJson(needs.setup.outputs.matrix) }} + environment: ${{ github.event.inputs.environment || 'DEV' }} + steps: - uses: actions/checkout@v6 From cd862baaa2680786a21706518dd35a730733851c Mon Sep 17 00:00:00 2001 From: Aaron Morton Date: Fri, 17 Apr 2026 10:17:18 +1200 Subject: [PATCH 25/89] try again --- .github/workflows/test-bench-run.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-bench-run.yaml b/.github/workflows/test-bench-run.yaml index f5e991250f..22815714f1 100644 --- a/.github/workflows/test-bench-run.yaml +++ b/.github/workflows/test-bench-run.yaml @@ -25,6 +25,8 @@ env: jobs: setup: runs-on: ubuntu-latest + environment: ${{ github.event.inputs.environment || 'DEV' }} + outputs: matrix: ${{ steps.set-matrix.outputs.TEST_TARGETS }} steps: @@ -43,8 +45,6 @@ jobs: matrix: include: ${{ fromJson(needs.setup.outputs.matrix) }} - environment: ${{ github.event.inputs.environment || 'DEV' }} - steps: - uses: actions/checkout@v6 From 917ebda54f21f257899e95b3bf697e73817e81c3 Mon Sep 17 00:00:00 2001 From: Eric Hare Date: Thu, 16 Apr 2026 15:19:52 -0700 Subject: [PATCH 26/89] Update test-bench-run.yaml --- .github/workflows/test-bench-run.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test-bench-run.yaml b/.github/workflows/test-bench-run.yaml index 22815714f1..12881c7f57 100644 --- a/.github/workflows/test-bench-run.yaml +++ b/.github/workflows/test-bench-run.yaml @@ -37,6 +37,7 @@ jobs: build: name: Unit tests runs-on: ubuntu-latest + environment: ${{ github.event.inputs.environment || 'DEV' }} # max run time 12 minutes timeout-minutes: 12 From c9b85b5dcd98ebeca3b647efa95a14bac124bd55 Mon Sep 17 00:00:00 2001 From: Eric Hare Date: Thu, 16 Apr 2026 15:27:23 -0700 Subject: [PATCH 27/89] Update test-bench-run.yaml --- .github/workflows/test-bench-run.yaml | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-bench-run.yaml b/.github/workflows/test-bench-run.yaml index 12881c7f57..b5f2c0b935 100644 --- a/.github/workflows/test-bench-run.yaml +++ b/.github/workflows/test-bench-run.yaml @@ -31,7 +31,14 @@ jobs: matrix: ${{ steps.set-matrix.outputs.TEST_TARGETS }} steps: - id: set-matrix - run: echo "TEST_TARGETS=${{ secrets.TEST_TARGETS }}" >> $GITHUB_OUTPUT + env: + TEST_TARGETS: ${{ vars.TEST_TARGETS }} + run: | + { + echo "TEST_TARGETS<> "$GITHUB_OUTPUT" # runs unit tests build: @@ -93,7 +100,7 @@ jobs: - name: Build & Test env: TEST_PLAN_FILE: classpath:integration-tests/test-plans/vectorize-astra-test-plan.yaml - Token: ${{ matrix.token }} + Token: ${{ secrets[matrix.token_secret] }} TARGET_NAME: ${{ matrix.name }} ENDPOINT: ${{ matrix.endpoint }} openAi_KEY: ${{ secrets.OPEN_AI_KEY }} From bb0018a3cf9651e3d4fd0144078f9bdce18f2168 Mon Sep 17 00:00:00 2001 From: Eric Hare Date: Thu, 16 Apr 2026 15:28:58 -0700 Subject: [PATCH 28/89] Update test-bench-run.yaml --- .github/workflows/test-bench-run.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test-bench-run.yaml b/.github/workflows/test-bench-run.yaml index b5f2c0b935..8e8aa0bf54 100644 --- a/.github/workflows/test-bench-run.yaml +++ b/.github/workflows/test-bench-run.yaml @@ -43,6 +43,7 @@ jobs: # runs unit tests build: name: Unit tests + needs: setup runs-on: ubuntu-latest environment: ${{ github.event.inputs.environment || 'DEV' }} From 26c23b274bf8abaae8af53b632ae887e2d5c334b Mon Sep 17 00:00:00 2001 From: Eric Hare Date: Thu, 16 Apr 2026 15:31:25 -0700 Subject: [PATCH 29/89] Fix the rendering of the title --- .github/workflows/test-bench-run.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-bench-run.yaml b/.github/workflows/test-bench-run.yaml index 8e8aa0bf54..ebbc2be465 100644 --- a/.github/workflows/test-bench-run.yaml +++ b/.github/workflows/test-bench-run.yaml @@ -42,7 +42,7 @@ jobs: # runs unit tests build: - name: Unit tests + name: Unit tests (${{ matrix.name }}) needs: setup runs-on: ubuntu-latest environment: ${{ github.event.inputs.environment || 'DEV' }} From 83f7ba77956a6afbf8a474be12fa189e189ae130 Mon Sep 17 00:00:00 2001 From: Aaron Morton Date: Fri, 17 Apr 2026 15:09:28 +1200 Subject: [PATCH 30/89] reduce logging --- .../sgv2/jsonapi/api/v1/vectorize/messaging/APIRequest.java | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/messaging/APIRequest.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/messaging/APIRequest.java index 400ed16dd2..650f407c6e 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/messaging/APIRequest.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/messaging/APIRequest.java @@ -86,11 +86,9 @@ protected Map getHeaders() { public RequestSpecification jsonRequest() { - // .log().uri() - // .log().body() return given() - .log() - .all() + .log().uri() + .log().body() .baseUri(connection.domain()) .port(connection.port()) .basePath(connection.basePath()) From c9b2f72559d88a93fabfb76b65ee288755170243 Mon Sep 17 00:00:00 2001 From: Aaron Morton Date: Fri, 17 Apr 2026 15:30:49 +1200 Subject: [PATCH 31/89] trying --- .github/workflows/test-bench-run.yaml | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/.github/workflows/test-bench-run.yaml b/.github/workflows/test-bench-run.yaml index ebbc2be465..544f236832 100644 --- a/.github/workflows/test-bench-run.yaml +++ b/.github/workflows/test-bench-run.yaml @@ -32,13 +32,9 @@ jobs: steps: - id: set-matrix env: - TEST_TARGETS: ${{ vars.TEST_TARGETS }} + TEST_TARGETS: ${{ secrets.TEST_TARGETS_SECRET }} run: | - { - echo "TEST_TARGETS<> "$GITHUB_OUTPUT" + echo "$TEST_TARGETS" >> "$GITHUB_OUTPUT" # runs unit tests build: @@ -101,7 +97,7 @@ jobs: - name: Build & Test env: TEST_PLAN_FILE: classpath:integration-tests/test-plans/vectorize-astra-test-plan.yaml - Token: ${{ secrets[matrix.token_secret] }} + Token: ${{ matrix.token }} TARGET_NAME: ${{ matrix.name }} ENDPOINT: ${{ matrix.endpoint }} openAi_KEY: ${{ secrets.OPEN_AI_KEY }} From 5171de4848d3bf875956580807bf0a1144f25238 Mon Sep 17 00:00:00 2001 From: Aaron Morton Date: Fri, 17 Apr 2026 15:34:19 +1200 Subject: [PATCH 32/89] testing --- .github/workflows/test-bench-run.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-bench-run.yaml b/.github/workflows/test-bench-run.yaml index 544f236832..519bd20d00 100644 --- a/.github/workflows/test-bench-run.yaml +++ b/.github/workflows/test-bench-run.yaml @@ -34,7 +34,7 @@ jobs: env: TEST_TARGETS: ${{ secrets.TEST_TARGETS_SECRET }} run: | - echo "$TEST_TARGETS" >> "$GITHUB_OUTPUT" + echo "TEST_TARGETS=$TEST_TARGETS" >> "$GITHUB_OUTPUT" # runs unit tests build: From 81177fbb963d2dcae34e3ab1667740c903d60340 Mon Sep 17 00:00:00 2001 From: Aaron Morton Date: Fri, 17 Apr 2026 16:11:00 +1200 Subject: [PATCH 33/89] simpler --- .github/workflows/test-bench-run.yaml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test-bench-run.yaml b/.github/workflows/test-bench-run.yaml index 519bd20d00..8f310ae97d 100644 --- a/.github/workflows/test-bench-run.yaml +++ b/.github/workflows/test-bench-run.yaml @@ -32,9 +32,8 @@ jobs: steps: - id: set-matrix env: - TEST_TARGETS: ${{ secrets.TEST_TARGETS_SECRET }} - run: | - echo "TEST_TARGETS=$TEST_TARGETS" >> "$GITHUB_OUTPUT" + TEST_TARGETS: ${{ vars.TEST_TARGETS }} + run: echo "$TEST_TARGETS" >> "$GITHUB_OUTPUT" # runs unit tests build: @@ -97,7 +96,7 @@ jobs: - name: Build & Test env: TEST_PLAN_FILE: classpath:integration-tests/test-plans/vectorize-astra-test-plan.yaml - Token: ${{ matrix.token }} + Token: ${{ secrets.API_TOKEN }} TARGET_NAME: ${{ matrix.name }} ENDPOINT: ${{ matrix.endpoint }} openAi_KEY: ${{ secrets.OPEN_AI_KEY }} From 3d0378e6544252d121eb8f639fe4f34c7ff5288b Mon Sep 17 00:00:00 2001 From: Aaron Morton Date: Fri, 17 Apr 2026 16:13:25 +1200 Subject: [PATCH 34/89] test --- .github/workflows/test-bench-run.yaml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-bench-run.yaml b/.github/workflows/test-bench-run.yaml index 8f310ae97d..ebbc2be465 100644 --- a/.github/workflows/test-bench-run.yaml +++ b/.github/workflows/test-bench-run.yaml @@ -33,7 +33,12 @@ jobs: - id: set-matrix env: TEST_TARGETS: ${{ vars.TEST_TARGETS }} - run: echo "$TEST_TARGETS" >> "$GITHUB_OUTPUT" + run: | + { + echo "TEST_TARGETS<> "$GITHUB_OUTPUT" # runs unit tests build: @@ -96,7 +101,7 @@ jobs: - name: Build & Test env: TEST_PLAN_FILE: classpath:integration-tests/test-plans/vectorize-astra-test-plan.yaml - Token: ${{ secrets.API_TOKEN }} + Token: ${{ secrets[matrix.token_secret] }} TARGET_NAME: ${{ matrix.name }} ENDPOINT: ${{ matrix.endpoint }} openAi_KEY: ${{ secrets.OPEN_AI_KEY }} From b72f7dfe95de6f1e3eb57a033e417181cd524a6e Mon Sep 17 00:00:00 2001 From: Aaron Morton Date: Fri, 17 Apr 2026 16:15:18 +1200 Subject: [PATCH 35/89] test --- .github/workflows/test-bench-run.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-bench-run.yaml b/.github/workflows/test-bench-run.yaml index ebbc2be465..5cfd790d3d 100644 --- a/.github/workflows/test-bench-run.yaml +++ b/.github/workflows/test-bench-run.yaml @@ -42,7 +42,7 @@ jobs: # runs unit tests build: - name: Unit tests (${{ matrix.name }}) + name: Test Bench- ${{ matrix.name }} needs: setup runs-on: ubuntu-latest environment: ${{ github.event.inputs.environment || 'DEV' }} @@ -101,7 +101,7 @@ jobs: - name: Build & Test env: TEST_PLAN_FILE: classpath:integration-tests/test-plans/vectorize-astra-test-plan.yaml - Token: ${{ secrets[matrix.token_secret] }} + Token: ${{ secrets.API_TOKEN }} TARGET_NAME: ${{ matrix.name }} ENDPOINT: ${{ matrix.endpoint }} openAi_KEY: ${{ secrets.OPEN_AI_KEY }} From 6062cc4a5371415b9a713e4559250e2cb1da02bd Mon Sep 17 00:00:00 2001 From: Aaron Morton Date: Fri, 17 Apr 2026 16:59:37 +1200 Subject: [PATCH 36/89] fake change --- .github/workflows/test-bench-run.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-bench-run.yaml b/.github/workflows/test-bench-run.yaml index 5cfd790d3d..81c3d0c744 100644 --- a/.github/workflows/test-bench-run.yaml +++ b/.github/workflows/test-bench-run.yaml @@ -42,7 +42,7 @@ jobs: # runs unit tests build: - name: Test Bench- ${{ matrix.name }} + name: Test Bench> ${{ matrix.name }} needs: setup runs-on: ubuntu-latest environment: ${{ github.event.inputs.environment || 'DEV' }} From 2ad4970af72da0c7afa2cb79d4e0195ea72229f1 Mon Sep 17 00:00:00 2001 From: Aaron Morton Date: Fri, 17 Apr 2026 17:20:49 +1200 Subject: [PATCH 37/89] put env in matrix --- .github/workflows/test-bench-run.yaml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-bench-run.yaml b/.github/workflows/test-bench-run.yaml index 81c3d0c744..15204da3a9 100644 --- a/.github/workflows/test-bench-run.yaml +++ b/.github/workflows/test-bench-run.yaml @@ -25,7 +25,11 @@ env: jobs: setup: runs-on: ubuntu-latest - environment: ${{ github.event.inputs.environment || 'DEV' }} + + strategy: + matrix: + environment: [ DEV, TEST ] + environment: ${{ matrix.environment }} outputs: matrix: ${{ steps.set-matrix.outputs.TEST_TARGETS }} @@ -42,7 +46,7 @@ jobs: # runs unit tests build: - name: Test Bench> ${{ matrix.name }} + name: Test Bench- ${{ matrix.environment }} - ${{ matrix.name }} needs: setup runs-on: ubuntu-latest environment: ${{ github.event.inputs.environment || 'DEV' }} From ad9a50831d64a81833c9603ea21d1b37f652bf5d Mon Sep 17 00:00:00 2001 From: Aaron Morton Date: Fri, 17 Apr 2026 17:23:52 +1200 Subject: [PATCH 38/89] fix --- .github/workflows/test-bench-run.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-bench-run.yaml b/.github/workflows/test-bench-run.yaml index 15204da3a9..2dd51e9079 100644 --- a/.github/workflows/test-bench-run.yaml +++ b/.github/workflows/test-bench-run.yaml @@ -49,7 +49,7 @@ jobs: name: Test Bench- ${{ matrix.environment }} - ${{ matrix.name }} needs: setup runs-on: ubuntu-latest - environment: ${{ github.event.inputs.environment || 'DEV' }} + environment: ${{ matrix.environment }} # max run time 12 minutes timeout-minutes: 12 From 21030b1da5838bf468cdc6dfd43896974c99383e Mon Sep 17 00:00:00 2001 From: Aaron Morton Date: Fri, 17 Apr 2026 20:47:33 +1200 Subject: [PATCH 39/89] remove TEST env, not working --- .github/workflows/test-bench-run.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-bench-run.yaml b/.github/workflows/test-bench-run.yaml index 2dd51e9079..0cc0b487bd 100644 --- a/.github/workflows/test-bench-run.yaml +++ b/.github/workflows/test-bench-run.yaml @@ -28,7 +28,7 @@ jobs: strategy: matrix: - environment: [ DEV, TEST ] + environment: [ DEV ] environment: ${{ matrix.environment }} outputs: From b668362570f43e21debe8feffaa98f31de6e5a4f Mon Sep 17 00:00:00 2001 From: Eric Hare Date: Fri, 17 Apr 2026 13:23:06 -0700 Subject: [PATCH 40/89] Roll back to preexisting env usage --- .github/workflows/test-bench-run.yaml | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/.github/workflows/test-bench-run.yaml b/.github/workflows/test-bench-run.yaml index 0cc0b487bd..f397f56a43 100644 --- a/.github/workflows/test-bench-run.yaml +++ b/.github/workflows/test-bench-run.yaml @@ -25,11 +25,7 @@ env: jobs: setup: runs-on: ubuntu-latest - - strategy: - matrix: - environment: [ DEV ] - environment: ${{ matrix.environment }} + environment: ${{ github.event.inputs.environment || 'DEV' }} outputs: matrix: ${{ steps.set-matrix.outputs.TEST_TARGETS }} @@ -46,10 +42,10 @@ jobs: # runs unit tests build: - name: Test Bench- ${{ matrix.environment }} - ${{ matrix.name }} + name: Test Bench - ${{ matrix.name }} needs: setup runs-on: ubuntu-latest - environment: ${{ matrix.environment }} + environment: ${{ github.event.inputs.environment || 'DEV' }} # max run time 12 minutes timeout-minutes: 12 From 6b30e0c9ab7e6ec82d8097f153975c4b4f569919 Mon Sep 17 00:00:00 2001 From: Aaron Morton Date: Tue, 21 Apr 2026 14:02:34 +1200 Subject: [PATCH 41/89] fixing env and reporting --- .github/workflows/test-bench-run.yaml | 2 +- .../v1/vectorize/TestBenchConsoleWriter.java | 150 -------------- .../api/v1/vectorize/assertions/Http.java | 2 +- .../assertions/SingleTestAssertion.java | 4 +- .../v1/vectorize/messaging/APIRequest.java | 6 +- .../{ => reporting}/DynamicTreeListener.java | 42 ++-- .../reporting/TestBenchConsoleWriter.java | 187 ++++++++++++++++++ .../testrun/DynamicTestExecutable.java | 10 +- ...it.platform.launcher.TestExecutionListener | 2 +- .../vectorize/vectorize-header-workflow.json | 3 +- .../vectorize/vectorize-shared-workflow.json | 2 +- 11 files changed, 230 insertions(+), 180 deletions(-) delete mode 100644 src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestBenchConsoleWriter.java rename src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/{ => reporting}/DynamicTreeListener.java (87%) create mode 100644 src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/reporting/TestBenchConsoleWriter.java diff --git a/.github/workflows/test-bench-run.yaml b/.github/workflows/test-bench-run.yaml index f397f56a43..6c1d31b5d8 100644 --- a/.github/workflows/test-bench-run.yaml +++ b/.github/workflows/test-bench-run.yaml @@ -42,7 +42,7 @@ jobs: # runs unit tests build: - name: Test Bench - ${{ matrix.name }} + name: Test Bench- ENV-${{ github.event.inputs.environment || 'DEV' }} Target- ${{ matrix.name }} needs: setup runs-on: ubuntu-latest environment: ${{ github.event.inputs.environment || 'DEV' }} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestBenchConsoleWriter.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestBenchConsoleWriter.java deleted file mode 100644 index 6b87fdee13..0000000000 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestBenchConsoleWriter.java +++ /dev/null @@ -1,150 +0,0 @@ -package io.stargate.sgv2.jsonapi.api.v1.vectorize; - -import static org.apache.maven.surefire.shared.utils.logging.MessageUtils.buffer; - -import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestUri; -import org.apache.maven.plugin.surefire.report.Theme; -import org.apache.maven.surefire.shared.utils.logging.MessageBuilder; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class TestBenchConsoleWriter { - private static final Logger LOGGER = LoggerFactory.getLogger(TestBenchConsoleWriter.class); - - private boolean firstLine = true; - - private final Theme theme; - - public TestBenchConsoleWriter() { - this.theme = Theme.EMOJI; - } - - /** - * We print the test has starting , so the output from the test comes after, we then output the - * summary later. - * - * @param tracker - */ - public void printTestStarted(DynamicTreeListener.TestTracker tracker) { - - var buffer = buffer(); - if (firstLine) { - - firstLine = false; - buffer - .newline() - .a(theme.dash().repeat(20)) - .newline() - .a("Running Test Bench, summary shown at completion...") - .newline() - .a(theme.dash().repeat(20)) - .newline(); - } - - // [INFO] ── io.stargate.sgv2.jsonapi.api.model.command.impl.InsertManyCommandTest - - // 1.316 s - // [INFO] ├─ ✔ noDocuments - 0.274 s - - // if there is no parent - if (tracker.parent() == null) { - buffer.a(theme.dash()).strong(tracker.identifier().getDisplayName()); - } else { - buffer - .a(theme.blank().repeat(tracker.depth() - 1)) - .a(theme.entry()) - .strong(tracker.identifier().getDisplayName()); - } - LOGGER.info(buffer.toString()); - } - - public void printTestPlanCompleted(DynamicTreeListener.TestTracker rootTracker) { - - var buffer = buffer(); - - buffer - .newline() - .a(theme.dash().repeat(20)) - .newline() - .a("Test Bench Summary") - .newline() - .a(theme.dash().repeat(20)) - .newline(); - - // [INFO] ── io.stargate.sgv2.jsonapi.api.model.command.impl.InsertManyCommandTest - - // 1.316 s - // [INFO] ├─ ✔ noDocuments - 0.274 s - - // if there is no parent - // if (tracker.parent() == null){ - // buffer.a(theme.dash()) - // .strong(tracker.identifier().getDisplayName()); - // } - // else { - // buffer.a(theme.blank().repeat(tracker.depth() -1)) - // .a(theme.entry()) - // .strong(tracker.identifier().getDisplayName()); - // } - writeCompletedSummary(buffer, rootTracker, true); - LOGGER.info(buffer.toString()); - } - - /** - * TestPlan: smoketest-aws-us-east-1 on astra workflows vectorize-header-workflow Workflow: - * vectorize-header-workflow Job: nvidia-vectorize TestSuite: vectorize-header-auth TestEnv: - * [MODEL=NV-Embed-QA, PROVIDER=nvidia] RESULTS.... TestEnv: [MODEL=nvidia/nv-embedqa-e5-v5, - * PROVIDER=nvidia] RESULTS... - * - * @param buffer - * @param tracker - * @param isRoot - */ - private void writeCompletedSummary( - MessageBuilder buffer, DynamicTreeListener.TestTracker tracker, boolean isRoot) { - - // the tree part - if (isRoot) { - buffer.a(theme.dash()); - } else { - buffer.a(theme.blank().repeat(tracker.depth() - 1)).a(theme.entry()); - } - - var timing = - tracker.stats() == null ? "" : " - %s s".formatted(tracker.stats().elapsedMillis() / 1000); - // name of the group - if (tracker.stats().noErrors()) { - buffer.a(theme.successful()); - } else { - buffer.a(theme.failed()); - } - - buffer.strong(tracker.identifier().getDisplayName()).a(timing).newline(); - - // If we have a TestEnv then we want to write out the summary of results for it, otherwise - // descend until we get one - if (tracker.runUri().leafType() == TestUri.Segment.ENV) { - - buffer - .a(theme.blank().repeat(tracker.depth())) - .a(theme.details()) - .a("Successful: " + tracker.stats().successful()) - .newline(); - buffer - .a(theme.blank().repeat(tracker.depth())) - .a(theme.details()) - .a("Failures: " + tracker.stats().failures()) - .newline(); - buffer - .a(theme.blank().repeat(tracker.depth())) - .a(theme.details()) - .a("Aborted: " + tracker.stats().aborted()) - .newline(); - buffer - .a(theme.blank().repeat(tracker.depth())) - .a(theme.details()) - .a("Skipped: " + tracker.stats().skipped()) - .newline(); - } else { - tracker.children().forEach(child -> writeCompletedSummary(buffer, child, false)); - } - } -} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Http.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Http.java index c2efa27e09..943b5e95f6 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Http.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Http.java @@ -14,7 +14,7 @@ public class Http { public static AssertionMatcher success(TestCommand testCommand, JsonNode args) { return described( - "http status is groovy", + "http status is " + HttpStatus.SC_OK, apiResponse -> apiResponse.validatable().statusCode(HttpStatus.SC_OK)); } } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/SingleTestAssertion.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/SingleTestAssertion.java index 1704100cfb..f9d4a7a705 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/SingleTestAssertion.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/SingleTestAssertion.java @@ -15,10 +15,10 @@ public void run(TestRunResponse testResponse) { try { matcher.match(testResponse.apiResponse()); } catch (AssertionError e) { - System.out.printf("Failed Assertion: name=%s, args=%s", name, args); +// System.out.printf("Failed Assertion: name=%s, args=%s", name, args); throw e; } catch (Exception e) { - System.out.printf("Error In Assertion: name=%s, args=%s", name, args); +// System.out.printf("Error In Assertion: name=%s, args=%s", name, args); throw e; } } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/messaging/APIRequest.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/messaging/APIRequest.java index 650f407c6e..bac584a43e 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/messaging/APIRequest.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/messaging/APIRequest.java @@ -87,8 +87,10 @@ protected Map getHeaders() { public RequestSpecification jsonRequest() { return given() - .log().uri() - .log().body() + .log() + .uri() + .log() + .body() .baseUri(connection.domain()) .port(connection.port()) .basePath(connection.basePath()) diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/DynamicTreeListener.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/reporting/DynamicTreeListener.java similarity index 87% rename from src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/DynamicTreeListener.java rename to src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/reporting/DynamicTreeListener.java index ce3a192183..62f0694679 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/DynamicTreeListener.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/reporting/DynamicTreeListener.java @@ -1,4 +1,4 @@ -package io.stargate.sgv2.jsonapi.api.v1.vectorize; +package io.stargate.sgv2.jsonapi.api.v1.vectorize.reporting; import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestUri; import java.util.*; @@ -13,12 +13,14 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import javax.annotation.Nullable; + public class DynamicTreeListener implements TestExecutionListener { private static final Logger LOGGER = LoggerFactory.getLogger(DynamicTreeListener.class); - private TestTracker rootTracker; - private final Map testTrackers = new ConcurrentHashMap<>(); + private TestReportingTracker rootTracker; + private final Map testTrackers = new ConcurrentHashMap<>(); private final Map startTimes = new ConcurrentHashMap<>(); private final Map containerStats = new ConcurrentHashMap<>(); @@ -30,7 +32,7 @@ public void testPlanExecutionStarted(TestPlan testPlan) {} @Override public void testPlanExecutionFinished(TestPlan testPlan) { - writer.printTestPlanCompleted(rootTracker); + writer.allTestsFinished(rootTracker); } @Override @@ -41,14 +43,11 @@ public void executionStarted(TestIdentifier id) { if (!isTestBenchNode(id)) { return; } - // System.out.println("XXX " + id.getDisplayName()); var tracker = getCreateTestTracker(id); if (tracker == null) { return; } - // System.out.println("YYY " + tracker.runUri); - // System.out.println("ZZZ " + tracker.runUri.leafType()); - writer.printTestStarted(tracker); + writer.testStarted(tracker); } @Override @@ -139,7 +138,7 @@ private static boolean isTestBenchNode(TestIdentifier testIdentifier) { // return Optional.empty(); // } - private TestTracker getCreateTestTracker(TestIdentifier testIdentifier) { + private TestReportingTracker getCreateTestTracker(TestIdentifier testIdentifier) { var existingTracker = testTrackers.get(testIdentifier.getUniqueIdObject()); if (existingTracker != null) { @@ -169,7 +168,7 @@ private TestTracker getCreateTestTracker(TestIdentifier testIdentifier) { "parentID not found for testIdentifier: " + testIdentifier.toString()) : null; - var tracker = new TestTracker(testIdentifier, testUri.get(), parentTracker); + var tracker = new TestReportingTracker(testIdentifier, testUri.get(), parentTracker); if (rootTracker == null) { rootTracker = tracker; @@ -179,19 +178,21 @@ private TestTracker getCreateTestTracker(TestIdentifier testIdentifier) { } /** */ - public class TestTracker { + public class TestReportingTracker { private final TestIdentifier identifier; private final TestUri runUri; - private final TestTracker parent; + private final TestReportingTracker parent; private final int depth; - private final List children = new ArrayList<>(); + private final List children = new ArrayList<>(); + private Optional throwable = Optional.empty(); private TestExecutionResult.Status junitStatus; private final TestContainerStats stats; - public TestTracker(TestIdentifier identifier, TestUri runUri, TestTracker parent) { + public TestReportingTracker( + TestIdentifier identifier, TestUri runUri, TestReportingTracker parent) { this.identifier = identifier; this.runUri = runUri; this.parent = parent; @@ -203,6 +204,14 @@ public TestTracker(TestIdentifier identifier, TestUri runUri, TestTracker parent } } + public Optional throwable() { + return throwable; + } + + @Nullable + public TestExecutionResult.Status junitStatus(){ + return junitStatus; + } public TestIdentifier identifier() { return identifier; } @@ -211,11 +220,11 @@ public TestUri runUri() { return runUri; } - public TestTracker parent() { + public TestReportingTracker parent() { return parent; } - public List children() { + public List children() { return children; } @@ -229,6 +238,7 @@ public TestContainerStats stats() { public void executionFinished(TestExecutionResult result) { junitStatus = result.getStatus(); + throwable = result.getThrowable(); if (stats != null) { stats.testCompleted(result); } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/reporting/TestBenchConsoleWriter.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/reporting/TestBenchConsoleWriter.java new file mode 100644 index 0000000000..2da782f772 --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/reporting/TestBenchConsoleWriter.java @@ -0,0 +1,187 @@ +package io.stargate.sgv2.jsonapi.api.v1.vectorize.reporting; + +import static org.apache.maven.surefire.shared.utils.logging.MessageUtils.buffer; + +import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestUri; +import java.util.Objects; +import org.apache.maven.plugin.surefire.report.Theme; +import org.apache.maven.surefire.shared.utils.logging.MessageBuilder; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Writes messages for the output of a Test Bench runs using a LOGGER. + * + *

Works in two modes, because we need to wait for all the tests to complete so we know what was + * successful. For info on how the output will look like see {@link Theme} and {@link + * org.apache.maven.surefire.shared.utils.logging.AnsiMessageBuilder} + * + *

Call {@link #testStarted(DynamicTreeListener.TestReportingTracker)} when a test is being + * executed and it has started, will print the progress of the tests. There is no correlating test + * finished. + * + *

Call {@link #allTestsFinished(DynamicTreeListener.TestReportingTracker)} with the root + * tracker for the rest run, this should be called once all the tests have completed so we know what + * failed and how long things took. This will print the full test tree, but only go down to the + * request + assertion level for test scenarios that have failed. + */ +public class TestBenchConsoleWriter { + + private static final Logger LOGGER = LoggerFactory.getLogger(TestBenchConsoleWriter.class); + + private boolean firstLine = true; + + private final Theme theme; + + public TestBenchConsoleWriter() { + this(Theme.UNICODE); + } + + public TestBenchConsoleWriter(Theme theme) { + this.theme = Objects.requireNonNull(theme, "theme must not be null"); + } + + /** + * Writes that a test has started, without knowing how it will end, used when the test suite is + * running. + * + * @param tracker Tracker for the test that is running. + */ + public void testStarted(DynamicTreeListener.TestReportingTracker tracker) { + + var buffer = buffer(); + if (firstLine) { + firstLine = false; + buffer + .newline() + .a(theme.dash().repeat(20)) + .newline() + .a("Running Test Bench, results shown at completion...") + .newline() + .a(theme.dash().repeat(20)) + .newline(); + } + + // if there is no parent, this is the first test that is running. + // display name is the name the TestBench put on the container + if (tracker.parent() == null) { + buffer.a(theme.down()).strong(tracker.identifier().getDisplayName()); + } else { + buffer + .a(theme.blank().repeat(tracker.depth() - 1)) + .a(theme.entry()) + .strong(tracker.identifier().getDisplayName()); + } + LOGGER.info(buffer.toString()); + } + + /** + * Call when all the tests have finished running, so we can print a summary for test suites that pass, and + * details for those that fail. + * @param rootTracker + */ + public void allTestsFinished(DynamicTreeListener.TestReportingTracker rootTracker) { + + var buffer = buffer(); + + buffer + .newline() + .a(theme.dash().repeat(20)) + .newline() + .a("Test Bench Results") + .newline() + .a(theme.dash().repeat(20)) + .newline(); + + writeCompletedSummary(buffer, rootTracker, true); + LOGGER.info(buffer.toString()); + } + + /** + * TestPlan: smoketest-aws-us-east-1 on astra workflows vectorize-header-workflow Workflow: + * vectorize-header-workflow Job: nvidia-vectorize TestSuite: vectorize-header-auth TestEnv: + * [MODEL=NV-Embed-QA, PROVIDER=nvidia] RESULTS.... TestEnv: [MODEL=nvidia/nv-embedqa-e5-v5, + * PROVIDER=nvidia] RESULTS... + * + * @param buffer + * @param tracker + * @param isRoot + */ + private void writeCompletedSummary( + MessageBuilder buffer, + DynamicTreeListener.TestReportingTracker tracker, + boolean isRoot) { + + // the tree part of the line + if (isRoot) { + buffer.a(theme.down()); + } else { + buffer.a(theme.blank().repeat(tracker.depth() - 1)).a(theme.entry()); + } + + var timingInfo = tracker.stats() == null ? + "" + : + " - %s s".formatted(tracker.stats().elapsedMillis() / 1000); + + // Icon for success for failure, + // if we have stats then this is a container we we use that status, otherwise we use the JUNIT execution status + if (tracker.stats() == null){ + switch (tracker.junitStatus()) { + case SUCCESSFUL -> buffer.a(theme.successful()); + case FAILED, ABORTED -> buffer.a(theme.failed()); + case null -> {} + } + } else { + if (tracker.stats().noErrors()) { + buffer.a(theme.successful()); + } else { + buffer.a(theme.failed()); + } + } + +// if (tracker.stats() == null) { +// buffer.a(theme.blank()); +// } else if (tracker.stats().noErrors()) { +// buffer.a(theme.successful()); +// } else { +// buffer.a(theme.failed()); +// } + + + // The name of the container or test + buffer.strong(tracker.identifier().getDisplayName()).a(timingInfo).newline(); + + // if we have stats write a stats line, these are aggregate for all things below. + // alternative is to only print them for Test Environment these are lines like + // TestEnv: [MODEL=text-embedding-3-small, PROVIDER=openai] + // to do that do this test: (tracker.runUri().leafType() == TestUri.Segment.ENV) + + if (tracker.stats() != null) { + writeStatsLine(buffer, tracker); + } + + // we do not want to descent below the Test Envirinment unless there is a failure, otherwise there is too + // much output. Tend ENV line looks like + // TestEnv: [MODEL=text-embedding-3-small, PROVIDER=openai] - 26 s + + var noErrors = tracker.stats() != null && tracker.stats().noErrors(); + + // If we have a TestEnv then we want to write out the summary of results for it, otherwise + // descend until we get one + if (!noErrors || tracker.runUri().leafType() != TestUri.Segment.ENV) { + tracker.children().forEach(child -> writeCompletedSummary(buffer, child, false)); + } + } + + private void writeStatsLine(MessageBuilder buffer, DynamicTreeListener.TestReportingTracker tracker) { + buffer + .a(theme.blank().repeat(tracker.depth())) + .a(theme.entry()) + .a("Successful: ").a( tracker.stats().successful()).a(", ") + .a("Failures: ").a( tracker.stats().failures()).a(", ") + .a("Aborted: ").a( tracker.stats().aborted()).a(", ") + .a("Skipped: ").a( tracker.stats().skipped()) + .newline(); + } +} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testrun/DynamicTestExecutable.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testrun/DynamicTestExecutable.java index 9e688bd8c4..2e34a1114b 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testrun/DynamicTestExecutable.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testrun/DynamicTestExecutable.java @@ -25,8 +25,8 @@ public DynamicTestExecutable(String description, TestUri testUri, Executable exe this.executable = executable; var truncated = - (description != null && description.length() > 60) - ? description.substring(0, 57) + "..." + (description != null && description.length() > 80) + ? description.substring(0, 77) + "..." : description; this.trimmedDisplayName = truncated; @@ -49,9 +49,9 @@ public void execute() throws Throwable { } private void beforeExecute() { - if (isTrimmed) { - System.out.printf(description + "\n"); - } + // if (isTrimmed) { + // System.out.printf(description + "\n"); + // } } private void afterExecute() { diff --git a/src/test/resources/META-INF/services/org.junit.platform.launcher.TestExecutionListener b/src/test/resources/META-INF/services/org.junit.platform.launcher.TestExecutionListener index 1dbce58eb5..2017f464d7 100644 --- a/src/test/resources/META-INF/services/org.junit.platform.launcher.TestExecutionListener +++ b/src/test/resources/META-INF/services/org.junit.platform.launcher.TestExecutionListener @@ -1 +1 @@ -io.stargate.sgv2.jsonapi.api.v1.vectorize.DynamicTreeListener \ No newline at end of file +io.stargate.sgv2.jsonapi.api.v1.vectorize.reporting.DynamicTreeListener \ No newline at end of file diff --git a/src/test/resources/integration-tests/vectorize/vectorize-header-workflow.json b/src/test/resources/integration-tests/vectorize/vectorize-header-workflow.json index 314c8a7e4c..c3ccd20573 100644 --- a/src/test/resources/integration-tests/vectorize/vectorize-header-workflow.json +++ b/src/test/resources/integration-tests/vectorize/vectorize-header-workflow.json @@ -8,6 +8,7 @@ "meta": { "name": "open-ai-vectorize", "tags": [ + "disabled" ] }, "fromEnvironment": { @@ -177,7 +178,7 @@ "matrix": { "MODEL": [ "NV-Embed-QA", - "nvidia/nv-embedqa-e5-v5-XX" + "nvidia/nv-embedqa-e5-v5" ] }, "tests": [ diff --git a/src/test/resources/integration-tests/vectorize/vectorize-shared-workflow.json b/src/test/resources/integration-tests/vectorize/vectorize-shared-workflow.json index 54a3444226..40f38a52d0 100644 --- a/src/test/resources/integration-tests/vectorize/vectorize-shared-workflow.json +++ b/src/test/resources/integration-tests/vectorize/vectorize-shared-workflow.json @@ -7,7 +7,7 @@ { "meta": { "name": "open-ai-vectorize", - "tags": [] + "tags": ["disabled"] }, "fromEnvironment": { "x-embedding-api-key": "openAi_KEY", From 0241dfae0d5c15d22a63aa0c0d48c487c3d6ac6f Mon Sep 17 00:00:00 2001 From: Aaron Morton Date: Tue, 21 Apr 2026 14:28:02 +1200 Subject: [PATCH 42/89] add failing test for checking output --- pom.xml | 1 + src/test/resources/application.yaml | 2 +- .../integration-tests/vectorize/vectorize-header-workflow.json | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index bd037882c4..d5b32a45c6 100644 --- a/pom.xml +++ b/pom.xml @@ -483,6 +483,7 @@ verify + ${argLine} diff --git a/src/test/resources/application.yaml b/src/test/resources/application.yaml index 074c9b1c2d..44839370f2 100644 --- a/src/test/resources/application.yaml +++ b/src/test/resources/application.yaml @@ -26,7 +26,7 @@ quarkus: # Fine for JVM mode; if building native images, use index-dependency instead 'io.quarkus.deployment.steps.ReflectiveHierarchyStep': level: ERROR - 'io.stargate.sgv2.jsonapi.api.v1.vectorize.TestBenchConsoleWriter': + 'io.stargate.sgv2.jsonapi.api.v1.vectorize.reporting.TestBenchConsoleWriter': level: INFO handlers: - PLAIN_CONSOLE diff --git a/src/test/resources/integration-tests/vectorize/vectorize-header-workflow.json b/src/test/resources/integration-tests/vectorize/vectorize-header-workflow.json index c3ccd20573..8f73e40445 100644 --- a/src/test/resources/integration-tests/vectorize/vectorize-header-workflow.json +++ b/src/test/resources/integration-tests/vectorize/vectorize-header-workflow.json @@ -178,7 +178,7 @@ "matrix": { "MODEL": [ "NV-Embed-QA", - "nvidia/nv-embedqa-e5-v5" + "nvidia/nv-embedqa-e5-v5-XXX" ] }, "tests": [ From 904d61fb5d8696afc151977455535b43205f8468 Mon Sep 17 00:00:00 2001 From: Aaron Morton Date: Tue, 21 Apr 2026 14:44:49 +1200 Subject: [PATCH 43/89] fix reporting of failure, i think --- pom.xml | 2 +- .../api/v1/vectorize/testrun/DynamicTestExecutable.java | 4 ++-- .../vectorize/vectorize-header-workflow.json | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pom.xml b/pom.xml index d5b32a45c6..3e9631adab 100644 --- a/pom.xml +++ b/pom.xml @@ -58,7 +58,7 @@ (specifically, 3.5.3; latest being 3.5.5) See: https://github.com/fabriciorby/maven-surefire-junit5-tree-reporter?tab=readme-ov-file#important --> - 3.5.3 + 3.5.5 false diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testrun/DynamicTestExecutable.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testrun/DynamicTestExecutable.java index 2e34a1114b..a6ac29ec0c 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testrun/DynamicTestExecutable.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testrun/DynamicTestExecutable.java @@ -25,8 +25,8 @@ public DynamicTestExecutable(String description, TestUri testUri, Executable exe this.executable = executable; var truncated = - (description != null && description.length() > 80) - ? description.substring(0, 77) + "..." + (description != null && description.length() > 120) + ? description.substring(0, 117) + "..." : description; this.trimmedDisplayName = truncated; diff --git a/src/test/resources/integration-tests/vectorize/vectorize-header-workflow.json b/src/test/resources/integration-tests/vectorize/vectorize-header-workflow.json index 8f73e40445..c3ccd20573 100644 --- a/src/test/resources/integration-tests/vectorize/vectorize-header-workflow.json +++ b/src/test/resources/integration-tests/vectorize/vectorize-header-workflow.json @@ -178,7 +178,7 @@ "matrix": { "MODEL": [ "NV-Embed-QA", - "nvidia/nv-embedqa-e5-v5-XXX" + "nvidia/nv-embedqa-e5-v5" ] }, "tests": [ From 09cc9adfe6795ab9ee74556fb329ec8b973ea08c Mon Sep 17 00:00:00 2001 From: Aaron Morton Date: Tue, 21 Apr 2026 14:49:36 +1200 Subject: [PATCH 44/89] failing test --- .../integration-tests/vectorize/vectorize-header-workflow.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/resources/integration-tests/vectorize/vectorize-header-workflow.json b/src/test/resources/integration-tests/vectorize/vectorize-header-workflow.json index c3ccd20573..3ed6e77ffb 100644 --- a/src/test/resources/integration-tests/vectorize/vectorize-header-workflow.json +++ b/src/test/resources/integration-tests/vectorize/vectorize-header-workflow.json @@ -178,7 +178,7 @@ "matrix": { "MODEL": [ "NV-Embed-QA", - "nvidia/nv-embedqa-e5-v5" + "nvidia/nv-embedqa-e5-v5-xxx" ] }, "tests": [ From 9f7f9ed65f4e69f8a738dcc728f6f2a958df4f94 Mon Sep 17 00:00:00 2001 From: Aaron Morton Date: Tue, 21 Apr 2026 15:02:54 +1200 Subject: [PATCH 45/89] do not fail fast --- .github/workflows/test-bench-run.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/test-bench-run.yaml b/.github/workflows/test-bench-run.yaml index 6c1d31b5d8..00edfa015d 100644 --- a/.github/workflows/test-bench-run.yaml +++ b/.github/workflows/test-bench-run.yaml @@ -51,6 +51,8 @@ jobs: timeout-minutes: 12 strategy: + # do not fail fast, we want to run on all the different target dbs + fail-fast: false matrix: include: ${{ fromJson(needs.setup.outputs.matrix) }} From 914933fbe0f6a7f0f64dd64143d19295a44f91c2 Mon Sep 17 00:00:00 2001 From: Aaron Morton Date: Tue, 21 Apr 2026 15:57:39 +1200 Subject: [PATCH 46/89] improve names, enable all tests --- ...nch-run.yaml => test-bench-vectorize.yaml} | 14 +++++-- .../test-plans/vectorize-astra-test-plan.yaml | 2 +- .../vectorize/vectorize-header-workflow.json | 42 +++++++++---------- .../vectorize/vectorize-shared-workflow.json | 8 ++-- 4 files changed, 36 insertions(+), 30 deletions(-) rename .github/workflows/{test-bench-run.yaml => test-bench-vectorize.yaml} (88%) diff --git a/.github/workflows/test-bench-run.yaml b/.github/workflows/test-bench-vectorize.yaml similarity index 88% rename from .github/workflows/test-bench-run.yaml rename to .github/workflows/test-bench-vectorize.yaml index 00edfa015d..0f9a322cb5 100644 --- a/.github/workflows/test-bench-run.yaml +++ b/.github/workflows/test-bench-vectorize.yaml @@ -1,4 +1,4 @@ -name: API Test Bench Run +name: Test Bench - Vectorize Run on: workflow_dispatch: @@ -102,11 +102,17 @@ jobs: EOF - name: Build & Test env: - TEST_PLAN_FILE: classpath:integration-tests/test-plans/vectorize-astra-test-plan.yaml - Token: ${{ secrets.API_TOKEN }} + TEST_PLAN_FILE: classpath:integration-tests/test-plans/test-bench-vectorize.yaml + Token: ${{ secrets.DATA_API_TOKEN }} TARGET_NAME: ${{ matrix.name }} ENDPOINT: ${{ matrix.endpoint }} - openAi_KEY: ${{ secrets.OPEN_AI_KEY }} + HUGGINGFACE_KEY: ${{ secrets.HUGGINGFACE_KEY }} + JINA_AI_KEY: ${{ secrets.JINA_AI_KEY }} + MISTRAL_KEY: ${{ secrets.MISTRAL_KEY }} + OPEN_AI_KEY: ${{ secrets.OPEN_AI_KEY }} + UPSTAGE_AI_KEY: ${{ secrets.UPSTAGE_AI_KEY }} + VOYAGE_AI_KEY: ${{ secrets.VOYAGE_AI_KEY }} + run: | ./mvnw -B -ntp verify -Dfmt.skip -DskipUnitTests -Dit.test=TestBenchTests \ No newline at end of file diff --git a/src/test/resources/integration-tests/test-plans/vectorize-astra-test-plan.yaml b/src/test/resources/integration-tests/test-plans/vectorize-astra-test-plan.yaml index a02047b93f..d571a3a4f6 100644 --- a/src/test/resources/integration-tests/test-plans/vectorize-astra-test-plan.yaml +++ b/src/test/resources/integration-tests/test-plans/vectorize-astra-test-plan.yaml @@ -1,4 +1,4 @@ -name: Astra ENV +name: Test Plan - Astra - Vectorize Workflows customTarget: name: ${TARGET_NAME} backend: astra diff --git a/src/test/resources/integration-tests/vectorize/vectorize-header-workflow.json b/src/test/resources/integration-tests/vectorize/vectorize-header-workflow.json index 3ed6e77ffb..f4f6492e9b 100644 --- a/src/test/resources/integration-tests/vectorize/vectorize-header-workflow.json +++ b/src/test/resources/integration-tests/vectorize/vectorize-header-workflow.json @@ -8,12 +8,12 @@ "meta": { "name": "open-ai-vectorize", "tags": [ - "disabled" + ] }, "fromEnvironment": { - "x-embedding-api-key": "openAi_KEY", - "Token": "Token" + "x-embedding-api-key": "OPEN_AI_KEY", + "Token": "DATA_API_TOKEN" }, "variables": { "PROVIDER": "openai", @@ -33,12 +33,12 @@ { "meta": { "name": "voyageAI-vectorize", - "tags": ["disabled" + "tags": [ ] }, "fromEnvironment": { - "x-embedding-api-key": "voyageAI_KEY", - "Token": "Token" + "x-embedding-api-key": "VOYAGE_AI_KEY", + "Token": "DATA_API_TOKEN" }, "variables": { "PROVIDER": "voyageAI", @@ -60,12 +60,12 @@ { "meta": { "name": "jinaAI-vectorize", - "tags": ["disabled" + "tags": [ ] }, "fromEnvironment": { - "x-embedding-api-key": "jinaAI_KEY", - "Token": "Token" + "x-embedding-api-key": "JINA_AI_KEY", + "Token": "DATA_API_TOKEN" }, "variables": { "PROVIDER": "jinaAI", @@ -88,12 +88,12 @@ "meta": { "name": "huggingface-non-dedicated-vectorize", "tags": [ - "disabled" + ] }, "fromEnvironment": { - "x-embedding-api-key": "huggingface_KEY", - "Token": "Token" + "x-embedding-api-key": "HUGGINGFACE_KEY", + "Token": "DATA_API_TOKEN" }, "variables": { "PROVIDER": "huggingface", @@ -117,12 +117,12 @@ "meta": { "name": "mistral-vectorize", "tags": [ - "disabled" + ] }, "fromEnvironment": { - "x-embedding-api-key": "mistral_KEY", - "Token": "Token" + "x-embedding-api-key": "MISTRAL_KEY", + "Token": "DATA_API_TOKEN" }, "variables": { "PROVIDER": "mistral", @@ -141,12 +141,12 @@ "meta": { "name": "upstageAI-vectorize", "tags": [ - "disabled" + ] }, "fromEnvironment": { - "x-embedding-api-key": "upstageAI_KEY", - "Token": "Token" + "x-embedding-api-key": "UPSTAGE_AI_KEY", + "Token": "DATA_API_TOKEN" }, "variables": { "PROVIDER": "upstageAI", @@ -168,8 +168,8 @@ ] }, "fromEnvironment": { - "x-embedding-api-key": "Token", - "Token": "Token" + "x-embedding-api-key": "DATA_API_TOKEN", + "Token": "DATA_API_TOKEN" }, "variables": { "PROVIDER": "nvidia", @@ -178,7 +178,7 @@ "matrix": { "MODEL": [ "NV-Embed-QA", - "nvidia/nv-embedqa-e5-v5-xxx" + "nvidia/nv-embedqa-e5-v5" ] }, "tests": [ diff --git a/src/test/resources/integration-tests/vectorize/vectorize-shared-workflow.json b/src/test/resources/integration-tests/vectorize/vectorize-shared-workflow.json index 40f38a52d0..c379cac5de 100644 --- a/src/test/resources/integration-tests/vectorize/vectorize-shared-workflow.json +++ b/src/test/resources/integration-tests/vectorize/vectorize-shared-workflow.json @@ -59,7 +59,7 @@ { "meta": { "name": "jinaAI-vectorize", - "tags": ["disabled" + "tags": [ ] }, "fromEnvironment": { @@ -87,7 +87,7 @@ "meta": { "name": "huggingface-non-dedicated-vectorize", "tags": [ - "disabled" + ] }, "fromEnvironment": { @@ -116,7 +116,7 @@ "meta": { "name": "mistral-vectorize", "tags": [ - "disabled" + ] }, "fromEnvironment": { @@ -140,7 +140,7 @@ "meta": { "name": "upstageAI-vectorize", "tags": [ - "disabled" + ] }, "fromEnvironment": { From 1f3c7540aae9a86f7a038900bc61abaad81ec3b6 Mon Sep 17 00:00:00 2001 From: Aaron Morton Date: Tue, 21 Apr 2026 16:01:27 +1200 Subject: [PATCH 47/89] fix file name --- .github/workflows/test-bench-vectorize.yaml | 2 +- ...rize-astra-test-plan.yaml => test-plan-astra-vectorize.yaml} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename src/test/resources/integration-tests/test-plans/{vectorize-astra-test-plan.yaml => test-plan-astra-vectorize.yaml} (100%) diff --git a/.github/workflows/test-bench-vectorize.yaml b/.github/workflows/test-bench-vectorize.yaml index 0f9a322cb5..3ad87ede30 100644 --- a/.github/workflows/test-bench-vectorize.yaml +++ b/.github/workflows/test-bench-vectorize.yaml @@ -102,7 +102,7 @@ jobs: EOF - name: Build & Test env: - TEST_PLAN_FILE: classpath:integration-tests/test-plans/test-bench-vectorize.yaml + TEST_PLAN_FILE: classpath:integration-tests/test-plans/test-plan-astra-vectorize.yaml Token: ${{ secrets.DATA_API_TOKEN }} TARGET_NAME: ${{ matrix.name }} ENDPOINT: ${{ matrix.endpoint }} diff --git a/src/test/resources/integration-tests/test-plans/vectorize-astra-test-plan.yaml b/src/test/resources/integration-tests/test-plans/test-plan-astra-vectorize.yaml similarity index 100% rename from src/test/resources/integration-tests/test-plans/vectorize-astra-test-plan.yaml rename to src/test/resources/integration-tests/test-plans/test-plan-astra-vectorize.yaml From 6bd433a88f12a5977c890bf32fc6475b6ae8d6f7 Mon Sep 17 00:00:00 2001 From: Aaron Morton Date: Tue, 21 Apr 2026 16:30:20 +1200 Subject: [PATCH 48/89] enable shared auth tests --- .github/workflows/test-bench-vectorize.yaml | 2 +- .../v1/vectorize/messaging/APIRequest.java | 16 +++-- .../api/v1/vectorize/testrun/TestRunEnv.java | 2 +- .../vectorize/vectorize-shared-workflow.json | 62 ++++++++++--------- 4 files changed, 45 insertions(+), 37 deletions(-) diff --git a/.github/workflows/test-bench-vectorize.yaml b/.github/workflows/test-bench-vectorize.yaml index 3ad87ede30..8bfdfc1730 100644 --- a/.github/workflows/test-bench-vectorize.yaml +++ b/.github/workflows/test-bench-vectorize.yaml @@ -103,7 +103,7 @@ jobs: - name: Build & Test env: TEST_PLAN_FILE: classpath:integration-tests/test-plans/test-plan-astra-vectorize.yaml - Token: ${{ secrets.DATA_API_TOKEN }} + DATA_API_TOKEN: ${{ secrets.DATA_API_TOKEN }} TARGET_NAME: ${{ matrix.name }} ENDPOINT: ${{ matrix.endpoint }} HUGGINGFACE_KEY: ${{ secrets.HUGGINGFACE_KEY }} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/messaging/APIRequest.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/messaging/APIRequest.java index bac584a43e..cd6d8411d6 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/messaging/APIRequest.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/messaging/APIRequest.java @@ -5,6 +5,7 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; +import com.google.common.base.Strings; import io.restassured.http.ContentType; import io.restassured.response.Response; import io.restassured.response.ValidatableResponse; @@ -14,6 +15,8 @@ import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestRunEnv; import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.TestCommand; import io.stargate.sgv2.jsonapi.config.constants.HttpConstants; + +import java.util.HashMap; import java.util.Map; public class APIRequest { @@ -77,11 +80,14 @@ private ValidatableResponse executeRequest(RequestSpecification requestSpec) { protected Map getHeaders() { - return Map.of( - HttpConstants.AUTHENTICATION_TOKEN_HEADER_NAME, - integrationEnv.requiredValue("Token"), - HttpConstants.EMBEDDING_AUTHENTICATION_TOKEN_HEADER_NAME, - integrationEnv.requiredValue("x-embedding-api-key")); + var headers = new HashMap(); + headers.put(HttpConstants.AUTHENTICATION_TOKEN_HEADER_NAME, integrationEnv.requiredValue("Token")); + + var embeddingApiKey = integrationEnv.get("embeddingApiKey"); + if (!Strings.isNullOrEmpty(embeddingApiKey)){ + headers.put(HttpConstants.AUTHENTICATION_TOKEN_HEADER_NAME, embeddingApiKey); + } + return headers; } public RequestSpecification jsonRequest() { diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testrun/TestRunEnv.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testrun/TestRunEnv.java index 6d035237c1..820b1e55ec 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testrun/TestRunEnv.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testrun/TestRunEnv.java @@ -95,7 +95,7 @@ public StringSubstitutor substitutor() { .setEnableUndefinedVariableException(true); } - private String get(String name) { + public String get(String name) { var value = vars.get(name); if (value == null) { diff --git a/src/test/resources/integration-tests/vectorize/vectorize-shared-workflow.json b/src/test/resources/integration-tests/vectorize/vectorize-shared-workflow.json index c379cac5de..b975896f9a 100644 --- a/src/test/resources/integration-tests/vectorize/vectorize-shared-workflow.json +++ b/src/test/resources/integration-tests/vectorize/vectorize-shared-workflow.json @@ -7,11 +7,12 @@ { "meta": { "name": "open-ai-vectorize", - "tags": ["disabled"] + "tags": [ + + ] }, "fromEnvironment": { - "x-embedding-api-key": "openAi_KEY", - "Token": "Token" + "Token": "DATA_API_TOKEN" }, "variables": { "PROVIDER": "openai", @@ -32,28 +33,31 @@ { "meta": { "name": "voyageAI-vectorize", - "tags": ["disabled" + "tags": [ ] }, "fromEnvironment": { - "x-embedding-api-key": "voyageAI_KEY", - "Token": "Token" + "Token": "DATA_API_TOKEN" }, "variables": { "PROVIDER": "voyageAI", - "COLLECTION_NAME": "${PROVIDER}-${MODEL}" + "COLLECTION_NAME": "${PROVIDER}-${MODEL}", + "CREDENTIAL" : "kent-voyageai-key" + }, "matrix": { "MODEL": [ "voyage-large-2-instruct", "voyage-law-2", - "voyage-code-2", "voyage-large-2", - "voyage-2" + "voyage-2", + "voyage-code-2", + "voyage-finance-2", + "voyage-multilingual-2" ] }, "tests": [ - "vectorize-header-auth" + "vectorize-shared-auth" ] }, { @@ -63,12 +67,12 @@ ] }, "fromEnvironment": { - "x-embedding-api-key": "jinaAI_KEY", - "Token": "Token" + "Token": "DATA_API_TOKEN" }, "variables": { "PROVIDER": "jinaAI", - "COLLECTION_NAME": "${PROVIDER}-${MODEL}" + "COLLECTION_NAME": "${PROVIDER}-${MODEL}", + "CREDENTIAL" : "kent-jinaaiai-key" }, "matrix": { "MODEL": [ @@ -76,27 +80,27 @@ "jina-embeddings-v2-base-de", "jina-embeddings-v2-base-es", "jina-embeddings-v2-base-code", - "jina-embeddings-v2-base-zh" + "jina-embeddings-v2-base-zh", + "jina-embeddings-v3" ] }, "tests": [ - "vectorize-header-auth" + "vectorize-shared-auth" ] }, { "meta": { "name": "huggingface-non-dedicated-vectorize", "tags": [ - ] }, "fromEnvironment": { - "x-embedding-api-key": "huggingface_KEY", - "Token": "Token" + "Token": "DATA_API_TOKEN" }, "variables": { "PROVIDER": "huggingface", - "COLLECTION_NAME": "${PROVIDER}-${MODEL}" + "COLLECTION_NAME": "${PROVIDER}-${MODEL}", + "CREDENTIAL" : "yuqi-huggingface-serverless-key" }, "matrix": { "MODEL": [ @@ -109,23 +113,22 @@ ] }, "tests": [ - "vectorize-header-auth" + "vectorize-shared-auth" ] }, { "meta": { "name": "mistral-vectorize", "tags": [ - ] }, "fromEnvironment": { - "x-embedding-api-key": "mistral_KEY", - "Token": "Token" + "Token": "DATA_API_TOKEN" }, "variables": { "PROVIDER": "mistral", - "COLLECTION_NAME": "${PROVIDER}-${MODEL}" + "COLLECTION_NAME": "${PROVIDER}-${MODEL}", + "CREDENTIAL" : "kent-mistralai-key" }, "matrix": { "MODEL": [ @@ -133,23 +136,22 @@ ] }, "tests": [ - "vectorize-header-auth" + "vectorize-shared-auth" ] }, { "meta": { "name": "upstageAI-vectorize", "tags": [ - ] }, "fromEnvironment": { - "x-embedding-api-key": "upstageAI_KEY", - "Token": "Token" + "Token": "DATA_API_TOKEN" }, "variables": { "PROVIDER": "upstageAI", - "COLLECTION_NAME": "${PROVIDER}-${MODEL}" + "COLLECTION_NAME": "${PROVIDER}-${MODEL}", + "CREDENTIAL" : "kent-upstage-key" }, "matrix": { "MODEL": [ @@ -157,7 +159,7 @@ ] }, "tests": [ - "vectorize-header-auth" + "vectorize-shared-auth" ] } ] From b168ed7132b75eff227809c1c6e8a0a6be584686 Mon Sep 17 00:00:00 2001 From: Aaron Morton Date: Tue, 21 Apr 2026 16:51:59 +1200 Subject: [PATCH 49/89] increase timeout, bug fix --- .github/workflows/test-bench-vectorize.yaml | 3 +-- .../sgv2/jsonapi/api/v1/vectorize/messaging/APIRequest.java | 6 +++--- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test-bench-vectorize.yaml b/.github/workflows/test-bench-vectorize.yaml index 8bfdfc1730..7aead11ca1 100644 --- a/.github/workflows/test-bench-vectorize.yaml +++ b/.github/workflows/test-bench-vectorize.yaml @@ -47,8 +47,7 @@ jobs: runs-on: ubuntu-latest environment: ${{ github.event.inputs.environment || 'DEV' }} - # max run time 12 minutes - timeout-minutes: 12 + timeout-minutes: 60 strategy: # do not fail fast, we want to run on all the different target dbs diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/messaging/APIRequest.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/messaging/APIRequest.java index cd6d8411d6..fccc0db137 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/messaging/APIRequest.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/messaging/APIRequest.java @@ -81,11 +81,11 @@ private ValidatableResponse executeRequest(RequestSpecification requestSpec) { protected Map getHeaders() { var headers = new HashMap(); - headers.put(HttpConstants.AUTHENTICATION_TOKEN_HEADER_NAME, integrationEnv.requiredValue("Token")); + headers.put(HttpConstants.AUTHENTICATION_TOKEN_HEADER_NAME, integrationEnv.requiredValue(HttpConstants.AUTHENTICATION_TOKEN_HEADER_NAME)); - var embeddingApiKey = integrationEnv.get("embeddingApiKey"); + var embeddingApiKey = integrationEnv.get(HttpConstants.EMBEDDING_AUTHENTICATION_TOKEN_HEADER_NAME); if (!Strings.isNullOrEmpty(embeddingApiKey)){ - headers.put(HttpConstants.AUTHENTICATION_TOKEN_HEADER_NAME, embeddingApiKey); + headers.put(HttpConstants.EMBEDDING_AUTHENTICATION_TOKEN_HEADER_NAME, embeddingApiKey); } return headers; } From e67a48d06e73f653f026078acef1affa33ab378d Mon Sep 17 00:00:00 2001 From: Aaron Morton Date: Wed, 22 Apr 2026 07:37:16 +1200 Subject: [PATCH 50/89] tweak output --- .github/workflows/test-bench-vectorize.yaml | 2 +- .../v1/vectorize/reporting/TestBenchConsoleWriter.java | 9 ++++----- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/.github/workflows/test-bench-vectorize.yaml b/.github/workflows/test-bench-vectorize.yaml index 7aead11ca1..ba66800746 100644 --- a/.github/workflows/test-bench-vectorize.yaml +++ b/.github/workflows/test-bench-vectorize.yaml @@ -47,7 +47,7 @@ jobs: runs-on: ubuntu-latest environment: ${{ github.event.inputs.environment || 'DEV' }} - timeout-minutes: 60 + timeout-minutes: 120 strategy: # do not fail fast, we want to run on all the different target dbs diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/reporting/TestBenchConsoleWriter.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/reporting/TestBenchConsoleWriter.java index 2da782f772..7d17194a4d 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/reporting/TestBenchConsoleWriter.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/reporting/TestBenchConsoleWriter.java @@ -150,7 +150,7 @@ private void writeCompletedSummary( // The name of the container or test - buffer.strong(tracker.identifier().getDisplayName()).a(timingInfo).newline(); + buffer.strong(tracker.identifier().getDisplayName()).a(timingInfo); // if we have stats write a stats line, these are aggregate for all things below. // alternative is to only print them for Test Environment these are lines like @@ -158,8 +158,10 @@ private void writeCompletedSummary( // to do that do this test: (tracker.runUri().leafType() == TestUri.Segment.ENV) if (tracker.stats() != null) { + buffer.a(theme.blank()); writeStatsLine(buffer, tracker); } + buffer.newline(); // we do not want to descent below the Test Envirinment unless there is a failure, otherwise there is too // much output. Tend ENV line looks like @@ -176,12 +178,9 @@ private void writeCompletedSummary( private void writeStatsLine(MessageBuilder buffer, DynamicTreeListener.TestReportingTracker tracker) { buffer - .a(theme.blank().repeat(tracker.depth())) - .a(theme.entry()) .a("Successful: ").a( tracker.stats().successful()).a(", ") .a("Failures: ").a( tracker.stats().failures()).a(", ") .a("Aborted: ").a( tracker.stats().aborted()).a(", ") - .a("Skipped: ").a( tracker.stats().skipped()) - .newline(); + .a("Skipped: ").a( tracker.stats().skipped()); } } From 393f74c1ea88097aec1f2c694ddcfa40e5122886 Mon Sep 17 00:00:00 2001 From: Aaron Morton Date: Wed, 22 Apr 2026 13:37:50 +1200 Subject: [PATCH 51/89] abort tests after previous failure --- .github/workflows/test-bench-vectorize.yaml | 2 +- .../assertions/SingleTestAssertion.java | 4 +- .../vectorize/assertions/TestAssertion.java | 3 +- .../assertions/TestAssertionContainer.java | 5 +- .../reporting/DynamicTreeListener.java | 32 +++++++--- .../reporting/TestBenchConsoleWriter.java | 9 ++- .../vectorize/targets/CassandraBackend.java | 7 +- .../testrun/DynamicTestExecutable.java | 19 ++++-- .../testrun/TestExecutionCondition.java | 64 +++++++++++++++++++ .../api/v1/vectorize/testrun/TestRunEnv.java | 3 +- .../v1/vectorize/testrun/TestRunRequest.java | 6 +- .../api/v1/vectorize/testrun/TestUri.java | 10 +++ .../api/v1/vectorize/testspec/TestCase.java | 6 +- .../v1/vectorize/testspec/TestSuiteSpec.java | 13 ++-- .../vectorize/vectorize-header-workflow.json | 5 +- 15 files changed, 153 insertions(+), 35 deletions(-) create mode 100644 src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testrun/TestExecutionCondition.java diff --git a/.github/workflows/test-bench-vectorize.yaml b/.github/workflows/test-bench-vectorize.yaml index ba66800746..8584ccbe45 100644 --- a/.github/workflows/test-bench-vectorize.yaml +++ b/.github/workflows/test-bench-vectorize.yaml @@ -114,4 +114,4 @@ jobs: run: | - ./mvnw -B -ntp verify -Dfmt.skip -DskipUnitTests -Dit.test=TestBenchTests \ No newline at end of file + ./mvnw -B -ntp verify -Dfmt.skip -DskipUnitTests -Dfailsafe.redirectTestOutputToFile=true -Dit.test=TestBenchTests \ No newline at end of file diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/SingleTestAssertion.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/SingleTestAssertion.java index f9d4a7a705..841f709662 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/SingleTestAssertion.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/SingleTestAssertion.java @@ -2,6 +2,7 @@ import com.fasterxml.jackson.databind.JsonNode; import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.DynamicTestExecutable; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestExecutionCondition; import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestRunResponse; import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestUri; import java.util.concurrent.atomic.AtomicReference; @@ -25,7 +26,7 @@ public void run(TestRunResponse testResponse) { @Override public DynamicNode testNodes( - TestUri.Builder uriBuilder, AtomicReference testResponse) { + TestUri.Builder uriBuilder, AtomicReference testResponse, TestExecutionCondition testExecutionCondition) { var matcherDesc = (matcher instanceof Describable d) ? d.describe() : ""; @@ -33,6 +34,7 @@ public DynamicNode testNodes( new DynamicTestExecutable( "%s [%s]".formatted(name(), matcherDesc), uriBuilder.addSegment(TestUri.Segment.ASSERTION, name()), + testExecutionCondition, () -> { var resp = testResponse.get(); if (resp == null) { diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/TestAssertion.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/TestAssertion.java index e7a43ccf92..215d781343 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/TestAssertion.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/TestAssertion.java @@ -2,6 +2,7 @@ import com.fasterxml.jackson.databind.JsonNode; import io.stargate.sgv2.jsonapi.api.v1.vectorize.TestPlan; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestExecutionCondition; import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestRunResponse; import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestUri; import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.AssertionTemplateSpec; @@ -21,7 +22,7 @@ public interface TestAssertion { void run(TestRunResponse testResponse); - DynamicNode testNodes(TestUri.Builder uriBuilder, AtomicReference testResponse); + DynamicNode testNodes(TestUri.Builder uriBuilder, AtomicReference testResponse, TestExecutionCondition testExecutionCondition); static List forSuccess(TestPlan testPlan, TestCommand testCommand) { diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/TestAssertionContainer.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/TestAssertionContainer.java index 6510245817..b8f7e9d237 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/TestAssertionContainer.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/TestAssertionContainer.java @@ -3,6 +3,7 @@ import static org.junit.jupiter.api.DynamicContainer.dynamicContainer; import com.fasterxml.jackson.databind.JsonNode; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestExecutionCondition; import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestRunResponse; import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestUri; import java.util.List; @@ -33,12 +34,12 @@ public void run(TestRunResponse testResponse) { @Override public DynamicNode testNodes( - TestUri.Builder uriBuilder, AtomicReference testResponse) { + TestUri.Builder uriBuilder, AtomicReference testResponse, TestExecutionCondition testExecutionCondition) { uriBuilder.addSegment(TestUri.Segment.ASSERTION, name()); var childs = assertions.stream() - .map(assertion -> assertion.testNodes(uriBuilder.clone(), testResponse)) + .map(assertion -> assertion.testNodes(uriBuilder.clone(), testResponse, testExecutionCondition)) .toList(); return dynamicContainer(name, childs); diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/reporting/DynamicTreeListener.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/reporting/DynamicTreeListener.java index 62f0694679..08a9297c1a 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/reporting/DynamicTreeListener.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/reporting/DynamicTreeListener.java @@ -239,14 +239,23 @@ public TestContainerStats stats() { public void executionFinished(TestExecutionResult result) { junitStatus = result.getStatus(); throwable = result.getThrowable(); + if (stats != null) { - stats.testCompleted(result); + stats.testCompleted(this, result); } if (parent != null) { - parent.executionFinished(result); + parent.descendantExecutionFinished(this, result); } } + private void descendantExecutionFinished(TestReportingTracker originalTracker, TestExecutionResult result) { + if (stats != null) { + stats.testCompleted(originalTracker, result); + } + if (parent != null) { + parent.descendantExecutionFinished(originalTracker, result); + } + } public void executionSkipped() { if (stats != null) { stats.testSkipped(); @@ -299,17 +308,20 @@ public int skipped() { return skipped; } - public boolean noErrors() { - return aborted == 0 && failures == 0; - } +// public boolean noErrors() { +// return aborted == 0 && failures == 0; +// } - public void testCompleted(TestExecutionResult result) { + public void testCompleted(TestReportingTracker tracker, TestExecutionResult result) { lastFinishedAtMillis = System.currentTimeMillis(); - switch (result.getStatus()) { - case FAILED -> failures++; - case ABORTED -> aborted++; - case SUCCESSFUL -> successful++; + // we only update the stats IF the test we are tracking is a TEST, we do not update for containers. + if (tracker.identifier().isTest()) { + switch (result.getStatus()) { + case FAILED -> failures++; + case ABORTED -> aborted++; + case SUCCESSFUL -> successful++; + } } } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/reporting/TestBenchConsoleWriter.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/reporting/TestBenchConsoleWriter.java index 7d17194a4d..470946de56 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/reporting/TestBenchConsoleWriter.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/reporting/TestBenchConsoleWriter.java @@ -133,7 +133,7 @@ private void writeCompletedSummary( case null -> {} } } else { - if (tracker.stats().noErrors()) { + if (tracker.stats().failures() ==0 && tracker.stats().aborted() == 0) { buffer.a(theme.successful()); } else { buffer.a(theme.failed()); @@ -167,11 +167,14 @@ private void writeCompletedSummary( // much output. Tend ENV line looks like // TestEnv: [MODEL=text-embedding-3-small, PROVIDER=openai] - 26 s - var noErrors = tracker.stats() != null && tracker.stats().noErrors(); + // If we have a TestEnv then we want to write out the summary of results for it, otherwise // descend until we get one - if (!noErrors || tracker.runUri().leafType() != TestUri.Segment.ENV) { + // OF if there are FAILURES then we descend, these are tests that ran but assertion failed. + // we do not descend for aborted, these are tests that did not run because of previous failure. + var hasFailures = tracker.stats() != null && tracker.stats().failures() > 0; + if (!tracker.runUri().leafType().descendantOf(TestUri.Segment.ENV) || hasFailures) { tracker.children().forEach(child -> writeCompletedSummary(buffer, child, false)); } } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/targets/CassandraBackend.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/targets/CassandraBackend.java index eb80347f16..988b13866f 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/targets/CassandraBackend.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/targets/CassandraBackend.java @@ -5,6 +5,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import io.stargate.sgv2.jsonapi.api.v1.vectorize.TestPlan; import io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions.TestAssertion; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestExecutionCondition; import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestRunRequest; import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestUri; import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.Job; @@ -50,7 +51,8 @@ public Optional beforeJob(TestPlan testPlan, TestUri.Builder uriBui command, testPlan.target(), env, - TestAssertion.forSuccess(testPlan, command)); + TestAssertion.forSuccess(testPlan, command), + new TestExecutionCondition.AlwaysTrue("CassandraBackend.beforeJob()")); return Optional.of(setupRequest.testNodes(uriBuilder)); } @@ -74,7 +76,8 @@ public Optional afterJob(TestPlan testPlan, TestUri.Builder uriBuil command, testPlan.target(), job.withoutMatrix(testPlan), - TestAssertion.forSuccess(testPlan, command)); + TestAssertion.forSuccess(testPlan, command), + new TestExecutionCondition.AlwaysTrue("CassandraBackend.afterJob()")); return Optional.of(setupRequest.testNodes(uriBuilder)); } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testrun/DynamicTestExecutable.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testrun/DynamicTestExecutable.java index a6ac29ec0c..5bd1ba3336 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testrun/DynamicTestExecutable.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testrun/DynamicTestExecutable.java @@ -2,6 +2,7 @@ import static org.junit.jupiter.api.DynamicTest.dynamicTest; +import org.junit.jupiter.api.Assumptions; import org.junit.jupiter.api.DynamicTest; import org.junit.jupiter.api.function.Executable; @@ -13,15 +14,17 @@ public class DynamicTestExecutable implements Executable { private final String trimmedDisplayName; private final boolean isTrimmed; + private final TestExecutionCondition testExecutionCondition; - public DynamicTestExecutable(String description, TestUri.Builder testUri, Executable executable) { - this(description, testUri.build(), executable); + public DynamicTestExecutable(String description, TestUri.Builder testUri, TestExecutionCondition testExecutionCondition, Executable executable) { + this(description, testUri.build(), testExecutionCondition, executable); } @SuppressWarnings("StringEquality") - public DynamicTestExecutable(String description, TestUri testUri, Executable executable) { + public DynamicTestExecutable(String description, TestUri testUri, TestExecutionCondition testExecutionCondition, Executable executable) { this.description = description; this.testUri = testUri; + this.testExecutionCondition = testExecutionCondition; this.executable = executable; var truncated = @@ -30,6 +33,7 @@ public DynamicTestExecutable(String description, TestUri testUri, Executable exe : description; this.trimmedDisplayName = truncated; + // using reference quality to see it is a diff object. this.isTrimmed = truncated != description; } @@ -43,8 +47,15 @@ public DynamicTest testNode() { @Override public void execute() throws Throwable { + Assumptions.assumeTrue(testExecutionCondition, testExecutionCondition.message()); beforeExecute(); - executable.execute(); + try { + executable.execute(); + } + catch (Throwable e) { + testExecutionCondition.abortFutureTests("Failed Upstream: " + trimmedDisplayName); + throw e; + } afterExecute(); } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testrun/TestExecutionCondition.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testrun/TestExecutionCondition.java new file mode 100644 index 0000000000..70b242ad4a --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testrun/TestExecutionCondition.java @@ -0,0 +1,64 @@ +package io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun; + +import java.util.function.BooleanSupplier; + +public interface TestExecutionCondition extends BooleanSupplier { + + void abortFutureTests(String message); + + String message(); + + public class Default implements TestExecutionCondition { + + private boolean condition = true; + private String message = ""; + + // to scope of this condition, i.e. this is for + // TestEnv: [MODEL=NV-Embed-QA, PROVIDER=nvidia] + private String scope; + + public Default(String scope) { + this.scope = scope; + } + @Override + public void abortFutureTests(String message) { + condition = false; + this.message = message; + } + + @Override + public boolean getAsBoolean() { + return condition; + } + + @Override + public String message() { + return "TestCondition: Scope=" + scope + ", Message=" + message; + } + } + + public class AlwaysTrue implements TestExecutionCondition { + + // to scope of this condition, i.e. this is for + // TestEnv: [MODEL=NV-Embed-QA, PROVIDER=nvidia] + // we never need a message, because always true, but here for debugging. + private String scope; + + public AlwaysTrue(String scope) { + this.scope = scope; + } + + @Override + public void abortFutureTests(String message) {} + + @Override + public boolean getAsBoolean() { + return true; + } + + @Override + public String message() { + return ""; + } + } +} \ No newline at end of file diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testrun/TestRunEnv.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testrun/TestRunEnv.java index 820b1e55ec..48e95f8b5a 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testrun/TestRunEnv.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testrun/TestRunEnv.java @@ -36,7 +36,8 @@ public DynamicContainer testNode( uriBuilder.addSegment(TestUri.Segment.ENV, d); var desc = "TestEnv: %s ".formatted(d); - var envNodes = testSuite.testNodesForEnvironment(testPlan, uriBuilder.clone(), this).stream(); + var testExecutionCondition = new TestExecutionCondition.Default(desc); + var envNodes = testSuite.testNodesForEnvironment(testPlan, uriBuilder.clone(), this, testExecutionCondition).stream(); return DynamicContainer.dynamicContainer( desc, diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testrun/TestRunRequest.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testrun/TestRunRequest.java index ec4b37c70c..2f5526dc95 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testrun/TestRunRequest.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testrun/TestRunRequest.java @@ -16,7 +16,8 @@ public record TestRunRequest( TestCommand testCommand, Target target, TestRunEnv testEnvironment, - List testAssertions) { + List testAssertions, + TestExecutionCondition testExecutionCondition) { public TestRunResponse execute() { @@ -40,6 +41,7 @@ public DynamicContainer testNodes(TestUri.Builder uriBuilder) { uriBuilder .clone() .addSegment(TestUri.Segment.COMMAND, testCommand.commandName().getApiName()), + testExecutionCondition, () -> atomicResponse.set(execute())); nodesBuilder.add(commandExecutable.testNode()); @@ -50,7 +52,7 @@ public DynamicContainer testNodes(TestUri.Builder uriBuilder) { testAssertions().stream() .map( testAssertion -> - testAssertion.testNodes(assertionsUriBuilder.clone(), atomicResponse)) + testAssertion.testNodes(assertionsUriBuilder.clone(), atomicResponse, testExecutionCondition)) .toList(); // if we have assertion tests, put them in a container diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testrun/TestUri.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testrun/TestUri.java index a9973b2394..a6fbada0d1 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testrun/TestUri.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testrun/TestUri.java @@ -80,6 +80,16 @@ public boolean isParentValid(Segment segment) { public String pathName() { return name().toLowerCase(); } + + public boolean descendantOf(Segment segment) { + if (parent == null){ + return false; + } + if (this.parent == segment) { + return true; + } + return this.parent.descendantOf(segment); + } } public record SegmentValue(Segment segment, String value) { diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/TestCase.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/TestCase.java index f7e5ff7d72..93e03c0d6d 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/TestCase.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/TestCase.java @@ -4,6 +4,7 @@ import com.fasterxml.jackson.databind.node.ObjectNode; import io.stargate.sgv2.jsonapi.api.v1.vectorize.TestPlan; import io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions.TestAssertion; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestExecutionCondition; import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestRunEnv; import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestRunRequest; import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestUri; @@ -16,7 +17,7 @@ public record TestCase( @JsonProperty("$include") String include) { public DynamicContainer testNodesForEnvironment( - TestPlan testPlan, TestUri.Builder uriBuilder, TestRunEnv testEnvironment) { + TestPlan testPlan, TestUri.Builder uriBuilder, TestRunEnv testEnvironment, TestExecutionCondition testExecutionCondition) { var testRequest = new TestRunRequest( @@ -24,7 +25,8 @@ public DynamicContainer testNodesForEnvironment( command(), testPlan.target(), testEnvironment, - TestAssertion.buildAssertions(testPlan, this)); + TestAssertion.buildAssertions(testPlan, this), + testExecutionCondition); return testRequest.testNodes(uriBuilder); } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/TestSuiteSpec.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/TestSuiteSpec.java index 8289428808..09a82c63ac 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/TestSuiteSpec.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/TestSuiteSpec.java @@ -4,6 +4,7 @@ import io.stargate.sgv2.jsonapi.api.v1.vectorize.*; import io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions.TestAssertion; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestExecutionCondition; import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestRunEnv; import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestRunRequest; import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestUri; @@ -31,7 +32,7 @@ public DynamicContainer testNode( } public Collection testNodesForEnvironment( - TestPlan testPlan, TestUri.Builder uriBuilder, TestRunEnv testEnvironment) { + TestPlan testPlan, TestUri.Builder uriBuilder, TestRunEnv testEnvironment, TestExecutionCondition testExecutionCondition) { List nodes = new ArrayList<>(); @@ -45,7 +46,8 @@ public Collection testNodesForEnvironment( setupCommand, testPlan.target(), testEnvironment, - TestAssertion.forSuccess(testPlan, setupCommand)); + TestAssertion.forSuccess(testPlan, setupCommand), + testExecutionCondition); nodes.add(setupRequest.testNodes(setupUriBuilder.clone())); } @@ -53,9 +55,11 @@ public Collection testNodesForEnvironment( var testUriBuilder = uriBuilder.clone().addSegment(TestUri.Segment.STAGE, "test"); for (var testCase : tests()) { nodes.add( - testCase.testNodesForEnvironment(testPlan, testUriBuilder.clone(), testEnvironment)); + testCase.testNodesForEnvironment(testPlan, testUriBuilder.clone(), testEnvironment, testExecutionCondition)); } + // NOTE: For Cleanup we use a condition that is always TRUE because we always want to try to run a cleanup task. + var alwaysTrueCondition = new TestExecutionCondition.AlwaysTrue("Cleanup Commands for parent URL: " + uriBuilder.build().uri().toString()); var cleanupUriBuilder = uriBuilder.clone().addSegment(TestUri.Segment.STAGE, "cleanup"); for (TestCommand cleanupCommand : cleanup()) { var cleanupRequest = @@ -64,7 +68,8 @@ public Collection testNodesForEnvironment( cleanupCommand, testPlan.target(), testEnvironment, - TestAssertion.forSuccess(testPlan, cleanupCommand)); + TestAssertion.forSuccess(testPlan, cleanupCommand), + alwaysTrueCondition); nodes.add(cleanupRequest.testNodes(cleanupUriBuilder.clone())); } diff --git a/src/test/resources/integration-tests/vectorize/vectorize-header-workflow.json b/src/test/resources/integration-tests/vectorize/vectorize-header-workflow.json index f4f6492e9b..d57d1d6871 100644 --- a/src/test/resources/integration-tests/vectorize/vectorize-header-workflow.json +++ b/src/test/resources/integration-tests/vectorize/vectorize-header-workflow.json @@ -8,7 +8,6 @@ "meta": { "name": "open-ai-vectorize", "tags": [ - ] }, "fromEnvironment": { @@ -34,6 +33,7 @@ "meta": { "name": "voyageAI-vectorize", "tags": [ + ] }, "fromEnvironment": { @@ -61,6 +61,7 @@ "meta": { "name": "jinaAI-vectorize", "tags": [ + ] }, "fromEnvironment": { @@ -177,7 +178,7 @@ }, "matrix": { "MODEL": [ - "NV-Embed-QA", + "NV-Embed-QA-XX", "nvidia/nv-embedqa-e5-v5" ] }, From 0f5219fdf05ebb194bfce3f5a4a2e1461b6f94aa Mon Sep 17 00:00:00 2001 From: Aaron Morton Date: Wed, 22 Apr 2026 13:44:22 +1200 Subject: [PATCH 52/89] typo fix --- .../integration-tests/vectorize/vectorize-shared-workflow.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/resources/integration-tests/vectorize/vectorize-shared-workflow.json b/src/test/resources/integration-tests/vectorize/vectorize-shared-workflow.json index b975896f9a..74a3448dea 100644 --- a/src/test/resources/integration-tests/vectorize/vectorize-shared-workflow.json +++ b/src/test/resources/integration-tests/vectorize/vectorize-shared-workflow.json @@ -72,7 +72,7 @@ "variables": { "PROVIDER": "jinaAI", "COLLECTION_NAME": "${PROVIDER}-${MODEL}", - "CREDENTIAL" : "kent-jinaaiai-key" + "CREDENTIAL" : "kent-jinaai-key" }, "matrix": { "MODEL": [ From 6cae9b5a8e86ab62f730e08cbcc283db46445a20 Mon Sep 17 00:00:00 2001 From: Aaron Morton Date: Wed, 22 Apr 2026 16:25:56 +1200 Subject: [PATCH 53/89] reporting improvements --- .github/workflows/test-bench-vectorize.yaml | 4 +- .../api/v1/vectorize/TestBenchTests.java | 22 ++- .../jsonapi/api/v1/vectorize/TestPlan.java | 130 ++------------ .../api/v1/vectorize/VectorizeAstra.java | 45 ----- .../jsonapi/api/v1/vectorize/VectorizeIT.java | 14 +- .../assertions/SingleTestAssertion.java | 10 +- .../vectorize/assertions/TestAssertion.java | 3 +- .../assertions/TestAssertionContainer.java | 11 +- .../lifecycle/TestPlanLifecycle.java | 14 +- .../reporting/DynamicTreeListener.java | 18 +- .../reporting/TestBenchConsoleWriter.java | 35 ++-- .../vectorize/targets/CassandraBackend.java | 24 +-- .../api/v1/vectorize/targets/Target.java | 26 +-- .../testrun/DynamicTestExecutable.java | 6 +- .../v1/vectorize/testrun/TestNodeFactory.java | 170 ++++++++++++++++++ .../api/v1/vectorize/testrun/TestRunEnv.java | 9 +- .../v1/vectorize/testrun/TestRunRequest.java | 22 +-- .../api/v1/vectorize/testspec/Job.java | 19 +- .../api/v1/vectorize/testspec/TestCase.java | 14 +- .../v1/vectorize/testspec/TestSuiteSpec.java | 40 +++-- .../v1/vectorize/testspec/WorkflowSpec.java | 23 ++- .../vectorize/vectorize-header-workflow.json | 5 +- 22 files changed, 353 insertions(+), 311 deletions(-) delete mode 100644 src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/VectorizeAstra.java create mode 100644 src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testrun/TestNodeFactory.java diff --git a/.github/workflows/test-bench-vectorize.yaml b/.github/workflows/test-bench-vectorize.yaml index 8584ccbe45..610647ad91 100644 --- a/.github/workflows/test-bench-vectorize.yaml +++ b/.github/workflows/test-bench-vectorize.yaml @@ -112,6 +112,6 @@ jobs: UPSTAGE_AI_KEY: ${{ secrets.UPSTAGE_AI_KEY }} VOYAGE_AI_KEY: ${{ secrets.VOYAGE_AI_KEY }} - + # reducing extra output with the -Dfailsafe.printSummary=false -Dfailsafe.trimStackTrace=true run: | - ./mvnw -B -ntp verify -Dfmt.skip -DskipUnitTests -Dfailsafe.redirectTestOutputToFile=true -Dit.test=TestBenchTests \ No newline at end of file + ./mvnw -B -ntp verify -Dfmt.skip -DskipUnitTests -Dfailsafe.printSummary=false -Dfailsafe.trimStackTrace=true -Dit.test=TestBenchTests \ No newline at end of file diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestBenchTests.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestBenchTests.java index dddd0fb048..639afbce72 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestBenchTests.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestBenchTests.java @@ -4,6 +4,8 @@ import java.nio.file.Path; import java.util.stream.Stream; import org.junit.jupiter.api.DynamicContainer; +import org.junit.jupiter.api.DynamicNode; +import org.junit.jupiter.api.DynamicTest; import org.junit.jupiter.api.TestFactory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -13,7 +15,7 @@ public class TestBenchTests { private static final Logger LOGGER = LoggerFactory.getLogger(TestBenchTests.class); @TestFactory - public Stream runTestPlanFile() { + public Stream runTestPlanFile() { var rawPath = System.getenv("TEST_PLAN_FILE"); LOGGER.info("runTestPlanFile() - getting TEST_PLAN_FILE from ENV, rawPath={}", rawPath); @@ -23,10 +25,24 @@ public Stream runTestPlanFile() { ? TargetsSpec.resourceDir(rawPath.substring("classpath:".length())) : Path.of(rawPath); - var testContext = TestPlan.fromFile(path); + var testPlan = TestPlan.fromFile(path); + LOGGER.info("runTestPlanFile() - building test plan tree"); + var testPlanNodeTree = testPlan.testNode(); - return testContext.testPlan().testNode(); + LOGGER.info("runTestPlanFile() - test plan tree build, totalNodeCount={}", testPlanNodeTree.totalNodeCount()); + System.setProperty("testbench.test.count", String.valueOf(testPlanNodeTree.totalNodeCount())); + return Stream.of(testPlanNodeTree.root()); } + + private static int countAllNodes(int accum, Stream nodes) { + return nodes.reduce(accum, (acc, node) -> switch (node) { + case DynamicTest t -> acc + 1; + case DynamicContainer c -> countAllNodes(acc, c.getChildren().map(n -> (DynamicNode) n)); + default -> acc; + }, Integer::sum); + } + + // // public static void main(String[] args) { // LauncherDiscoveryRequest request = LauncherDiscoveryRequestBuilder.request() diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestPlan.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestPlan.java index 5557626304..8cd0acfed9 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestPlan.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestPlan.java @@ -1,6 +1,6 @@ package io.stargate.sgv2.jsonapi.api.v1.vectorize; -import static org.junit.jupiter.api.DynamicContainer.dynamicContainer; + import com.fasterxml.jackson.core.JsonFactory; import com.fasterxml.jackson.core.StreamReadFeature; @@ -8,6 +8,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; import io.stargate.sgv2.jsonapi.api.v1.vectorize.targets.Target; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestNodeFactory; import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestRunEnv; import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestUri; import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.*; @@ -41,7 +42,7 @@ public record TestPlan( YAMLFactory.builder().enable(StreamReadFeature.INCLUDE_SOURCE_IN_LOCATION).build()) .configure(MapperFeature.ACCEPT_CASE_INSENSITIVE_ENUMS, true); - public static TestPlanContext fromFile(Path path) { + public static TestPlan fromFile(Path path) { LOGGER.info("fromFile() - Loading test plan file, path={}", path); TestPlanFile planFile; @@ -63,7 +64,7 @@ public static TestPlanContext fromFile(Path path) { ? create(planFile.customTarget(), planFile.workflows, planFile.ignoreDisabled) : create(planFile.targetName(), planFile.workflows(), planFile.ignoreDisabled()); - return new TestPlanContext(testPlan, planFile); + return testPlan; } public static TestPlan create(String targetName, List workflows) { @@ -101,7 +102,7 @@ public Stream selectedWorkflows() { .map(testSpec -> (WorkflowSpec) testSpec.spec()); } - public Stream testNode() { + public TestPlanNodeTree testNode() { var desc = "TestPlan: %s on %s workflows %s" @@ -114,114 +115,21 @@ public Stream testNode() { TestUri.builder(TestUri.Scheme.DATAAPI) .addSegment(TestUri.Segment.TARGET, target.configuration().name()); - return Stream.of( - dynamicContainer( + var testNodeFactory = new TestNodeFactory(this); + + var root = testNodeFactory.testPlanContainer( desc, uriBuilder.build().uri(), selectedWorkflows() - .map(workflow -> workflow.testNode(this, uriBuilder.clone(), ignoreDisabled)))); + .map(workflow -> workflow.testNode(testNodeFactory, uriBuilder.clone(), ignoreDisabled)) + .toList()); + return new TestPlanNodeTree(root, testNodeFactory.testNodeCount()); } public void updateJobForTarget(Job job) { target.updateJobForTarget(job); } - private static Optional containerIfPresent( - TestUri.Builder uriBuilder, - String namePrefix, - TestSpecMeta meta, - Optional dynamicNode) { - return dynamicNode.map( - node -> - dynamicContainer( - namePrefix + ": " + meta.name(), uriBuilder.build().uri(), Stream.of(node))); - } - - private static Stream streamIfPresent(Optional container) { - return container.stream().flatMap(Stream::of); - } - - private static Stream lifecycleNodes( - TestUri.Builder uriBuilder, - String namePrefix, - TestSpecMeta meta, - Supplier> nodeSupplier) { - var targetDynamicNode = nodeSupplier.get(); - return streamIfPresent(containerIfPresent(uriBuilder, namePrefix, meta, targetDynamicNode)); - } - - public Stream addLifecycle( - TestUri.Builder uriBuilder, - WorkflowSpec workflow, - Stream dynamicNodes) { - - var beforeUriBuilder = - uriBuilder.clone().addSegment(TestUri.Segment.LIFECYCLE, "before-workflow"); - var afterUriBuilder = - uriBuilder.clone().addSegment(TestUri.Segment.LIFECYCLE, "after-workflow"); - - return Stream.concat( - lifecycleNodes( - beforeUriBuilder, - "Before Workflow", - workflow.meta(), - () -> target.beforeWorkflow(this, beforeUriBuilder, workflow)), - Stream.concat( - dynamicNodes, - lifecycleNodes( - afterUriBuilder, - "After Workflow", - workflow.meta(), - () -> target.afterWorkflow(this, afterUriBuilder, workflow)))); - } - - public Stream addLifecycle( - TestUri.Builder uriBuilder, Job job, Stream dynamicNodes) { - - var beforeUriBuilder = uriBuilder.clone().addSegment(TestUri.Segment.LIFECYCLE, "before-job"); - var afterUriBuilder = uriBuilder.clone().addSegment(TestUri.Segment.LIFECYCLE, "after-job"); - - return Stream.concat( - lifecycleNodes( - beforeUriBuilder, - "Before Job", - job.meta(), - () -> target.beforeJob(this, beforeUriBuilder, job)), - Stream.concat( - dynamicNodes, - lifecycleNodes( - afterUriBuilder, - "After Job", - job.meta(), - () -> target.afterJob(this, afterUriBuilder, job)))); - } - - public Stream addLifecycle( - TestUri.Builder uriBuilder, - TestSuiteSpec testSuite, - TestRunEnv environment, - Stream dynamicNodes) { - - var beforeUriBuilder = - uriBuilder.clone().addSegment(TestUri.Segment.LIFECYCLE, "before-test-suite"); - var afterUriBuilder = - uriBuilder.clone().addSegment(TestUri.Segment.LIFECYCLE, "after-test-suite"); - - return Stream.concat( - lifecycleNodes( - beforeUriBuilder, - "Before TestSuite", - testSuite.meta(), - () -> target.beforeTestSuite(this, beforeUriBuilder, testSuite, environment)), - Stream.concat( - dynamicNodes, - lifecycleNodes( - afterUriBuilder, - "After TestSuite", - testSuite.meta(), - () -> target.afterTestSuite(this, afterUriBuilder, testSuite, environment)))); - } - public record TestPlanFile( String name, String targetName, @@ -230,18 +138,6 @@ public record TestPlanFile( Boolean ignoreDisabled, Map envVars) {} - public record TestPlanContext(TestPlan testPlan, TestPlanFile testPlanFile) - implements AutoCloseable { - - public TestPlanContext { - if (testPlanFile.envVars != null) { - testPlanFile.envVars.forEach(System::setProperty); - } - } - - @Override - public void close() { - testPlanFile.envVars.keySet().forEach(System::clearProperty); - } - } + public record TestPlanNodeTree(DynamicNode root, + int totalNodeCount){} } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/VectorizeAstra.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/VectorizeAstra.java deleted file mode 100644 index 94784d0c02..0000000000 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/VectorizeAstra.java +++ /dev/null @@ -1,45 +0,0 @@ -package io.stargate.sgv2.jsonapi.api.v1.vectorize; - -import java.util.List; -import java.util.stream.Stream; -import org.junit.jupiter.api.DynamicContainer; -import org.junit.jupiter.api.TestFactory; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class VectorizeAstra { - private static final Logger LOGGER = LoggerFactory.getLogger(VectorizeAstra.class); - - private static final int TEST_PARALLELISM = 8; // ← set this - - @TestFactory - Stream jobs() { - - var testPlan = TestPlan.create("astra-dev", List.of("all-vectorize-workflow"), false); - - return testPlan.testNode(); - - // Parallel - // int maxConcurrentJobs = 2; - // integrationTarget.workflowStarting(workflow); - // try { - // var executor = java.util.concurrent.Executors.newFixedThreadPool(maxConcurrentJobs); - // try { - // var futures = jobs.stream() - // .map(job -> java.util.concurrent.CompletableFuture.runAsync(() -> { - // LOGGER.info("Starting job {}", job.meta()); - // new IntegrationJobRunner(integrationTarget, itCollection, job).run(); - // }, executor)) - // .toList(); - // - // // wait for all, fail if any failed - // futures.forEach(java.util.concurrent.CompletableFuture::join); - // } finally { - // executor.shutdown(); - // } - // } finally { - // integrationTarget.workflowFinished(workflow); - // } - - } -} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/VectorizeIT.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/VectorizeIT.java index 1721e5787c..472d2e4148 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/VectorizeIT.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/VectorizeIT.java @@ -13,11 +13,11 @@ @WithTestResource(value = DseTestResource.class) public class VectorizeIT extends AbstractCollectionIntegrationTestBase { - @TestFactory - Stream jobs() { - - var testPlan = TestPlan.create("integration", List.of("all-vectorize-workflow")); - - return testPlan.testNode(); - } +// @TestFactory +// Stream jobs() { +// +// var testPlan = TestPlan.create("integration", List.of("all-vectorize-workflow")); +// +// return testPlan.testNode(); +// } } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/SingleTestAssertion.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/SingleTestAssertion.java index 841f709662..3c228b9b8e 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/SingleTestAssertion.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/SingleTestAssertion.java @@ -1,10 +1,8 @@ package io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions; import com.fasterxml.jackson.databind.JsonNode; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.DynamicTestExecutable; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestExecutionCondition; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestRunResponse; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestUri; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.*; + import java.util.concurrent.atomic.AtomicReference; import org.junit.jupiter.api.DynamicNode; @@ -26,7 +24,7 @@ public void run(TestRunResponse testResponse) { @Override public DynamicNode testNodes( - TestUri.Builder uriBuilder, AtomicReference testResponse, TestExecutionCondition testExecutionCondition) { + TestNodeFactory testNodeFactory, TestUri.Builder uriBuilder, AtomicReference testResponse, TestExecutionCondition testExecutionCondition) { var matcherDesc = (matcher instanceof Describable d) ? d.describe() : ""; @@ -43,6 +41,6 @@ public DynamicNode testNodes( run(resp); }); - return executable.testNode(); + return executable.testNode(testNodeFactory); } } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/TestAssertion.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/TestAssertion.java index 215d781343..aa3e250bdd 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/TestAssertion.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/TestAssertion.java @@ -3,6 +3,7 @@ import com.fasterxml.jackson.databind.JsonNode; import io.stargate.sgv2.jsonapi.api.v1.vectorize.TestPlan; import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestExecutionCondition; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestNodeFactory; import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestRunResponse; import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestUri; import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.AssertionTemplateSpec; @@ -22,7 +23,7 @@ public interface TestAssertion { void run(TestRunResponse testResponse); - DynamicNode testNodes(TestUri.Builder uriBuilder, AtomicReference testResponse, TestExecutionCondition testExecutionCondition); + DynamicNode testNodes(TestNodeFactory testNodeFactory, TestUri.Builder uriBuilder, AtomicReference testResponse, TestExecutionCondition testExecutionCondition); static List forSuccess(TestPlan testPlan, TestCommand testCommand) { diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/TestAssertionContainer.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/TestAssertionContainer.java index b8f7e9d237..b851884e0e 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/TestAssertionContainer.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/TestAssertionContainer.java @@ -1,9 +1,10 @@ package io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions; -import static org.junit.jupiter.api.DynamicContainer.dynamicContainer; + import com.fasterxml.jackson.databind.JsonNode; import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestExecutionCondition; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestNodeFactory; import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestRunResponse; import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestUri; import java.util.List; @@ -34,14 +35,14 @@ public void run(TestRunResponse testResponse) { @Override public DynamicNode testNodes( - TestUri.Builder uriBuilder, AtomicReference testResponse, TestExecutionCondition testExecutionCondition) { + TestNodeFactory testNodeFactory, TestUri.Builder uriBuilder, AtomicReference testResponse, TestExecutionCondition testExecutionCondition) { uriBuilder.addSegment(TestUri.Segment.ASSERTION, name()); var childs = assertions.stream() - .map(assertion -> assertion.testNodes(uriBuilder.clone(), testResponse, testExecutionCondition)) - .toList(); + .map(assertion -> assertion.testNodes(testNodeFactory, uriBuilder.clone(), testResponse, testExecutionCondition)) + .toList(); - return dynamicContainer(name, childs); + return testNodeFactory.testPlanContainer(name, uriBuilder.build().uri(), childs); } } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/lifecycle/TestPlanLifecycle.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/lifecycle/TestPlanLifecycle.java index d79ba00192..53133be634 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/lifecycle/TestPlanLifecycle.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/lifecycle/TestPlanLifecycle.java @@ -1,6 +1,6 @@ package io.stargate.sgv2.jsonapi.api.v1.vectorize.lifecycle; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.TestPlan; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestNodeFactory; import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestRunEnv; import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestUri; import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.Job; @@ -12,30 +12,30 @@ public interface TestPlanLifecycle { default Optional beforeWorkflow( - TestPlan testPlan, TestUri.Builder uriBuilder, WorkflowSpec workflow) { + TestNodeFactory testNodeFactory, TestUri.Builder uriBuilder, WorkflowSpec workflow) { return Optional.empty(); } default Optional afterWorkflow( - TestPlan testPlan, TestUri.Builder uriBuilder, WorkflowSpec workflow) { + TestNodeFactory testNodeFactory, TestUri.Builder uriBuilder, WorkflowSpec workflow) { return Optional.empty(); } - default Optional beforeJob(TestPlan testPlan, TestUri.Builder uriBuilder, Job job) { + default Optional beforeJob(TestNodeFactory testNodeFactory, TestUri.Builder uriBuilder, Job job) { return Optional.empty(); } - default Optional afterJob(TestPlan testPlan, TestUri.Builder uriBuilder, Job job) { + default Optional afterJob(TestNodeFactory testNodeFactory, TestUri.Builder uriBuilder, Job job) { return Optional.empty(); } default Optional beforeTestSuite( - TestPlan testPlan, TestUri.Builder uriBuilder, TestSuiteSpec test, TestRunEnv env) { + TestNodeFactory testNodeFactory, TestUri.Builder uriBuilder, TestSuiteSpec test, TestRunEnv env) { return Optional.empty(); } default Optional afterTestSuite( - TestPlan testPlan, TestUri.Builder uriBuilder, TestSuiteSpec test, TestRunEnv env) { + TestNodeFactory testNodeFactory, TestUri.Builder uriBuilder, TestSuiteSpec test, TestRunEnv env) { return Optional.empty(); } } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/reporting/DynamicTreeListener.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/reporting/DynamicTreeListener.java index 08a9297c1a..d3ab9c5a0c 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/reporting/DynamicTreeListener.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/reporting/DynamicTreeListener.java @@ -19,6 +19,8 @@ public class DynamicTreeListener implements TestExecutionListener { private static final Logger LOGGER = LoggerFactory.getLogger(DynamicTreeListener.class); + private Integer totalTestCount = null; + private int startedTestCount = 0; private TestReportingTracker rootTracker; private final Map testTrackers = new ConcurrentHashMap<>(); @@ -28,7 +30,8 @@ public class DynamicTreeListener implements TestExecutionListener { private final TestBenchConsoleWriter writer = new TestBenchConsoleWriter(); @Override - public void testPlanExecutionStarted(TestPlan testPlan) {} + public void testPlanExecutionStarted(TestPlan testPlan) { + } @Override public void testPlanExecutionFinished(TestPlan testPlan) { @@ -36,7 +39,8 @@ public void testPlanExecutionFinished(TestPlan testPlan) { } @Override - public void dynamicTestRegistered(TestIdentifier testIdentifier) {} + public void dynamicTestRegistered(TestIdentifier id) { + } @Override public void executionStarted(TestIdentifier id) { @@ -47,7 +51,15 @@ public void executionStarted(TestIdentifier id) { if (tracker == null) { return; } - writer.testStarted(tracker); + + // we will not see the test count until we see the first dymamic test node we create, e.g. + //TestPlan: smoketest-aws-us-east-1 on astra workflows vectorize-header-workflow + // if we have a tracker, its one of our tests. + if (totalTestCount == null) { + totalTestCount = Integer.parseInt(System.getProperty("testbench.test.count", "0")); + } + + writer.testStarted(totalTestCount,++startedTestCount, tracker); } @Override diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/reporting/TestBenchConsoleWriter.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/reporting/TestBenchConsoleWriter.java index 470946de56..9f8c3c5bdc 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/reporting/TestBenchConsoleWriter.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/reporting/TestBenchConsoleWriter.java @@ -47,7 +47,7 @@ public TestBenchConsoleWriter(Theme theme) { * * @param tracker Tracker for the test that is running. */ - public void testStarted(DynamicTreeListener.TestReportingTracker tracker) { + public void testStarted(int totalTestCount, int startedTestCount, DynamicTreeListener.TestReportingTracker tracker) { var buffer = buffer(); if (firstLine) { @@ -65,13 +65,16 @@ public void testStarted(DynamicTreeListener.TestReportingTracker tracker) { // if there is no parent, this is the first test that is running. // display name is the name the TestBench put on the container if (tracker.parent() == null) { - buffer.a(theme.down()).strong(tracker.identifier().getDisplayName()); + buffer.a(theme.down()); } else { buffer .a(theme.blank().repeat(tracker.depth() - 1)) - .a(theme.entry()) - .strong(tracker.identifier().getDisplayName()); + .a(theme.entry()); } + + buffer.a(startedTestCount).a(" of ").a(totalTestCount).a(": "); + buffer.strong(tracker.identifier().getDisplayName()); + LOGGER.info(buffer.toString()); } @@ -93,7 +96,7 @@ public void allTestsFinished(DynamicTreeListener.TestReportingTracker rootTracke .a(theme.dash().repeat(20)) .newline(); - writeCompletedSummary(buffer, rootTracker, true); + writeCompletedSummary(buffer, rootTracker, true, false); LOGGER.info(buffer.toString()); } @@ -110,7 +113,8 @@ public void allTestsFinished(DynamicTreeListener.TestReportingTracker rootTracke private void writeCompletedSummary( MessageBuilder buffer, DynamicTreeListener.TestReportingTracker tracker, - boolean isRoot) { + boolean isRoot, + boolean parentFailures) { // the tree part of the line if (isRoot) { @@ -140,14 +144,6 @@ private void writeCompletedSummary( } } -// if (tracker.stats() == null) { -// buffer.a(theme.blank()); -// } else if (tracker.stats().noErrors()) { -// buffer.a(theme.successful()); -// } else { -// buffer.a(theme.failed()); -// } - // The name of the container or test buffer.strong(tracker.identifier().getDisplayName()).a(timingInfo); @@ -173,9 +169,11 @@ private void writeCompletedSummary( // descend until we get one // OF if there are FAILURES then we descend, these are tests that ran but assertion failed. // we do not descend for aborted, these are tests that did not run because of previous failure. - var hasFailures = tracker.stats() != null && tracker.stats().failures() > 0; - if (!tracker.runUri().leafType().descendantOf(TestUri.Segment.ENV) || hasFailures) { - tracker.children().forEach(child -> writeCompletedSummary(buffer, child, false)); + for (var child : tracker.children()){ + var hasFailures = child.stats() != null && child.stats().failures() > 0; + if (!child.runUri().leafType().descendantOf(TestUri.Segment.ENV) || hasFailures || parentFailures) { + writeCompletedSummary(buffer, child, false, hasFailures); + } } } @@ -183,7 +181,6 @@ private void writeStatsLine(MessageBuilder buffer, DynamicTreeListener.TestRepor buffer .a("Successful: ").a( tracker.stats().successful()).a(", ") .a("Failures: ").a( tracker.stats().failures()).a(", ") - .a("Aborted: ").a( tracker.stats().aborted()).a(", ") - .a("Skipped: ").a( tracker.stats().skipped()); + .a("Aborted: ").a( tracker.stats().aborted()).a(", "); } } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/targets/CassandraBackend.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/targets/CassandraBackend.java index 988b13866f..39c5f3f0cb 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/targets/CassandraBackend.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/targets/CassandraBackend.java @@ -3,9 +3,9 @@ import static io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestRunEnv.toSafeSchemaIdentifier; import com.fasterxml.jackson.databind.ObjectMapper; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.TestPlan; import io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions.TestAssertion; import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestExecutionCondition; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestNodeFactory; import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestRunRequest; import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestUri; import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.Job; @@ -32,7 +32,7 @@ public void updateJobForTarget(Job job) { } @Override - public Optional beforeJob(TestPlan testPlan, TestUri.Builder uriBuilder, Job job) { + public Optional beforeJob(TestNodeFactory testNodeFactory, TestUri.Builder uriBuilder, Job job) { var command = TestCommand.fromJson( @@ -44,21 +44,21 @@ public Optional beforeJob(TestPlan testPlan, TestUri.Builder uriBui } """); - var env = job.withoutMatrix(testPlan); + var env = job.withoutMatrix(testNodeFactory.testPlan()); var setupRequest = new TestRunRequest( env.substitutor().replace("createKeyspace: ${KEYSPACE_NAME}"), command, - testPlan.target(), + testNodeFactory.testPlan().target(), env, - TestAssertion.forSuccess(testPlan, command), + TestAssertion.forSuccess(testNodeFactory.testPlan(), command), new TestExecutionCondition.AlwaysTrue("CassandraBackend.beforeJob()")); - return Optional.of(setupRequest.testNodes(uriBuilder)); + return Optional.of(setupRequest.testNodes(testNodeFactory, uriBuilder)); } @Override - public Optional afterJob(TestPlan testPlan, TestUri.Builder uriBuilder, Job job) { + public Optional afterJob(TestNodeFactory testNodeFactory, TestUri.Builder uriBuilder, Job job) { var command = TestCommand.fromJson( """ @@ -69,16 +69,16 @@ public Optional afterJob(TestPlan testPlan, TestUri.Builder uriBuil } """); - var env = job.withoutMatrix(testPlan); + var env = job.withoutMatrix(testNodeFactory.testPlan()); var setupRequest = new TestRunRequest( env.substitutor().replace("dropKeyspace: ${KEYSPACE_NAME}"), command, - testPlan.target(), - job.withoutMatrix(testPlan), - TestAssertion.forSuccess(testPlan, command), + testNodeFactory.testPlan().target(), + job.withoutMatrix(testNodeFactory.testPlan()), + TestAssertion.forSuccess(testNodeFactory.testPlan(), command), new TestExecutionCondition.AlwaysTrue("CassandraBackend.afterJob()")); - return Optional.of(setupRequest.testNodes(uriBuilder)); + return Optional.of(setupRequest.testNodes(testNodeFactory, uriBuilder)); } } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/targets/Target.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/targets/Target.java index 88dbc3c13a..27a9168f5a 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/targets/Target.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/targets/Target.java @@ -1,8 +1,8 @@ package io.stargate.sgv2.jsonapi.api.v1.vectorize.targets; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.*; import io.stargate.sgv2.jsonapi.api.v1.vectorize.lifecycle.TestPlanLifecycle; import io.stargate.sgv2.jsonapi.api.v1.vectorize.messaging.APIRequest; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestNodeFactory; import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestRunEnv; import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestUri; import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.Job; @@ -52,35 +52,35 @@ public APIRequest apiRequest(TestCommand testCommand, TestRunEnv env) { @Override public Optional beforeWorkflow( - TestPlan testPlan, TestUri.Builder uriBuilder, WorkflowSpec workflow) { - return backend.beforeWorkflow(testPlan, uriBuilder, workflow); + TestNodeFactory testNodeFactory, TestUri.Builder uriBuilder, WorkflowSpec workflow) { + return backend.beforeWorkflow(testNodeFactory, uriBuilder, workflow); } @Override public Optional afterWorkflow( - TestPlan testPlan, TestUri.Builder uriBuilder, WorkflowSpec workflow) { - return backend.afterWorkflow(testPlan, uriBuilder, workflow); + TestNodeFactory testNodeFactory, TestUri.Builder uriBuilder, WorkflowSpec workflow) { + return backend.afterWorkflow(testNodeFactory, uriBuilder, workflow); } @Override - public Optional beforeJob(TestPlan testPlan, TestUri.Builder uriBuilder, Job job) { - return backend.beforeJob(testPlan, uriBuilder, job); + public Optional beforeJob(TestNodeFactory testNodeFactory, TestUri.Builder uriBuilder, Job job) { + return backend.beforeJob(testNodeFactory, uriBuilder, job); } @Override - public Optional afterJob(TestPlan testPlan, TestUri.Builder uriBuilder, Job job) { - return backend.afterJob(testPlan, uriBuilder, job); + public Optional afterJob(TestNodeFactory testNodeFactory, TestUri.Builder uriBuilder, Job job) { + return backend.afterJob(testNodeFactory, uriBuilder, job); } @Override public Optional beforeTestSuite( - TestPlan testPlan, TestUri.Builder uriBuilder, TestSuiteSpec test, TestRunEnv env) { - return backend.beforeTestSuite(testPlan, uriBuilder, test, env); + TestNodeFactory testNodeFactory, TestUri.Builder uriBuilder, TestSuiteSpec test, TestRunEnv env) { + return backend.beforeTestSuite(testNodeFactory, uriBuilder, test, env); } @Override public Optional afterTestSuite( - TestPlan testPlan, TestUri.Builder uriBuilder, TestSuiteSpec test, TestRunEnv env) { - return backend.afterTestSuite(testPlan, uriBuilder, test, env); + TestNodeFactory testNodeFactory, TestUri.Builder uriBuilder, TestSuiteSpec test, TestRunEnv env) { + return backend.afterTestSuite(testNodeFactory, uriBuilder, test, env); } } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testrun/DynamicTestExecutable.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testrun/DynamicTestExecutable.java index 5bd1ba3336..c436e1617c 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testrun/DynamicTestExecutable.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testrun/DynamicTestExecutable.java @@ -1,7 +1,5 @@ package io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun; -import static org.junit.jupiter.api.DynamicTest.dynamicTest; - import org.junit.jupiter.api.Assumptions; import org.junit.jupiter.api.DynamicTest; import org.junit.jupiter.api.function.Executable; @@ -41,8 +39,8 @@ public String trimmedDisplayName() { return trimmedDisplayName; } - public DynamicTest testNode() { - return dynamicTest(trimmedDisplayName, testUri.uri(), this); + public DynamicTest testNode(TestNodeFactory testNodeFactory) { + return testNodeFactory.testPlanTest(trimmedDisplayName, testUri.uri(), this); } @Override diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testrun/TestNodeFactory.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testrun/TestNodeFactory.java new file mode 100644 index 0000000000..0de3ac7620 --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testrun/TestNodeFactory.java @@ -0,0 +1,170 @@ +package io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun; + +import io.stargate.sgv2.jsonapi.api.v1.vectorize.TestPlan; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.Job; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.TestSpecMeta; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.TestSuiteSpec; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.WorkflowSpec; +import org.junit.jupiter.api.DynamicContainer; +import org.junit.jupiter.api.DynamicNode; +import org.junit.jupiter.api.DynamicTest; +import org.junit.jupiter.api.function.Executable; + +import java.net.URI; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Optional; +import java.util.function.Supplier; +import java.util.stream.Stream; + +/** + * A container we can pass around when building the TestNodes that has the test plan + * we are working with, and has a counter for the number of nodes we created. + * + *

+ * So we can output them for the test reporter to report progress. + *

+ */ +public class TestNodeFactory { + + private final TestPlan testPlan; + private int totalNodeCount = 0; + + public TestNodeFactory(TestPlan testPlan) { + this.testPlan = testPlan; + } + + public TestPlan testPlan() { + return testPlan; + } + + public int testNodeCount(){ + return totalNodeCount; + } + + /** + * NOTE: This is forcing the use of a List, so we greedily create all the test nodes, so we can count + * how many their are, so we can show progress. + * @param displayName + * @param testSourceUri + * @param dynamicNodes + * @return + */ + public DynamicContainer testPlanContainer(String displayName, URI testSourceUri, + List dynamicNodes){ + totalNodeCount++; + return DynamicContainer.dynamicContainer(displayName, testSourceUri, dynamicNodes.stream()); + } + + public DynamicTest testPlanTest(String description, URI uri, Executable executable){ + + totalNodeCount++; + return DynamicTest.dynamicTest(description, uri, executable); + } + + public List addLifecycle( + TestUri.Builder uriBuilder, + WorkflowSpec workflow, + List dynamicNodes) { + + var beforeUriBuilder = + uriBuilder.clone().addSegment(TestUri.Segment.LIFECYCLE, "before-workflow"); + var afterUriBuilder = + uriBuilder.clone().addSegment(TestUri.Segment.LIFECYCLE, "after-workflow"); + + var nodes = new ArrayList(); + nodes.addAll(lifecycleNodes( + beforeUriBuilder, + "Before Workflow", + workflow.meta(), + () -> testPlan.target().beforeWorkflow(this, beforeUriBuilder, workflow))); + nodes.addAll(dynamicNodes); + nodes.addAll(lifecycleNodes( + afterUriBuilder, + "After Workflow", + workflow.meta(), + () -> testPlan.target().afterWorkflow(this, afterUriBuilder, workflow))); + return nodes; + } + + public List addLifecycle( + TestUri.Builder uriBuilder, Job job, List dynamicNodes) { + + var beforeUriBuilder = uriBuilder.clone().addSegment(TestUri.Segment.LIFECYCLE, "before-job"); + var afterUriBuilder = uriBuilder.clone().addSegment(TestUri.Segment.LIFECYCLE, "after-job"); + + var nodes = new ArrayList(); + nodes.addAll(lifecycleNodes( + beforeUriBuilder, + "Before Job", + job.meta(), + () -> testPlan.target().beforeJob(this, beforeUriBuilder, job))); + nodes.addAll(dynamicNodes); + nodes.addAll(lifecycleNodes( + afterUriBuilder, + "After Job", + job.meta(), + () -> testPlan.target().afterJob(this, afterUriBuilder, job))); + return nodes; + } + + public List addLifecycle( + TestUri.Builder uriBuilder, + TestSuiteSpec testSuite, + TestRunEnv environment, + List dynamicNodes) { + + var beforeUriBuilder = + uriBuilder.clone().addSegment(TestUri.Segment.LIFECYCLE, "before-test-suite"); + var afterUriBuilder = + uriBuilder.clone().addSegment(TestUri.Segment.LIFECYCLE, "after-test-suite"); + + var nodes = new ArrayList(); + nodes.addAll(lifecycleNodes( + beforeUriBuilder, + "Before TestSuite", + testSuite.meta(), + () -> testPlan.target().beforeTestSuite(this, beforeUriBuilder, testSuite, environment))); + nodes.addAll(dynamicNodes); + nodes.addAll(lifecycleNodes( + afterUriBuilder, + "After TestSuite", + testSuite.meta(), + () -> testPlan.target().afterTestSuite(this, afterUriBuilder, testSuite, environment))); + return nodes; + } + +// private Optional containerIfPresent( +// TestUri.Builder uriBuilder, +// String namePrefix, +// TestSpecMeta meta, +// Optional dynamicNode) { +// +// return dynamicNode.map( +// node -> +// testPlanContainer( +// namePrefix + ": " + meta.name(), uriBuilder.build().uri(), List.of(node))); +// } +// +// private Stream streamIfPresent(Optional container) { +// return container.stream().flatMap(Stream::of); +// } + + private List lifecycleNodes( + TestUri.Builder uriBuilder, + String namePrefix, + TestSpecMeta meta, + Supplier> nodeSupplier) { + + var targetDynamicNode = nodeSupplier.get(); + + if (targetDynamicNode.isEmpty()) { + return Collections.emptyList(); + } + + var lifecycleContainer = testPlanContainer( + namePrefix + ": " + meta.name(), uriBuilder.build().uri(), List.of(targetDynamicNode.get())); + return List.of(lifecycleContainer); + } +} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testrun/TestRunEnv.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testrun/TestRunEnv.java index 48e95f8b5a..42f6aef769 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testrun/TestRunEnv.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testrun/TestRunEnv.java @@ -1,6 +1,5 @@ package io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.TestPlan; import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.TestSuiteSpec; import java.util.HashMap; import java.util.Map; @@ -30,19 +29,19 @@ public TestRunEnv(Map vars) { } public DynamicContainer testNode( - TestPlan testPlan, TestUri.Builder uriBuilder, TestSuiteSpec testSuite) { + TestNodeFactory testNodeFactory, TestUri.Builder uriBuilder, TestSuiteSpec testSuite) { var d = description(); uriBuilder.addSegment(TestUri.Segment.ENV, d); var desc = "TestEnv: %s ".formatted(d); var testExecutionCondition = new TestExecutionCondition.Default(desc); - var envNodes = testSuite.testNodesForEnvironment(testPlan, uriBuilder.clone(), this, testExecutionCondition).stream(); + var envNodes = testSuite.testNodesForEnvironment(testNodeFactory, uriBuilder.clone(), this, testExecutionCondition); - return DynamicContainer.dynamicContainer( + return testNodeFactory.testPlanContainer( desc, uriBuilder.build().uri(), - testPlan.addLifecycle(uriBuilder.clone(), testSuite, this, envNodes)); + testNodeFactory.addLifecycle(uriBuilder.clone(), testSuite, this, envNodes)); } private String description() { diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testrun/TestRunRequest.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testrun/TestRunRequest.java index 2f5526dc95..4dc8cf98c3 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testrun/TestRunRequest.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testrun/TestRunRequest.java @@ -1,10 +1,12 @@ package io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun; -import static org.junit.jupiter.api.DynamicContainer.dynamicContainer; + import io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions.TestAssertion; import io.stargate.sgv2.jsonapi.api.v1.vectorize.targets.Target; import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.TestCommand; + +import java.util.ArrayList; import java.util.List; import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Stream; @@ -25,12 +27,11 @@ public TestRunResponse execute() { return new TestRunResponse(this, apiRequest, apiRequest.execute()); } - public DynamicContainer testNodes(TestUri.Builder uriBuilder) { + public DynamicContainer testNodes(TestNodeFactory testNodeFactory, TestUri.Builder uriBuilder) { uriBuilder.addSegment(TestUri.Segment.REQUEST, name()); - Stream.Builder nodesBuilder = Stream.builder(); - ; + var nodes = new ArrayList(); AtomicReference atomicResponse = new AtomicReference<>(); @@ -43,7 +44,8 @@ public DynamicContainer testNodes(TestUri.Builder uriBuilder) { .addSegment(TestUri.Segment.COMMAND, testCommand.commandName().getApiName()), testExecutionCondition, () -> atomicResponse.set(execute())); - nodesBuilder.add(commandExecutable.testNode()); + + nodes.add(commandExecutable.testNode(testNodeFactory)); // tests for each assertion var assertionsUriBuilder = @@ -52,16 +54,16 @@ public DynamicContainer testNodes(TestUri.Builder uriBuilder) { testAssertions().stream() .map( testAssertion -> - testAssertion.testNodes(assertionsUriBuilder.clone(), atomicResponse, testExecutionCondition)) + testAssertion.testNodes(testNodeFactory, assertionsUriBuilder.clone(), atomicResponse, testExecutionCondition)) .toList(); // if we have assertion tests, put them in a container if (!assertionTests.isEmpty()) { - nodesBuilder.add( - dynamicContainer( - "Assertions", assertionsUriBuilder.build().uri(), assertionTests.stream())); + nodes.add( + testNodeFactory.testPlanContainer( + "Assertions", assertionsUriBuilder.build().uri(), assertionTests)); } - return dynamicContainer("Request: " + name, uriBuilder.build().uri(), nodesBuilder.build()); + return testNodeFactory.testPlanContainer("Request: " + name, uriBuilder.build().uri(), nodes); } } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/Job.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/Job.java index 24d372195b..6be04dec71 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/Job.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/Job.java @@ -1,8 +1,9 @@ package io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec; -import static org.junit.jupiter.api.DynamicContainer.dynamicContainer; + import io.stargate.sgv2.jsonapi.api.v1.vectorize.TestPlan; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestNodeFactory; import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestRunEnv; import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestUri; import java.util.ArrayList; @@ -22,22 +23,22 @@ public record Job( private static final Logger LOGGER = LoggerFactory.getLogger(Job.class); - public DynamicContainer testNode(TestPlan testPlan, TestUri.Builder uriBuilder) { + public DynamicContainer testNode(TestNodeFactory testNodeFactory, TestUri.Builder uriBuilder) { uriBuilder.addSegment(TestUri.Segment.JOB, meta.name()); var desc = "Job: %s ".formatted(meta.name()); - testPlan.updateJobForTarget(this); - var allEnvs = allEnvironments(testPlan); - var testSuiteNodes = - testSuites(testPlan) - .map(testSuite -> testSuite.testNode(testPlan, uriBuilder.clone(), allEnvs)); + testNodeFactory.testPlan().updateJobForTarget(this); + var allEnvs = allEnvironments(testNodeFactory.testPlan()); + var testSuiteNodes = testSuites(testNodeFactory.testPlan()) + .map(testSuite -> testSuite.testNode(testNodeFactory, uriBuilder.clone(), allEnvs)) + .toList(); - return dynamicContainer( + return testNodeFactory.testPlanContainer( desc, uriBuilder.build().uri(), - testPlan.addLifecycle(uriBuilder.clone(), this, testSuiteNodes)); + testNodeFactory.addLifecycle(uriBuilder.clone(), this, testSuiteNodes)); } public Stream testSuites(TestPlan testPlan) { diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/TestCase.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/TestCase.java index 93e03c0d6d..91aa2aa092 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/TestCase.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/TestCase.java @@ -2,12 +2,8 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.databind.node.ObjectNode; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.TestPlan; import io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions.TestAssertion; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestExecutionCondition; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestRunEnv; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestRunRequest; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestUri; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.*; import org.junit.jupiter.api.DynamicContainer; public record TestCase( @@ -17,17 +13,17 @@ public record TestCase( @JsonProperty("$include") String include) { public DynamicContainer testNodesForEnvironment( - TestPlan testPlan, TestUri.Builder uriBuilder, TestRunEnv testEnvironment, TestExecutionCondition testExecutionCondition) { + TestNodeFactory testNodeFactory, TestUri.Builder uriBuilder, TestRunEnv testEnvironment, TestExecutionCondition testExecutionCondition) { var testRequest = new TestRunRequest( "TestCase: name=%s".formatted(name, command.commandName()), command(), - testPlan.target(), + testNodeFactory.testPlan().target(), testEnvironment, - TestAssertion.buildAssertions(testPlan, this), + TestAssertion.buildAssertions(testNodeFactory.testPlan(), this), testExecutionCondition); - return testRequest.testNodes(uriBuilder); + return testRequest.testNodes(testNodeFactory, uriBuilder); } } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/TestSuiteSpec.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/TestSuiteSpec.java index 09a82c63ac..f7be335f5a 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/TestSuiteSpec.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/TestSuiteSpec.java @@ -1,13 +1,10 @@ package io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec; -import static org.junit.jupiter.api.DynamicContainer.dynamicContainer; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.*; + import io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions.TestAssertion; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestExecutionCondition; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestRunEnv; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestRunRequest; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestUri; +import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.*; + import java.util.ArrayList; import java.util.Collection; import java.util.List; @@ -19,21 +16,25 @@ public record TestSuiteSpec( implements TestSpec { public DynamicContainer testNode( - TestPlan testPlan, TestUri.Builder uriBuilder, List allEnvs) { + TestNodeFactory testNodeFactory, TestUri.Builder uriBuilder, List allEnvs) { uriBuilder.addSegment(TestUri.Segment.SUITE, meta().name()); var desc = "TestSuite: %s ".formatted(meta.name()); - - return dynamicContainer( + var childs = allEnvs.stream() + .map(testEnv -> testEnv.testNode(testNodeFactory, uriBuilder.clone(), this)) + .toList(); + return testNodeFactory.testPlanContainer( desc, uriBuilder.build().uri(), - allEnvs.stream().map(testEnv -> testEnv.testNode(testPlan, uriBuilder.clone(), this))); + childs); } - public Collection testNodesForEnvironment( - TestPlan testPlan, TestUri.Builder uriBuilder, TestRunEnv testEnvironment, TestExecutionCondition testExecutionCondition) { + public List testNodesForEnvironment( + TestNodeFactory testNodeFactory, TestUri.Builder uriBuilder, TestRunEnv testEnvironment, TestExecutionCondition testExecutionCondition) { + // not increasing the count of test nodes here, because this code is not actually making any + // test nodes, it is all in things we call, they do the increasing List nodes = new ArrayList<>(); int i = 1; @@ -44,18 +45,18 @@ public Collection testNodesForEnvironment( new TestRunRequest( "SetupRequest[%s]: %s".formatted(i++, setupCommand.commandName()), setupCommand, - testPlan.target(), + testNodeFactory.testPlan().target(), testEnvironment, - TestAssertion.forSuccess(testPlan, setupCommand), + TestAssertion.forSuccess(testNodeFactory.testPlan(), setupCommand), testExecutionCondition); - nodes.add(setupRequest.testNodes(setupUriBuilder.clone())); + nodes.add(setupRequest.testNodes(testNodeFactory, setupUriBuilder.clone())); } var testUriBuilder = uriBuilder.clone().addSegment(TestUri.Segment.STAGE, "test"); for (var testCase : tests()) { nodes.add( - testCase.testNodesForEnvironment(testPlan, testUriBuilder.clone(), testEnvironment, testExecutionCondition)); + testCase.testNodesForEnvironment(testNodeFactory, testUriBuilder.clone(), testEnvironment, testExecutionCondition)); } // NOTE: For Cleanup we use a condition that is always TRUE because we always want to try to run a cleanup task. @@ -66,11 +67,12 @@ public Collection testNodesForEnvironment( new TestRunRequest( "CleanupRequest[%s]: %s".formatted(i++, cleanupCommand.commandName()), cleanupCommand, - testPlan.target(), + testNodeFactory.testPlan().target(), testEnvironment, - TestAssertion.forSuccess(testPlan, cleanupCommand), + TestAssertion.forSuccess(testNodeFactory.testPlan(), cleanupCommand), alwaysTrueCondition); - nodes.add(cleanupRequest.testNodes(cleanupUriBuilder.clone())); + + nodes.add(cleanupRequest.testNodes(testNodeFactory, cleanupUriBuilder.clone())); } return nodes; diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/WorkflowSpec.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/WorkflowSpec.java index dfdb85a2ef..190dc71fe7 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/WorkflowSpec.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/WorkflowSpec.java @@ -1,20 +1,17 @@ package io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec; -import static org.junit.jupiter.api.DynamicContainer.dynamicContainer; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.TestPlan; + +import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestNodeFactory; import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestUri; import java.util.List; -import org.junit.jupiter.api.DynamicContainer; -public record WorkflowSpec(TestSpecMeta meta, List jobs) implements TestSpec { +import org.junit.jupiter.api.DynamicNode; - public DynamicContainer testNode(TestPlan testPlan, TestUri.Builder uriBuilder) { - return testNode(testPlan, uriBuilder, true); - } +public record WorkflowSpec(TestSpecMeta meta, List jobs) implements TestSpec { - public DynamicContainer testNode( - TestPlan testPlan, TestUri.Builder uriBuilder, boolean ignoreDisabled) { + public DynamicNode testNode( + TestNodeFactory testNodeFactory, TestUri.Builder uriBuilder, boolean ignoreDisabled) { uriBuilder.addSegment(TestUri.Segment.WORKFLOW, meta().name()); var desc = "Workflow: %s ".formatted(meta.name()); @@ -23,9 +20,11 @@ public DynamicContainer testNode( ignoreDisabled ? jobs().stream().filter(job -> !job.meta().tags().contains("disabled")) : jobs().stream(); - var jobNodes = testNodeJobs.map(job -> job.testNode(testPlan, uriBuilder.clone())); + var jobNodes = testNodeJobs + .map(job -> job.testNode(testNodeFactory, uriBuilder.clone())) + .toList(); - return dynamicContainer( - desc, uriBuilder.build().uri(), testPlan.addLifecycle(uriBuilder.clone(), this, jobNodes)); + return testNodeFactory.testPlanContainer( + desc, uriBuilder.build().uri(), testNodeFactory.addLifecycle(uriBuilder.clone(), this, jobNodes)); } } diff --git a/src/test/resources/integration-tests/vectorize/vectorize-header-workflow.json b/src/test/resources/integration-tests/vectorize/vectorize-header-workflow.json index d57d1d6871..f4f6492e9b 100644 --- a/src/test/resources/integration-tests/vectorize/vectorize-header-workflow.json +++ b/src/test/resources/integration-tests/vectorize/vectorize-header-workflow.json @@ -8,6 +8,7 @@ "meta": { "name": "open-ai-vectorize", "tags": [ + ] }, "fromEnvironment": { @@ -33,7 +34,6 @@ "meta": { "name": "voyageAI-vectorize", "tags": [ - ] }, "fromEnvironment": { @@ -61,7 +61,6 @@ "meta": { "name": "jinaAI-vectorize", "tags": [ - ] }, "fromEnvironment": { @@ -178,7 +177,7 @@ }, "matrix": { "MODEL": [ - "NV-Embed-QA-XX", + "NV-Embed-QA", "nvidia/nv-embedqa-e5-v5" ] }, From c624c30ad25fa46b2671a2ff89d489d471038b0a Mon Sep 17 00:00:00 2001 From: Aaron Morton Date: Thu, 23 Apr 2026 08:41:40 +1200 Subject: [PATCH 54/89] checking PROD env --- .github/workflows/test-bench-vectorize.yaml | 6 +++--- .../sgv2/jsonapi/api/v1/vectorize/messaging/APIRequest.java | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test-bench-vectorize.yaml b/.github/workflows/test-bench-vectorize.yaml index 610647ad91..26cab23579 100644 --- a/.github/workflows/test-bench-vectorize.yaml +++ b/.github/workflows/test-bench-vectorize.yaml @@ -25,7 +25,7 @@ env: jobs: setup: runs-on: ubuntu-latest - environment: ${{ github.event.inputs.environment || 'DEV' }} + environment: ${{ github.event.inputs.environment || 'PROD' }} outputs: matrix: ${{ steps.set-matrix.outputs.TEST_TARGETS }} @@ -42,10 +42,10 @@ jobs: # runs unit tests build: - name: Test Bench- ENV-${{ github.event.inputs.environment || 'DEV' }} Target- ${{ matrix.name }} + name: Test Bench- ENV-${{ github.event.inputs.environment || 'PROD' }} Target- ${{ matrix.name }} needs: setup runs-on: ubuntu-latest - environment: ${{ github.event.inputs.environment || 'DEV' }} + environment: ${{ github.event.inputs.environment || 'PROD' }} timeout-minutes: 120 diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/messaging/APIRequest.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/messaging/APIRequest.java index fccc0db137..b9b85e9821 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/messaging/APIRequest.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/messaging/APIRequest.java @@ -75,7 +75,7 @@ private ValidatableResponse executeRequest(RequestSpecification requestSpec) { throw new IllegalArgumentException("Do not know how to execute command: " + commandName); } - return response.then().log().all(); + return response.then().log().status().and().log().body(); } protected Map getHeaders() { From 04a00ffa498bccc93ec9ff4d94390b3c7ef7c160 Mon Sep 17 00:00:00 2001 From: Aaron Morton Date: Thu, 23 Apr 2026 14:24:17 +1200 Subject: [PATCH 55/89] move to dev, trying summary --- .github/workflows/test-bench-vectorize.yaml | 26 ++++++++++++++++--- .../test-plans/test-plan-astra-vectorize.yaml | 2 +- 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test-bench-vectorize.yaml b/.github/workflows/test-bench-vectorize.yaml index 26cab23579..d48694ca20 100644 --- a/.github/workflows/test-bench-vectorize.yaml +++ b/.github/workflows/test-bench-vectorize.yaml @@ -25,7 +25,7 @@ env: jobs: setup: runs-on: ubuntu-latest - environment: ${{ github.event.inputs.environment || 'PROD' }} + environment: ${{ github.event.inputs.environment || 'DEV' }} outputs: matrix: ${{ steps.set-matrix.outputs.TEST_TARGETS }} @@ -40,12 +40,32 @@ jobs: echo "EOF" } >> "$GITHUB_OUTPUT" + - id: write-setup-summary + name: Write setup summary + env: + TEST_TARGETS: ${{ vars.TEST_TARGETS }} + run: | + FORMATTED_TARGETS=$(echo "$TEST_TARGETS" | jq '.') + + cat >> $GITHUB_STEP_SUMMARY << 'EOF' + ## Test Run Setup + + **Environment:** ${{ github.event.inputs.environment || 'DEV' }} + **Triggered by:** ${{ github.actor }} + + ### Test Targets + EOF + + echo '```json' >> $GITHUB_STEP_SUMMARY + echo "$FORMATTED_TARGETS" >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + # runs unit tests build: - name: Test Bench- ENV-${{ github.event.inputs.environment || 'PROD' }} Target- ${{ matrix.name }} + name: Test Bench- ENV-${{ github.event.inputs.environment || 'DEV' }} Target- ${{ matrix.name }} needs: setup runs-on: ubuntu-latest - environment: ${{ github.event.inputs.environment || 'PROD' }} + environment: ${{ github.event.inputs.environment || 'DEV' }} timeout-minutes: 120 diff --git a/src/test/resources/integration-tests/test-plans/test-plan-astra-vectorize.yaml b/src/test/resources/integration-tests/test-plans/test-plan-astra-vectorize.yaml index d571a3a4f6..937dd2f67c 100644 --- a/src/test/resources/integration-tests/test-plans/test-plan-astra-vectorize.yaml +++ b/src/test/resources/integration-tests/test-plans/test-plan-astra-vectorize.yaml @@ -9,4 +9,4 @@ customTarget: workflows: - vectorize-header-workflow - vectorize-shared-workflow -ignoreDisabled: true \ No newline at end of file +ignoreDisabled: true From 06c669b8c7934d1f0a2ee787b3eefa4521552cf0 Mon Sep 17 00:00:00 2001 From: Aaron Morton Date: Thu, 23 Apr 2026 14:41:17 +1200 Subject: [PATCH 56/89] adding test report file --- .github/workflows/test-bench-vectorize.yaml | 8 ++- .../reporting/TestBenchConsoleWriter.java | 52 ++++++++++++++----- .../test-plans/test-plan-astra-vectorize.yaml | 2 +- .../vectorize/vectorize-header-workflow.json | 12 ++--- 4 files changed, 54 insertions(+), 20 deletions(-) diff --git a/.github/workflows/test-bench-vectorize.yaml b/.github/workflows/test-bench-vectorize.yaml index d48694ca20..529d2f7b35 100644 --- a/.github/workflows/test-bench-vectorize.yaml +++ b/.github/workflows/test-bench-vectorize.yaml @@ -134,4 +134,10 @@ jobs: # reducing extra output with the -Dfailsafe.printSummary=false -Dfailsafe.trimStackTrace=true run: | - ./mvnw -B -ntp verify -Dfmt.skip -DskipUnitTests -Dfailsafe.printSummary=false -Dfailsafe.trimStackTrace=true -Dit.test=TestBenchTests \ No newline at end of file + ./mvnw -B -ntp verify \ + -Dfmt.skip \ + -DskipUnitTests \ + -Dfailsafe.printSummary=false \ + -Dfailsafe.trimStackTrace=true \ + -Dit.test=TestBenchTests \ + -Dtest-bench-report-path=test-bench-report-${{ matrix.name }}.md \ No newline at end of file diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/reporting/TestBenchConsoleWriter.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/reporting/TestBenchConsoleWriter.java index 9f8c3c5bdc..67e96e6fb5 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/reporting/TestBenchConsoleWriter.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/reporting/TestBenchConsoleWriter.java @@ -3,6 +3,10 @@ import static org.apache.maven.surefire.shared.utils.logging.MessageUtils.buffer; import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestUri; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.Objects; import org.apache.maven.plugin.surefire.report.Theme; import org.apache.maven.surefire.shared.utils.logging.MessageBuilder; @@ -85,19 +89,43 @@ public void testStarted(int totalTestCount, int startedTestCount, DynamicTreeLis */ public void allTestsFinished(DynamicTreeListener.TestReportingTracker rootTracker) { - var buffer = buffer(); + var reportBuffer = buffer(); + writeCompletedSummary(reportBuffer, rootTracker, true, false); + var testReport = reportBuffer.toString(); + + if (LOGGER.isInfoEnabled()){ + var logLineBuffer = buffer(); + logLineBuffer + .newline() + .a(theme.dash().repeat(20)) + .newline() + .a("Test Bench Results") + .newline() + .a(theme.dash().repeat(20)) + .newline(); + logLineBuffer.a(testReport); + LOGGER.info(logLineBuffer.toString()); + } - buffer - .newline() - .a(theme.dash().repeat(20)) - .newline() - .a("Test Bench Results") - .newline() - .a(theme.dash().repeat(20)) - .newline(); - - writeCompletedSummary(buffer, rootTracker, true, false); - LOGGER.info(buffer.toString()); + var reportFilePath = System.getProperty("test-bench-report-path"); + if (reportFilePath != null) { + LOGGER.info("Writing report file to: {}", reportFilePath); + + // report is a markdown + var markdownReport = """ + ## %s + + ``` + %s + ``` + """.formatted(rootTracker.identifier().getDisplayName(), testReport); + + try { + Files.writeString(Path.of(reportFilePath), markdownReport); + } catch (IOException e) { + throw new RuntimeException(e); + } + } } /** diff --git a/src/test/resources/integration-tests/test-plans/test-plan-astra-vectorize.yaml b/src/test/resources/integration-tests/test-plans/test-plan-astra-vectorize.yaml index 937dd2f67c..4c1e3cbbff 100644 --- a/src/test/resources/integration-tests/test-plans/test-plan-astra-vectorize.yaml +++ b/src/test/resources/integration-tests/test-plans/test-plan-astra-vectorize.yaml @@ -8,5 +8,5 @@ customTarget: basePath: /api/json/v1 workflows: - vectorize-header-workflow - - vectorize-shared-workflow +# - vectorize-shared-workflow ignoreDisabled: true diff --git a/src/test/resources/integration-tests/vectorize/vectorize-header-workflow.json b/src/test/resources/integration-tests/vectorize/vectorize-header-workflow.json index f4f6492e9b..aebf54617c 100644 --- a/src/test/resources/integration-tests/vectorize/vectorize-header-workflow.json +++ b/src/test/resources/integration-tests/vectorize/vectorize-header-workflow.json @@ -8,7 +8,7 @@ "meta": { "name": "open-ai-vectorize", "tags": [ - +"disabled" ] }, "fromEnvironment": { @@ -33,7 +33,7 @@ { "meta": { "name": "voyageAI-vectorize", - "tags": [ + "tags": ["disabled" ] }, "fromEnvironment": { @@ -60,7 +60,7 @@ { "meta": { "name": "jinaAI-vectorize", - "tags": [ + "tags": ["disabled" ] }, "fromEnvironment": { @@ -87,7 +87,7 @@ { "meta": { "name": "huggingface-non-dedicated-vectorize", - "tags": [ + "tags": ["disabled" ] }, @@ -116,7 +116,7 @@ { "meta": { "name": "mistral-vectorize", - "tags": [ + "tags": ["disabled" ] }, @@ -140,7 +140,7 @@ { "meta": { "name": "upstageAI-vectorize", - "tags": [ + "tags": ["disabled" ] }, From af4fcf881bac86e16c7841d35c35d3cdac7dc5de Mon Sep 17 00:00:00 2001 From: Aaron Morton Date: Thu, 23 Apr 2026 14:52:18 +1200 Subject: [PATCH 57/89] push report to summary var --- .github/workflows/test-bench-vectorize.yaml | 10 +++++++++- .../vectorize/vectorize-header-workflow.json | 2 +- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-bench-vectorize.yaml b/.github/workflows/test-bench-vectorize.yaml index 529d2f7b35..d5077cc131 100644 --- a/.github/workflows/test-bench-vectorize.yaml +++ b/.github/workflows/test-bench-vectorize.yaml @@ -69,6 +69,9 @@ jobs: timeout-minutes: 120 + env: + TEST_BENCH_REPORT: test-bench-report-${{ matrix.name }}.md + strategy: # do not fail fast, we want to run on all the different target dbs fail-fast: false @@ -140,4 +143,9 @@ jobs: -Dfailsafe.printSummary=false \ -Dfailsafe.trimStackTrace=true \ -Dit.test=TestBenchTests \ - -Dtest-bench-report-path=test-bench-report-${{ matrix.name }}.md \ No newline at end of file + -Dtest-bench-report-path=$TEST_BENCH_REPORT + + - name: Write summary + if: always() + run: | + cat "$TEST_BENCH_REPORT" >> $GITHUB_STEP_SUMMARY \ No newline at end of file diff --git a/src/test/resources/integration-tests/vectorize/vectorize-header-workflow.json b/src/test/resources/integration-tests/vectorize/vectorize-header-workflow.json index aebf54617c..734f613ad0 100644 --- a/src/test/resources/integration-tests/vectorize/vectorize-header-workflow.json +++ b/src/test/resources/integration-tests/vectorize/vectorize-header-workflow.json @@ -178,7 +178,7 @@ "matrix": { "MODEL": [ "NV-Embed-QA", - "nvidia/nv-embedqa-e5-v5" + "nvidia/nv-embedqa-e5-v5XX" ] }, "tests": [ From bbe336fc0ca4b799a51c622af21bcbdbbef0842b Mon Sep 17 00:00:00 2001 From: Aaron Morton Date: Thu, 23 Apr 2026 15:05:32 +1200 Subject: [PATCH 58/89] disable jansi --- .github/workflows/test-bench-vectorize.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/test-bench-vectorize.yaml b/.github/workflows/test-bench-vectorize.yaml index d5077cc131..f861b83a60 100644 --- a/.github/workflows/test-bench-vectorize.yaml +++ b/.github/workflows/test-bench-vectorize.yaml @@ -70,7 +70,10 @@ jobs: timeout-minutes: 120 env: + # Path to where the summary report will be written so we can include in the GH summary TEST_BENCH_REPORT: test-bench-report-${{ matrix.name }}.md + # Disable JANSI in the logging and report writing, GH summary does render them properly. + TERM: dumb strategy: # do not fail fast, we want to run on all the different target dbs From 17978690a21a5cea1eb1332145091b35203992c9 Mon Sep 17 00:00:00 2001 From: Aaron Morton Date: Thu, 23 Apr 2026 15:18:34 +1200 Subject: [PATCH 59/89] try again disable ansi --- .github/workflows/test-bench-vectorize.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-bench-vectorize.yaml b/.github/workflows/test-bench-vectorize.yaml index f861b83a60..58d77ff251 100644 --- a/.github/workflows/test-bench-vectorize.yaml +++ b/.github/workflows/test-bench-vectorize.yaml @@ -72,8 +72,6 @@ jobs: env: # Path to where the summary report will be written so we can include in the GH summary TEST_BENCH_REPORT: test-bench-report-${{ matrix.name }}.md - # Disable JANSI in the logging and report writing, GH summary does render them properly. - TERM: dumb strategy: # do not fail fast, we want to run on all the different target dbs @@ -139,10 +137,12 @@ jobs: VOYAGE_AI_KEY: ${{ secrets.VOYAGE_AI_KEY }} # reducing extra output with the -Dfailsafe.printSummary=false -Dfailsafe.trimStackTrace=true + # -Dorg.fusesource.jansi.Ansi.disable=true - disable ansi because GH action summary does not handle run: | ./mvnw -B -ntp verify \ -Dfmt.skip \ -DskipUnitTests \ + -Dorg.fusesource.jansi.Ansi.disable=true \ -Dfailsafe.printSummary=false \ -Dfailsafe.trimStackTrace=true \ -Dit.test=TestBenchTests \ From f6865758c704a5f2727b1012845097346851bdb6 Mon Sep 17 00:00:00 2001 From: Aaron Morton Date: Thu, 23 Apr 2026 15:23:34 +1200 Subject: [PATCH 60/89] enable all dev tests again --- .github/workflows/test-bench-vectorize.yaml | 2 +- .../test-plans/test-plan-astra-vectorize.yaml | 2 +- .../vectorize/vectorize-header-workflow.json | 13 ++++++------- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/.github/workflows/test-bench-vectorize.yaml b/.github/workflows/test-bench-vectorize.yaml index 58d77ff251..6f82e079dd 100644 --- a/.github/workflows/test-bench-vectorize.yaml +++ b/.github/workflows/test-bench-vectorize.yaml @@ -137,7 +137,7 @@ jobs: VOYAGE_AI_KEY: ${{ secrets.VOYAGE_AI_KEY }} # reducing extra output with the -Dfailsafe.printSummary=false -Dfailsafe.trimStackTrace=true - # -Dorg.fusesource.jansi.Ansi.disable=true - disable ansi because GH action summary does not handle + # -Dorg.fusesource.jansi.Ansi.disable=true - disable ansi because GH action summary does not handle run: | ./mvnw -B -ntp verify \ -Dfmt.skip \ diff --git a/src/test/resources/integration-tests/test-plans/test-plan-astra-vectorize.yaml b/src/test/resources/integration-tests/test-plans/test-plan-astra-vectorize.yaml index 4c1e3cbbff..937dd2f67c 100644 --- a/src/test/resources/integration-tests/test-plans/test-plan-astra-vectorize.yaml +++ b/src/test/resources/integration-tests/test-plans/test-plan-astra-vectorize.yaml @@ -8,5 +8,5 @@ customTarget: basePath: /api/json/v1 workflows: - vectorize-header-workflow -# - vectorize-shared-workflow + - vectorize-shared-workflow ignoreDisabled: true diff --git a/src/test/resources/integration-tests/vectorize/vectorize-header-workflow.json b/src/test/resources/integration-tests/vectorize/vectorize-header-workflow.json index 734f613ad0..2aa573bf6d 100644 --- a/src/test/resources/integration-tests/vectorize/vectorize-header-workflow.json +++ b/src/test/resources/integration-tests/vectorize/vectorize-header-workflow.json @@ -8,7 +8,6 @@ "meta": { "name": "open-ai-vectorize", "tags": [ -"disabled" ] }, "fromEnvironment": { @@ -33,7 +32,7 @@ { "meta": { "name": "voyageAI-vectorize", - "tags": ["disabled" + "tags": [ ] }, "fromEnvironment": { @@ -60,7 +59,7 @@ { "meta": { "name": "jinaAI-vectorize", - "tags": ["disabled" + "tags": [ ] }, "fromEnvironment": { @@ -87,7 +86,7 @@ { "meta": { "name": "huggingface-non-dedicated-vectorize", - "tags": ["disabled" + "tags": [ ] }, @@ -116,7 +115,7 @@ { "meta": { "name": "mistral-vectorize", - "tags": ["disabled" + "tags": [ ] }, @@ -140,7 +139,7 @@ { "meta": { "name": "upstageAI-vectorize", - "tags": ["disabled" + "tags": [ ] }, @@ -178,7 +177,7 @@ "matrix": { "MODEL": [ "NV-Embed-QA", - "nvidia/nv-embedqa-e5-v5XX" + "nvidia/nv-embedqa-e5-v5" ] }, "tests": [ From aa685a37cfdb313ea924052e0f3b8d6caae4e225 Mon Sep 17 00:00:00 2001 From: Aaron Morton Date: Thu, 23 Apr 2026 15:54:18 +1200 Subject: [PATCH 61/89] testing with PROD again --- .github/workflows/test-bench-vectorize.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test-bench-vectorize.yaml b/.github/workflows/test-bench-vectorize.yaml index 6f82e079dd..944136d27a 100644 --- a/.github/workflows/test-bench-vectorize.yaml +++ b/.github/workflows/test-bench-vectorize.yaml @@ -25,7 +25,7 @@ env: jobs: setup: runs-on: ubuntu-latest - environment: ${{ github.event.inputs.environment || 'DEV' }} + environment: ${{ github.event.inputs.environment || 'PROD' }} outputs: matrix: ${{ steps.set-matrix.outputs.TEST_TARGETS }} @@ -50,7 +50,7 @@ jobs: cat >> $GITHUB_STEP_SUMMARY << 'EOF' ## Test Run Setup - **Environment:** ${{ github.event.inputs.environment || 'DEV' }} + **Environment:** ${{ github.event.inputs.environment || 'PROD' }} **Triggered by:** ${{ github.actor }} ### Test Targets @@ -62,10 +62,10 @@ jobs: # runs unit tests build: - name: Test Bench- ENV-${{ github.event.inputs.environment || 'DEV' }} Target- ${{ matrix.name }} + name: Test Bench- ENV-${{ github.event.inputs.environment || 'PROD' }} Target- ${{ matrix.name }} needs: setup runs-on: ubuntu-latest - environment: ${{ github.event.inputs.environment || 'DEV' }} + environment: ${{ github.event.inputs.environment || 'PROD' }} timeout-minutes: 120 From 45318f9f870f1b8a9dc8f07227d4dee296bea25b Mon Sep 17 00:00:00 2001 From: Aaron Morton Date: Thu, 23 Apr 2026 16:02:14 +1200 Subject: [PATCH 62/89] sanitize report name --- .github/workflows/test-bench-vectorize.yaml | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test-bench-vectorize.yaml b/.github/workflows/test-bench-vectorize.yaml index 944136d27a..e66f2b6f7b 100644 --- a/.github/workflows/test-bench-vectorize.yaml +++ b/.github/workflows/test-bench-vectorize.yaml @@ -69,10 +69,6 @@ jobs: timeout-minutes: 120 - env: - # Path to where the summary report will be written so we can include in the GH summary - TEST_BENCH_REPORT: test-bench-report-${{ matrix.name }}.md - strategy: # do not fail fast, we want to run on all the different target dbs fail-fast: false @@ -123,6 +119,14 @@ jobs: EOF + + - name: Set env + # Sanitize the name of the target, it could be things like "smoketest-prod-gcp-us-east4 (astra-serverless-prod-49)" + # TEST_BENCH_REPORT path to where the summary report will be written so we can include in the GH summary + run: | + SAFE_NAME=$(echo "${{ matrix.name }}" | sed 's/[^a-zA-Z0-9._-]/_/g') + echo "TEST_BENCH_REPORT=test-bench-report-${SAFE_NAME}.md" >> $GITHUB_ENV + - name: Build & Test env: TEST_PLAN_FILE: classpath:integration-tests/test-plans/test-plan-astra-vectorize.yaml From d8686b120b077244b22653cdccd6fc1a189420cb Mon Sep 17 00:00:00 2001 From: Aaron Morton Date: Fri, 24 Apr 2026 08:29:22 +1200 Subject: [PATCH 63/89] improve report with sections --- .github/workflows/test-bench-vectorize.yaml | 8 ++++---- .../v1/vectorize/reporting/TestBenchConsoleWriter.java | 8 ++++++-- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/.github/workflows/test-bench-vectorize.yaml b/.github/workflows/test-bench-vectorize.yaml index e66f2b6f7b..477d64c4c6 100644 --- a/.github/workflows/test-bench-vectorize.yaml +++ b/.github/workflows/test-bench-vectorize.yaml @@ -25,7 +25,7 @@ env: jobs: setup: runs-on: ubuntu-latest - environment: ${{ github.event.inputs.environment || 'PROD' }} + environment: ${{ github.event.inputs.environment || 'DEV' }} outputs: matrix: ${{ steps.set-matrix.outputs.TEST_TARGETS }} @@ -50,7 +50,7 @@ jobs: cat >> $GITHUB_STEP_SUMMARY << 'EOF' ## Test Run Setup - **Environment:** ${{ github.event.inputs.environment || 'PROD' }} + **Environment:** ${{ github.event.inputs.environment || 'DEV' }} **Triggered by:** ${{ github.actor }} ### Test Targets @@ -62,10 +62,10 @@ jobs: # runs unit tests build: - name: Test Bench- ENV-${{ github.event.inputs.environment || 'PROD' }} Target- ${{ matrix.name }} + name: Test Bench- ENV-${{ github.event.inputs.environment || 'DEV' }} Target- ${{ matrix.name }} needs: setup runs-on: ubuntu-latest - environment: ${{ github.event.inputs.environment || 'PROD' }} + environment: ${{ github.event.inputs.environment || 'DEV' }} timeout-minutes: 120 diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/reporting/TestBenchConsoleWriter.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/reporting/TestBenchConsoleWriter.java index 67e96e6fb5..a814746499 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/reporting/TestBenchConsoleWriter.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/reporting/TestBenchConsoleWriter.java @@ -112,12 +112,16 @@ public void allTestsFinished(DynamicTreeListener.TestReportingTracker rootTracke LOGGER.info("Writing report file to: {}", reportFilePath); // report is a markdown + // GitHub collapsable sections + // https://docs.github.com/en/get-started/writing-on-github/working-with-advanced-formatting/organizing-information-with-collapsed-sections var markdownReport = """ ## %s - +
+ Test Bench Summary ``` %s ``` +
""".formatted(rootTracker.identifier().getDisplayName(), testReport); try { @@ -209,6 +213,6 @@ private void writeStatsLine(MessageBuilder buffer, DynamicTreeListener.TestRepor buffer .a("Successful: ").a( tracker.stats().successful()).a(", ") .a("Failures: ").a( tracker.stats().failures()).a(", ") - .a("Aborted: ").a( tracker.stats().aborted()).a(", "); + .a("Aborted: ").a( tracker.stats().aborted()); } } From f6bcff4a011a0d1643e4278d23d69af66bbc8f02 Mon Sep 17 00:00:00 2001 From: Aaron Morton Date: Fri, 24 Apr 2026 11:13:26 +1200 Subject: [PATCH 64/89] summary report tweaks --- .../reporting/TestBenchConsoleWriter.java | 3 +++ .../v1/vectorize/testrun/TestNodeFactory.java | 20 +++++++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/reporting/TestBenchConsoleWriter.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/reporting/TestBenchConsoleWriter.java index a814746499..fc6383295f 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/reporting/TestBenchConsoleWriter.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/reporting/TestBenchConsoleWriter.java @@ -117,10 +117,13 @@ public void allTestsFinished(DynamicTreeListener.TestReportingTracker rootTracke var markdownReport = """ ## %s
+ Test Bench Summary + ``` %s ``` +
""".formatted(rootTracker.identifier().getDisplayName(), testReport); diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testrun/TestNodeFactory.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testrun/TestNodeFactory.java index 0de3ac7620..3457e3ca35 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testrun/TestNodeFactory.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testrun/TestNodeFactory.java @@ -167,4 +167,24 @@ private List lifecycleNodes( namePrefix + ": " + meta.name(), uriBuilder.build().uri(), List.of(targetDynamicNode.get())); return List.of(lifecycleContainer); } + + public static class NodeCode { + + private static final char[] ALPHABET = + "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789".toCharArray(); + private static final int BASE = ALPHABET.length; // 62 + private static final int LENGTH = 3; + + private int counter = 0; + + public String next() { + int n = counter++; + char[] code = new char[LENGTH]; + for (int i = LENGTH - 1; i >= 0; i--) { + code[i] = ALPHABET[n % BASE]; + n /= BASE; + } + return new String(code); + } + } } From 1dbe28c7fe1d8bae4c04fa0fb9464756ed1e7586 Mon Sep 17 00:00:00 2001 From: Aaron Morton Date: Fri, 24 Apr 2026 11:22:00 +1200 Subject: [PATCH 65/89] improve summary report --- .../jsonapi/api/v1/vectorize/TestPlan.java | 5 +- .../reporting/TestBenchConsoleWriter.java | 62 +++++++++++-------- 2 files changed, 37 insertions(+), 30 deletions(-) diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestPlan.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestPlan.java index 8cd0acfed9..98bb657c6d 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestPlan.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestPlan.java @@ -105,11 +105,10 @@ public Stream selectedWorkflows() { public TestPlanNodeTree testNode() { var desc = - "TestPlan: %s on %s workflows %s" + "TestPlan: %s on %s" .formatted( target.configuration().name(), - target.configuration().backend(), - workflows.isEmpty() ? "" : String.join(", ", workflows)); + target.configuration().backend()); var uriBuilder = TestUri.builder(TestUri.Scheme.DATAAPI) diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/reporting/TestBenchConsoleWriter.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/reporting/TestBenchConsoleWriter.java index fc6383295f..4d7d657143 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/reporting/TestBenchConsoleWriter.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/reporting/TestBenchConsoleWriter.java @@ -111,11 +111,19 @@ public void allTestsFinished(DynamicTreeListener.TestReportingTracker rootTracke if (reportFilePath != null) { LOGGER.info("Writing report file to: {}", reportFilePath); + // this is the info for the top node, so peeps know if they want to go down into the report. + var testPlanNodeDesc = buffer(); + writeTestDesc(testPlanNodeDesc, rootTracker); + testPlanNodeDesc.newline(); + // report is a markdown // GitHub collapsable sections // https://docs.github.com/en/get-started/writing-on-github/working-with-advanced-formatting/organizing-information-with-collapsed-sections var markdownReport = """ ## %s + + %s +
Test Bench Summary @@ -125,7 +133,7 @@ public void allTestsFinished(DynamicTreeListener.TestReportingTracker rootTracke ```
- """.formatted(rootTracker.identifier().getDisplayName(), testReport); + """.formatted(rootTracker.identifier().getDisplayName(), testPlanNodeDesc.toString(), testReport); try { Files.writeString(Path.of(reportFilePath), markdownReport); @@ -158,13 +166,25 @@ private void writeCompletedSummary( buffer.a(theme.blank().repeat(tracker.depth() - 1)).a(theme.entry()); } - var timingInfo = tracker.stats() == null ? - "" - : - " - %s s".formatted(tracker.stats().elapsedMillis() / 1000); + writeTestDesc(buffer, tracker); + buffer.newline(); + + // If we have a TestEnv then we want to write out the summary of results for it, otherwise + // descend until we get one + // OF if there are FAILURES then we descend, these are tests that ran but assertion failed. + // we do not descend for aborted, these are tests that did not run because of previous failure. + for (var child : tracker.children()){ + var hasFailures = child.stats() != null && child.stats().failures() > 0; + if (!child.runUri().leafType().descendantOf(TestUri.Segment.ENV) || hasFailures || parentFailures) { + writeCompletedSummary(buffer, child, false, hasFailures); + } + } + } + + private void writeTestDesc(MessageBuilder buffer, DynamicTreeListener.TestReportingTracker tracker) { // Icon for success for failure, - // if we have stats then this is a container we we use that status, otherwise we use the JUNIT execution status + // if we have stats then this is a container we use that status, otherwise we use the JUNIT execution status if (tracker.stats() == null){ switch (tracker.junitStatus()) { case SUCCESSFUL -> buffer.a(theme.successful()); @@ -179,9 +199,13 @@ private void writeCompletedSummary( } } - // The name of the container or test - buffer.strong(tracker.identifier().getDisplayName()).a(timingInfo); + buffer.strong(tracker.identifier().getDisplayName()); + + // timing info if available + if (tracker.stats() != null){ + buffer.a(" - %s s".formatted(tracker.stats().elapsedMillis() / 1000); + } // if we have stats write a stats line, these are aggregate for all things below. // alternative is to only print them for Test Environment these are lines like @@ -190,29 +214,13 @@ private void writeCompletedSummary( if (tracker.stats() != null) { buffer.a(theme.blank()); - writeStatsLine(buffer, tracker); + writeTestStats(buffer, tracker); } - buffer.newline(); - // we do not want to descent below the Test Envirinment unless there is a failure, otherwise there is too - // much output. Tend ENV line looks like - // TestEnv: [MODEL=text-embedding-3-small, PROVIDER=openai] - 26 s - - - - // If we have a TestEnv then we want to write out the summary of results for it, otherwise - // descend until we get one - // OF if there are FAILURES then we descend, these are tests that ran but assertion failed. - // we do not descend for aborted, these are tests that did not run because of previous failure. - for (var child : tracker.children()){ - var hasFailures = child.stats() != null && child.stats().failures() > 0; - if (!child.runUri().leafType().descendantOf(TestUri.Segment.ENV) || hasFailures || parentFailures) { - writeCompletedSummary(buffer, child, false, hasFailures); - } - } + // NOTE: does not add new line, caller should } - private void writeStatsLine(MessageBuilder buffer, DynamicTreeListener.TestReportingTracker tracker) { + private void writeTestStats(MessageBuilder buffer, DynamicTreeListener.TestReportingTracker tracker) { buffer .a("Successful: ").a( tracker.stats().successful()).a(", ") .a("Failures: ").a( tracker.stats().failures()).a(", ") From d66c57c8fa8bf59e54cd26c175a8b43922d606a1 Mon Sep 17 00:00:00 2001 From: Aaron Morton Date: Fri, 24 Apr 2026 11:30:02 +1200 Subject: [PATCH 66/89] add node code, reduce tests for now --- .../reporting/TestBenchConsoleWriter.java | 2 +- .../v1/vectorize/testrun/TestNodeFactory.java | 27 +++++++------------ .../test-plans/test-plan-astra-vectorize.yaml | 2 +- .../vectorize/vectorize-header-workflow.json | 12 ++++----- 4 files changed, 17 insertions(+), 26 deletions(-) diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/reporting/TestBenchConsoleWriter.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/reporting/TestBenchConsoleWriter.java index 4d7d657143..2eb8d0cc79 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/reporting/TestBenchConsoleWriter.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/reporting/TestBenchConsoleWriter.java @@ -204,7 +204,7 @@ private void writeTestDesc(MessageBuilder buffer, DynamicTreeListener.TestReport // timing info if available if (tracker.stats() != null){ - buffer.a(" - %s s".formatted(tracker.stats().elapsedMillis() / 1000); + buffer.a(" - %s s".formatted(tracker.stats().elapsedMillis() / 1000)); } // if we have stats write a stats line, these are aggregate for all things below. diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testrun/TestNodeFactory.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testrun/TestNodeFactory.java index 3457e3ca35..26cff5ed77 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testrun/TestNodeFactory.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testrun/TestNodeFactory.java @@ -28,6 +28,7 @@ */ public class TestNodeFactory { + private final NodeCode nodeCode = new NodeCode(); private final TestPlan testPlan; private int totalNodeCount = 0; @@ -54,13 +55,17 @@ public int testNodeCount(){ public DynamicContainer testPlanContainer(String displayName, URI testSourceUri, List dynamicNodes){ totalNodeCount++; - return DynamicContainer.dynamicContainer(displayName, testSourceUri, dynamicNodes.stream()); + return DynamicContainer.dynamicContainer(appendNodeCode(displayName), testSourceUri, dynamicNodes.stream()); } public DynamicTest testPlanTest(String description, URI uri, Executable executable){ totalNodeCount++; - return DynamicTest.dynamicTest(description, uri, executable); + return DynamicTest.dynamicTest(appendNodeCode(description), uri, executable); + } + + private String appendNodeCode(String displayName){ + return "%s (node:%s)".formatted(displayName, nodeCode.next()); } public List addLifecycle( @@ -135,22 +140,6 @@ public List addLifecycle( return nodes; } -// private Optional containerIfPresent( -// TestUri.Builder uriBuilder, -// String namePrefix, -// TestSpecMeta meta, -// Optional dynamicNode) { -// -// return dynamicNode.map( -// node -> -// testPlanContainer( -// namePrefix + ": " + meta.name(), uriBuilder.build().uri(), List.of(node))); -// } -// -// private Stream streamIfPresent(Optional container) { -// return container.stream().flatMap(Stream::of); -// } - private List lifecycleNodes( TestUri.Builder uriBuilder, String namePrefix, @@ -173,6 +162,8 @@ public static class NodeCode { private static final char[] ALPHABET = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789".toCharArray(); private static final int BASE = ALPHABET.length; // 62 + + // 3 characters of base 62 coding above gives 62^3 = 238,328 private static final int LENGTH = 3; private int counter = 0; diff --git a/src/test/resources/integration-tests/test-plans/test-plan-astra-vectorize.yaml b/src/test/resources/integration-tests/test-plans/test-plan-astra-vectorize.yaml index 937dd2f67c..4c1e3cbbff 100644 --- a/src/test/resources/integration-tests/test-plans/test-plan-astra-vectorize.yaml +++ b/src/test/resources/integration-tests/test-plans/test-plan-astra-vectorize.yaml @@ -8,5 +8,5 @@ customTarget: basePath: /api/json/v1 workflows: - vectorize-header-workflow - - vectorize-shared-workflow +# - vectorize-shared-workflow ignoreDisabled: true diff --git a/src/test/resources/integration-tests/vectorize/vectorize-header-workflow.json b/src/test/resources/integration-tests/vectorize/vectorize-header-workflow.json index 2aa573bf6d..efde4c1d23 100644 --- a/src/test/resources/integration-tests/vectorize/vectorize-header-workflow.json +++ b/src/test/resources/integration-tests/vectorize/vectorize-header-workflow.json @@ -7,7 +7,7 @@ { "meta": { "name": "open-ai-vectorize", - "tags": [ + "tags": ["disabled" ] }, "fromEnvironment": { @@ -32,7 +32,7 @@ { "meta": { "name": "voyageAI-vectorize", - "tags": [ + "tags": ["disabled" ] }, "fromEnvironment": { @@ -59,7 +59,7 @@ { "meta": { "name": "jinaAI-vectorize", - "tags": [ + "tags": ["disabled" ] }, "fromEnvironment": { @@ -86,7 +86,7 @@ { "meta": { "name": "huggingface-non-dedicated-vectorize", - "tags": [ + "tags": ["disabled" ] }, @@ -116,7 +116,7 @@ "meta": { "name": "mistral-vectorize", "tags": [ - + "disabled" ] }, "fromEnvironment": { @@ -140,7 +140,7 @@ "meta": { "name": "upstageAI-vectorize", "tags": [ - + "disabled" ] }, "fromEnvironment": { From 36851af85c37eb00548432719ef9c96513dd4ecb Mon Sep 17 00:00:00 2001 From: Aaron Morton Date: Fri, 24 Apr 2026 11:45:03 +1200 Subject: [PATCH 67/89] added failure report section --- .../reporting/TestBenchConsoleWriter.java | 34 ++++++++++++++++++- .../vectorize/vectorize-header-workflow.json | 2 +- 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/reporting/TestBenchConsoleWriter.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/reporting/TestBenchConsoleWriter.java index 2eb8d0cc79..7328d60b12 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/reporting/TestBenchConsoleWriter.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/reporting/TestBenchConsoleWriter.java @@ -116,6 +116,15 @@ public void allTestsFinished(DynamicTreeListener.TestReportingTracker rootTracke writeTestDesc(testPlanNodeDesc, rootTracker); testPlanNodeDesc.newline(); + String failureReport; + if (rootTracker.stats().failures() == 0){ + failureReport = "No failures"; + } else { + var failureReportBuffer = buffer(); + writeFailureMessages(failureReportBuffer, rootTracker); + failureReport = failureReportBuffer.toString(); + } + // report is a markdown // GitHub collapsable sections // https://docs.github.com/en/get-started/writing-on-github/working-with-advanced-formatting/organizing-information-with-collapsed-sections @@ -132,8 +141,18 @@ public void allTestsFinished(DynamicTreeListener.TestReportingTracker rootTracke %s ``` +
+
- """.formatted(rootTracker.identifier().getDisplayName(), testPlanNodeDesc.toString(), testReport); + + Test Bench Failures + + ``` + %s + ``` + +
+ """.formatted(rootTracker.identifier().getDisplayName(), testPlanNodeDesc.toString(), testReport, failureReport); try { Files.writeString(Path.of(reportFilePath), markdownReport); @@ -181,6 +200,19 @@ private void writeCompletedSummary( } } + private void writeFailureMessages(MessageBuilder buffer, DynamicTreeListener.TestReportingTracker tracker) { + + // if we have a throwable, write out the tree node test and the error it generated. + if (tracker.throwable().isPresent()){ + writeTestDesc(buffer, tracker); + buffer.newline(); + buffer.newline(); + buffer.a("```").newline(); + buffer.a(tracker.throwable().get().getMessage()).newline(); + buffer.a("```").newline().newline(); + } + tracker.children().forEach(child -> writeFailureMessages(buffer, child)); + } private void writeTestDesc(MessageBuilder buffer, DynamicTreeListener.TestReportingTracker tracker) { // Icon for success for failure, diff --git a/src/test/resources/integration-tests/vectorize/vectorize-header-workflow.json b/src/test/resources/integration-tests/vectorize/vectorize-header-workflow.json index efde4c1d23..0372823e9c 100644 --- a/src/test/resources/integration-tests/vectorize/vectorize-header-workflow.json +++ b/src/test/resources/integration-tests/vectorize/vectorize-header-workflow.json @@ -177,7 +177,7 @@ "matrix": { "MODEL": [ "NV-Embed-QA", - "nvidia/nv-embedqa-e5-v5" + "nvidia/nv-embedqa-e5-v5-XXX" ] }, "tests": [ From 444fd5106d4184a98ac3a1735601b2e071106212 Mon Sep 17 00:00:00 2001 From: Aaron Morton Date: Fri, 24 Apr 2026 11:49:24 +1200 Subject: [PATCH 68/89] tweak failure report --- .../api/v1/vectorize/reporting/TestBenchConsoleWriter.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/reporting/TestBenchConsoleWriter.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/reporting/TestBenchConsoleWriter.java index 7328d60b12..e2c8c4bda7 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/reporting/TestBenchConsoleWriter.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/reporting/TestBenchConsoleWriter.java @@ -207,9 +207,8 @@ private void writeFailureMessages(MessageBuilder buffer, DynamicTreeListener.Tes writeTestDesc(buffer, tracker); buffer.newline(); buffer.newline(); - buffer.a("```").newline(); - buffer.a(tracker.throwable().get().getMessage()).newline(); - buffer.a("```").newline().newline(); + buffer.a(tracker.throwable().get().getMessage()); + buffer.newline().a("-----").newline().newline(); } tracker.children().forEach(child -> writeFailureMessages(buffer, child)); } From 06d3b539c5ca38e0053cf80b0c492b886600b4cf Mon Sep 17 00:00:00 2001 From: Aaron Morton Date: Fri, 24 Apr 2026 11:51:49 +1200 Subject: [PATCH 69/89] improve failure report --- .../api/v1/vectorize/reporting/TestBenchConsoleWriter.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/reporting/TestBenchConsoleWriter.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/reporting/TestBenchConsoleWriter.java index e2c8c4bda7..1dedea4001 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/reporting/TestBenchConsoleWriter.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/reporting/TestBenchConsoleWriter.java @@ -10,6 +10,7 @@ import java.util.Objects; import org.apache.maven.plugin.surefire.report.Theme; import org.apache.maven.surefire.shared.utils.logging.MessageBuilder; +import org.junit.platform.engine.TestExecutionResult; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -203,7 +204,8 @@ private void writeCompletedSummary( private void writeFailureMessages(MessageBuilder buffer, DynamicTreeListener.TestReportingTracker tracker) { // if we have a throwable, write out the tree node test and the error it generated. - if (tracker.throwable().isPresent()){ + // ignoring the ABORTED + if (tracker.throwable().isPresent() && (tracker.junitStatus() == TestExecutionResult.Status.FAILED )) { writeTestDesc(buffer, tracker); buffer.newline(); buffer.newline(); From 4bc563741ad0a61fb4fe723735b620a34e5d4bec Mon Sep 17 00:00:00 2001 From: Aaron Morton Date: Fri, 24 Apr 2026 11:55:30 +1200 Subject: [PATCH 70/89] test prod --- .github/workflows/test-bench-vectorize.yaml | 8 ++++---- .../test-plans/test-plan-astra-vectorize.yaml | 2 +- .../vectorize/vectorize-header-workflow.json | 14 +++++++------- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/workflows/test-bench-vectorize.yaml b/.github/workflows/test-bench-vectorize.yaml index 477d64c4c6..e66f2b6f7b 100644 --- a/.github/workflows/test-bench-vectorize.yaml +++ b/.github/workflows/test-bench-vectorize.yaml @@ -25,7 +25,7 @@ env: jobs: setup: runs-on: ubuntu-latest - environment: ${{ github.event.inputs.environment || 'DEV' }} + environment: ${{ github.event.inputs.environment || 'PROD' }} outputs: matrix: ${{ steps.set-matrix.outputs.TEST_TARGETS }} @@ -50,7 +50,7 @@ jobs: cat >> $GITHUB_STEP_SUMMARY << 'EOF' ## Test Run Setup - **Environment:** ${{ github.event.inputs.environment || 'DEV' }} + **Environment:** ${{ github.event.inputs.environment || 'PROD' }} **Triggered by:** ${{ github.actor }} ### Test Targets @@ -62,10 +62,10 @@ jobs: # runs unit tests build: - name: Test Bench- ENV-${{ github.event.inputs.environment || 'DEV' }} Target- ${{ matrix.name }} + name: Test Bench- ENV-${{ github.event.inputs.environment || 'PROD' }} Target- ${{ matrix.name }} needs: setup runs-on: ubuntu-latest - environment: ${{ github.event.inputs.environment || 'DEV' }} + environment: ${{ github.event.inputs.environment || 'PROD' }} timeout-minutes: 120 diff --git a/src/test/resources/integration-tests/test-plans/test-plan-astra-vectorize.yaml b/src/test/resources/integration-tests/test-plans/test-plan-astra-vectorize.yaml index 4c1e3cbbff..937dd2f67c 100644 --- a/src/test/resources/integration-tests/test-plans/test-plan-astra-vectorize.yaml +++ b/src/test/resources/integration-tests/test-plans/test-plan-astra-vectorize.yaml @@ -8,5 +8,5 @@ customTarget: basePath: /api/json/v1 workflows: - vectorize-header-workflow -# - vectorize-shared-workflow + - vectorize-shared-workflow ignoreDisabled: true diff --git a/src/test/resources/integration-tests/vectorize/vectorize-header-workflow.json b/src/test/resources/integration-tests/vectorize/vectorize-header-workflow.json index 0372823e9c..2aa573bf6d 100644 --- a/src/test/resources/integration-tests/vectorize/vectorize-header-workflow.json +++ b/src/test/resources/integration-tests/vectorize/vectorize-header-workflow.json @@ -7,7 +7,7 @@ { "meta": { "name": "open-ai-vectorize", - "tags": ["disabled" + "tags": [ ] }, "fromEnvironment": { @@ -32,7 +32,7 @@ { "meta": { "name": "voyageAI-vectorize", - "tags": ["disabled" + "tags": [ ] }, "fromEnvironment": { @@ -59,7 +59,7 @@ { "meta": { "name": "jinaAI-vectorize", - "tags": ["disabled" + "tags": [ ] }, "fromEnvironment": { @@ -86,7 +86,7 @@ { "meta": { "name": "huggingface-non-dedicated-vectorize", - "tags": ["disabled" + "tags": [ ] }, @@ -116,7 +116,7 @@ "meta": { "name": "mistral-vectorize", "tags": [ - "disabled" + ] }, "fromEnvironment": { @@ -140,7 +140,7 @@ "meta": { "name": "upstageAI-vectorize", "tags": [ - "disabled" + ] }, "fromEnvironment": { @@ -177,7 +177,7 @@ "matrix": { "MODEL": [ "NV-Embed-QA", - "nvidia/nv-embedqa-e5-v5-XXX" + "nvidia/nv-embedqa-e5-v5" ] }, "tests": [ From e26d357982f92021b72201112fae6eebedb1c1e4 Mon Sep 17 00:00:00 2001 From: Aaron Morton Date: Fri, 24 Apr 2026 14:21:30 +1200 Subject: [PATCH 71/89] poke --- .../api/v1/vectorize/reporting/DynamicTreeListener.java | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/reporting/DynamicTreeListener.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/reporting/DynamicTreeListener.java index d3ab9c5a0c..28d9c1cb40 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/reporting/DynamicTreeListener.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/reporting/DynamicTreeListener.java @@ -319,11 +319,7 @@ public int failures() { public int skipped() { return skipped; } - -// public boolean noErrors() { -// return aborted == 0 && failures == 0; -// } - + public void testCompleted(TestReportingTracker tracker, TestExecutionResult result) { lastFinishedAtMillis = System.currentTimeMillis(); From 83c03561543a6b5ed6d2438ef06ea226219c6687 Mon Sep 17 00:00:00 2001 From: Aaron Morton Date: Thu, 30 Apr 2026 13:30:09 +1200 Subject: [PATCH 72/89] checking header only, on prod --- .../integration-tests/test-plans/test-plan-astra-vectorize.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/resources/integration-tests/test-plans/test-plan-astra-vectorize.yaml b/src/test/resources/integration-tests/test-plans/test-plan-astra-vectorize.yaml index 937dd2f67c..4c1e3cbbff 100644 --- a/src/test/resources/integration-tests/test-plans/test-plan-astra-vectorize.yaml +++ b/src/test/resources/integration-tests/test-plans/test-plan-astra-vectorize.yaml @@ -8,5 +8,5 @@ customTarget: basePath: /api/json/v1 workflows: - vectorize-header-workflow - - vectorize-shared-workflow +# - vectorize-shared-workflow ignoreDisabled: true From 438d6df316732ecfbdb9a5c3a60f4721556ac738 Mon Sep 17 00:00:00 2001 From: Aaron Morton Date: Thu, 30 Apr 2026 14:09:42 +1200 Subject: [PATCH 73/89] testing basic retry --- .../v1/vectorize/messaging/APIRequest.java | 44 ++++++++++++++++--- 1 file changed, 38 insertions(+), 6 deletions(-) diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/messaging/APIRequest.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/messaging/APIRequest.java index b9b85e9821..32cde5fe8d 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/messaging/APIRequest.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/messaging/APIRequest.java @@ -15,11 +15,16 @@ import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestRunEnv; import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.TestCommand; import io.stargate.sgv2.jsonapi.config.constants.HttpConstants; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.util.HashMap; +import java.util.List; import java.util.Map; +import java.util.concurrent.ThreadLocalRandom; public class APIRequest { + private static final Logger LOGGER = LoggerFactory.getLogger(APIRequest.class); public static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); @@ -56,26 +61,53 @@ private RequestSpecification requestSpec() { return jsonRequest().body(requestString).when(); } - private ValidatableResponse executeRequest(RequestSpecification requestSpec) { + private ValidatableResponse executeRequest(RequestSpecification requestSpec){ + return executeRequest(requestSpec, 1); + } + private ValidatableResponse executeRequest(RequestSpecification requestSpec, int attemptCount) { var commandName = TestCommand.commandName(request); - Response response; + Response rawRresponse; if (commandName.getTargets().contains(CommandTarget.COLLECTION) || commandName.getTargets().contains(CommandTarget.TABLE)) { - response = + rawRresponse = requestSpec.post( COLLECTION_PATH, integrationEnv.requiredValue("KEYSPACE_NAME"), integrationEnv.requiredValue("COLLECTION_NAME")); } else if (commandName.getTargets().contains(CommandTarget.KEYSPACE)) { - response = requestSpec.post(KEYSPACE_PATH, integrationEnv.requiredValue("KEYSPACE_NAME")); + rawRresponse = requestSpec.post(KEYSPACE_PATH, integrationEnv.requiredValue("KEYSPACE_NAME")); } else if (commandName.getTargets().contains(CommandTarget.DATABASE)) { - response = requestSpec.post(DB_PATH); + rawRresponse = requestSpec.post(DB_PATH); } else { throw new IllegalArgumentException("Do not know how to execute command: " + commandName); } - return response.then().log().status().and().log().body(); + var validatableResponse = rawRresponse.then(); + // Logging the response we got, even we want to retry, will want the result in the output + validatableResponse.log().status().and().log().body(); + + if (attemptCount <=3 ) { + var body = rawRresponse.body().print(); + // TODO: XXXX: put this in the test plan + var retryMatch = List.of("EMBEDDING_PROVIDER_RATE_LIMITED", "EMBEDDING_PROVIDER_TIMEOUT"); + for (var match : retryMatch) { + if (body.contains(match)) { + // Exponential backoff with jitter: 2^attempt seconds base (1s, 2s, 4s...) plus up to 500ms + // random jitter to avoid thundering herd if multiple tests hit the rate limit simultaneously + long sleepMs = (long) (1000 * Math.pow(2, attemptCount)) + ThreadLocalRandom.current().nextLong(500); + LOGGER.info("executeRequest() - Retrying, found retry string in response. match={}, sleepMs={} ms, attemptCount={}", match, sleepMs, attemptCount); + try { + Thread.sleep(sleepMs); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException("Interrupted during retry sleep", e); + } + return executeRequest(requestSpec, attemptCount + 1); + } + } + } + return validatableResponse; } protected Map getHeaders() { From b59f5512dd77e1b8aaf3b8223accbe8922b3dd45 Mon Sep 17 00:00:00 2001 From: Aaron Morton Date: Thu, 30 Apr 2026 14:23:04 +1200 Subject: [PATCH 74/89] bug in getting body string --- .../sgv2/jsonapi/api/v1/vectorize/messaging/APIRequest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/messaging/APIRequest.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/messaging/APIRequest.java index 32cde5fe8d..ba480fa523 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/messaging/APIRequest.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/messaging/APIRequest.java @@ -88,7 +88,7 @@ private ValidatableResponse executeRequest(RequestSpecification requestSpec, int validatableResponse.log().status().and().log().body(); if (attemptCount <=3 ) { - var body = rawRresponse.body().print(); + var body = rawRresponse.body().asString(); // TODO: XXXX: put this in the test plan var retryMatch = List.of("EMBEDDING_PROVIDER_RATE_LIMITED", "EMBEDDING_PROVIDER_TIMEOUT"); for (var match : retryMatch) { From 6b517c9a2de68f5ef99914f4fd69e40bbcf72598 Mon Sep 17 00:00:00 2001 From: Aaron Morton Date: Thu, 30 Apr 2026 14:44:58 +1200 Subject: [PATCH 75/89] improve retry --- .../v1/vectorize/messaging/APIRequest.java | 75 +++++++++++-------- 1 file changed, 44 insertions(+), 31 deletions(-) diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/messaging/APIRequest.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/messaging/APIRequest.java index ba480fa523..61b283cdd3 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/messaging/APIRequest.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/messaging/APIRequest.java @@ -45,8 +45,48 @@ public APIRequest(Connection connection, TestRunEnv integrationEnv, ObjectNode r public APIResponse execute() { - var requestSpec = requestSpec(); - return new APIResponse(this, executeRequest(requestSpec)); + boolean retry = true; + int maxAttempts = 3; + int attempt = 1; + ValidatableResponse lastValidatableResponse = null; + + while (retry && attempt <=maxAttempts){ + attempt++; + retry = false; + + // Create a new request spec, there is some state that is left in it when a request is run + // for path params + var requestSpec = requestSpec(); + var rawResponse = executeRequest(requestSpec); + lastValidatableResponse = rawResponse.then(); + + // logg even if we retry + lastValidatableResponse.log().status().and().log().body(); + + if (attempt < maxAttempts) { + var body = rawResponse.body().asString(); + // TODO: XXXX: put this in the test plan + var retryMatch = List.of("EMBEDDING_PROVIDER_RATE_LIMITED", "EMBEDDING_PROVIDER_TIMEOUT"); + + for (var match : retryMatch) { + if (body.contains(match)) { + retry = true; + // Exponential backoff with jitter: 2^attempt seconds base (1s, 2s, 4s...) plus up to 500ms + // random jitter to avoid thundering herd if multiple tests hit the rate limit simultaneously + long sleepMs = (long) (1000 * Math.pow(2, attempt)) + ThreadLocalRandom.current().nextLong(500); + LOGGER.info("executeRequest() - Retrying, found retry string in response. match={}, sleepMs={} ms, attemptCount={}", match, sleepMs, attempt); + try { + Thread.sleep(sleepMs); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException("Interrupted during retry sleep", e); + } + } + } + } + } + return new APIResponse(this, lastValidatableResponse); + } private RequestSpecification requestSpec() { @@ -61,10 +101,7 @@ private RequestSpecification requestSpec() { return jsonRequest().body(requestString).when(); } - private ValidatableResponse executeRequest(RequestSpecification requestSpec){ - return executeRequest(requestSpec, 1); - } - private ValidatableResponse executeRequest(RequestSpecification requestSpec, int attemptCount) { + private Response executeRequest(RequestSpecification requestSpec) { var commandName = TestCommand.commandName(request); Response rawRresponse; @@ -83,31 +120,7 @@ private ValidatableResponse executeRequest(RequestSpecification requestSpec, int throw new IllegalArgumentException("Do not know how to execute command: " + commandName); } - var validatableResponse = rawRresponse.then(); - // Logging the response we got, even we want to retry, will want the result in the output - validatableResponse.log().status().and().log().body(); - - if (attemptCount <=3 ) { - var body = rawRresponse.body().asString(); - // TODO: XXXX: put this in the test plan - var retryMatch = List.of("EMBEDDING_PROVIDER_RATE_LIMITED", "EMBEDDING_PROVIDER_TIMEOUT"); - for (var match : retryMatch) { - if (body.contains(match)) { - // Exponential backoff with jitter: 2^attempt seconds base (1s, 2s, 4s...) plus up to 500ms - // random jitter to avoid thundering herd if multiple tests hit the rate limit simultaneously - long sleepMs = (long) (1000 * Math.pow(2, attemptCount)) + ThreadLocalRandom.current().nextLong(500); - LOGGER.info("executeRequest() - Retrying, found retry string in response. match={}, sleepMs={} ms, attemptCount={}", match, sleepMs, attemptCount); - try { - Thread.sleep(sleepMs); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - throw new RuntimeException("Interrupted during retry sleep", e); - } - return executeRequest(requestSpec, attemptCount + 1); - } - } - } - return validatableResponse; + return rawRresponse; } protected Map getHeaders() { From cf26885341b996aa4e4d16e76b80bad3e36dd439 Mon Sep 17 00:00:00 2001 From: Aaron Morton Date: Thu, 30 Apr 2026 14:57:43 +1200 Subject: [PATCH 76/89] fix bad loop --- .../jsonapi/api/v1/vectorize/messaging/APIRequest.java | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/messaging/APIRequest.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/messaging/APIRequest.java index 61b283cdd3..f116a1ffae 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/messaging/APIRequest.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/messaging/APIRequest.java @@ -46,12 +46,11 @@ public APIRequest(Connection connection, TestRunEnv integrationEnv, ObjectNode r public APIResponse execute() { boolean retry = true; - int maxAttempts = 3; + int maxAttempts = 4; // 4 attempts, means 3 retries int attempt = 1; ValidatableResponse lastValidatableResponse = null; while (retry && attempt <=maxAttempts){ - attempt++; retry = false; // Create a new request spec, there is some state that is left in it when a request is run @@ -71,9 +70,9 @@ public APIResponse execute() { for (var match : retryMatch) { if (body.contains(match)) { retry = true; - // Exponential backoff with jitter: 2^attempt seconds base (1s, 2s, 4s...) plus up to 500ms + // Exponential backoff with jitter: 2^attempt seconds base (1s, 2s, 4s...) plus up to 1000ms // random jitter to avoid thundering herd if multiple tests hit the rate limit simultaneously - long sleepMs = (long) (1000 * Math.pow(2, attempt)) + ThreadLocalRandom.current().nextLong(500); + long sleepMs = (long) (1000 * Math.pow(2, attempt)) + ThreadLocalRandom.current().nextLong(1000); LOGGER.info("executeRequest() - Retrying, found retry string in response. match={}, sleepMs={} ms, attemptCount={}", match, sleepMs, attempt); try { Thread.sleep(sleepMs); @@ -81,9 +80,11 @@ public APIResponse execute() { Thread.currentThread().interrupt(); throw new RuntimeException("Interrupted during retry sleep", e); } + break; } } } + attempt++; } return new APIResponse(this, lastValidatableResponse); From 9633193de4e9ae3f4ba89df17f1d25139e92bbb1 Mon Sep 17 00:00:00 2001 From: Aaron Morton Date: Thu, 30 Apr 2026 15:15:53 +1200 Subject: [PATCH 77/89] trying different jitter --- .../jsonapi/api/v1/vectorize/messaging/APIRequest.java | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/messaging/APIRequest.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/messaging/APIRequest.java index f116a1ffae..40ca26b1b4 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/messaging/APIRequest.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/messaging/APIRequest.java @@ -70,9 +70,13 @@ public APIResponse execute() { for (var match : retryMatch) { if (body.contains(match)) { retry = true; - // Exponential backoff with jitter: 2^attempt seconds base (1s, 2s, 4s...) plus up to 1000ms - // random jitter to avoid thundering herd if multiple tests hit the rate limit simultaneously - long sleepMs = (long) (1000 * Math.pow(2, attempt)) + ThreadLocalRandom.current().nextLong(1000); + // Service has a concurrency limit and retrying runners can collide regardless of jitter. + // Base backoff is long enough to wait out an in-flight request (5s, 10s, 20s...), + // plus a small random offset to avoid re-synchronising after the wait. + long baseMs = (long) (5000 * Math.pow(2, attempt)); + long jitterMs = ThreadLocalRandom.current().nextLong(2000); + long sleepMs = baseMs + jitterMs; + LOGGER.info("executeRequest() - Retrying, found retry string in response. match={}, sleepMs={} ms, attemptCount={}", match, sleepMs, attempt); try { Thread.sleep(sleepMs); From 0d7e69aa1fd4222bf71cc7a386e7e399c00b779e Mon Sep 17 00:00:00 2001 From: Aaron Morton Date: Thu, 30 Apr 2026 15:26:09 +1200 Subject: [PATCH 78/89] more tests --- .../sgv2/jsonapi/api/v1/vectorize/messaging/APIRequest.java | 2 +- .../integration-tests/test-plans/test-plan-astra-vectorize.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/messaging/APIRequest.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/messaging/APIRequest.java index 40ca26b1b4..4c36039d40 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/messaging/APIRequest.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/messaging/APIRequest.java @@ -46,7 +46,7 @@ public APIRequest(Connection connection, TestRunEnv integrationEnv, ObjectNode r public APIResponse execute() { boolean retry = true; - int maxAttempts = 4; // 4 attempts, means 3 retries + int maxAttempts = 6; // 6 attempts, means 5 retries int attempt = 1; ValidatableResponse lastValidatableResponse = null; diff --git a/src/test/resources/integration-tests/test-plans/test-plan-astra-vectorize.yaml b/src/test/resources/integration-tests/test-plans/test-plan-astra-vectorize.yaml index 4c1e3cbbff..937dd2f67c 100644 --- a/src/test/resources/integration-tests/test-plans/test-plan-astra-vectorize.yaml +++ b/src/test/resources/integration-tests/test-plans/test-plan-astra-vectorize.yaml @@ -8,5 +8,5 @@ customTarget: basePath: /api/json/v1 workflows: - vectorize-header-workflow -# - vectorize-shared-workflow + - vectorize-shared-workflow ignoreDisabled: true From a9c03ac54a78469fce3943395c62d54468d61925 Mon Sep 17 00:00:00 2001 From: Aaron Morton Date: Mon, 4 May 2026 10:18:03 +1200 Subject: [PATCH 79/89] testing with new keys --- .../vectorize/vectorize-shared-workflow.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/resources/integration-tests/vectorize/vectorize-shared-workflow.json b/src/test/resources/integration-tests/vectorize/vectorize-shared-workflow.json index 74a3448dea..4e1fffdfef 100644 --- a/src/test/resources/integration-tests/vectorize/vectorize-shared-workflow.json +++ b/src/test/resources/integration-tests/vectorize/vectorize-shared-workflow.json @@ -17,7 +17,7 @@ "variables": { "PROVIDER": "openai", "COLLECTION_NAME": "${PROVIDER}-${MODEL}", - "CREDENTIAL" : "kent-open-ai-key" + "CREDENTIAL" : "open-ai-key" }, "matrix": { "MODEL": [ @@ -72,7 +72,7 @@ "variables": { "PROVIDER": "jinaAI", "COLLECTION_NAME": "${PROVIDER}-${MODEL}", - "CREDENTIAL" : "kent-jinaai-key" + "CREDENTIAL" : "aaron-jinaai-key" }, "matrix": { "MODEL": [ From aa1ce3a159743137ad9e8bef204be5c4bea92c11 Mon Sep 17 00:00:00 2001 From: Aaron Morton Date: Mon, 4 May 2026 11:53:00 +1200 Subject: [PATCH 80/89] refactoring for commit --- .github/workflows/test-bench-vectorize.yaml | 4 +- .../api/v1/vectorize/targets/Backend.java | 9 ----- .../testspec/TargetConfiguration.java | 5 --- .../TestBenchByTestPlan.java} | 37 ++++++------------- .../v1/vectorize => testbench}/TestPlan.java | 29 +++++++-------- .../vectorize => testbench}/VectorizeIT.java | 6 +-- .../assertions/AssertionFactory.java | 6 +-- .../assertions/AssertionFactoryRegistry.java | 2 +- .../assertions/AssertionMatcher.java | 4 +- .../assertions/AssertionName.java | 4 +- .../assertions/BodyAssertion.java | 4 +- .../assertions/Describable.java | 2 +- .../DescribableAssertionMatcher.java | 4 +- .../assertions/Documents.java | 4 +- .../assertions/Http.java | 6 +-- .../assertions/Response.java | 4 +- .../assertions/SingleTestAssertion.java | 5 ++- .../assertions/Status.java | 4 +- .../assertions/Templated.java | 6 +-- .../assertions/TestAssertion.java | 18 ++++----- .../assertions/TestAssertionContainer.java | 10 ++--- .../lifecycle/TestPlanLifecycle.java | 16 ++++---- .../messaging/APIRequest.java | 8 ++-- .../messaging/APIResponse.java | 2 +- .../reporting/DynamicTreeListener.java | 4 +- .../reporting/TestBenchConsoleWriter.java | 4 +- .../targets/AstraBackend.java | 4 +- .../jsonapi/testbench/targets/Backend.java | 9 +++++ .../targets/CassandraBackend.java | 18 ++++----- .../targets/Connection.java | 2 +- .../targets/Target.java | 22 +++++------ .../testrun/DynamicTestExecutable.java | 2 +- .../testrun/TestExecutionCondition.java | 2 +- .../testrun/TestNodeFactory.java | 13 +++---- .../testrun/TestRunEnv.java | 4 +- .../testrun/TestRunRequest.java | 10 ++--- .../testrun/TestRunResponse.java | 6 +-- .../testrun/TestUri.java | 2 +- .../testspec/AssertionTemplateSpec.java | 2 +- .../vectorize => testbench}/testspec/Job.java | 10 ++--- .../testspec/SpecFile.java | 2 +- .../testspec/SpecFiles.java | 2 +- .../testspec/TargetConfiguration.java | 5 +++ .../testspec/TargetsSpec.java | 2 +- .../testspec/TestCase.java | 6 +-- .../testspec/TestCommand.java | 4 +- .../testspec/TestEnvAccess.java | 2 +- .../testspec/TestSpec.java | 2 +- .../testspec/TestSpecKind.java | 2 +- .../testspec/TestSpecMeta.java | 2 +- .../testspec/TestSuiteSpec.java | 8 ++-- .../testspec/WorkflowSpec.java | 6 +-- ...it.platform.launcher.TestExecutionListener | 2 +- .../assertions/assertion-templates.json | 0 .../targets/targets.json | 0 .../testplans}/test-plan-astra-vectorize.yaml | 0 .../testsuites}/findEmbeddingProviders.json | 0 .../testsuites}/vectorize-base.json | 0 .../testsuites}/vectorize-header-auth.json | 0 .../testsuites}/vectorize-shared-auth.json | 0 .../findEmbeddingProviders-workflow.json | 0 .../workflows}/vectorize-header-workflow.json | 0 .../workflows}/vectorize-shared-workflow.json | 0 63 files changed, 170 insertions(+), 188 deletions(-) delete mode 100644 src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/targets/Backend.java delete mode 100644 src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/TargetConfiguration.java rename src/test/java/io/stargate/sgv2/jsonapi/{api/v1/vectorize/TestBenchTests.java => testbench/TestBenchByTestPlan.java} (52%) rename src/test/java/io/stargate/sgv2/jsonapi/{api/v1/vectorize => testbench}/TestPlan.java (81%) rename src/test/java/io/stargate/sgv2/jsonapi/{api/v1/vectorize => testbench}/VectorizeIT.java (74%) rename src/test/java/io/stargate/sgv2/jsonapi/{api/v1/vectorize => testbench}/assertions/AssertionFactory.java (94%) rename src/test/java/io/stargate/sgv2/jsonapi/{api/v1/vectorize => testbench}/assertions/AssertionFactoryRegistry.java (96%) rename src/test/java/io/stargate/sgv2/jsonapi/{api/v1/vectorize => testbench}/assertions/AssertionMatcher.java (70%) rename src/test/java/io/stargate/sgv2/jsonapi/{api/v1/vectorize => testbench}/assertions/AssertionName.java (91%) rename src/test/java/io/stargate/sgv2/jsonapi/{api/v1/vectorize => testbench}/assertions/BodyAssertion.java (85%) rename src/test/java/io/stargate/sgv2/jsonapi/{api/v1/vectorize => testbench}/assertions/Describable.java (55%) rename src/test/java/io/stargate/sgv2/jsonapi/{api/v1/vectorize => testbench}/assertions/DescribableAssertionMatcher.java (81%) rename src/test/java/io/stargate/sgv2/jsonapi/{api/v1/vectorize => testbench}/assertions/Documents.java (84%) rename src/test/java/io/stargate/sgv2/jsonapi/{api/v1/vectorize => testbench}/assertions/Http.java (63%) rename src/test/java/io/stargate/sgv2/jsonapi/{api/v1/vectorize => testbench}/assertions/Response.java (93%) rename src/test/java/io/stargate/sgv2/jsonapi/{api/v1/vectorize => testbench}/assertions/SingleTestAssertion.java (92%) rename src/test/java/io/stargate/sgv2/jsonapi/{api/v1/vectorize => testbench}/assertions/Status.java (72%) rename src/test/java/io/stargate/sgv2/jsonapi/{api/v1/vectorize => testbench}/assertions/Templated.java (85%) rename src/test/java/io/stargate/sgv2/jsonapi/{api/v1/vectorize => testbench}/assertions/TestAssertion.java (83%) rename src/test/java/io/stargate/sgv2/jsonapi/{api/v1/vectorize => testbench}/assertions/TestAssertionContainer.java (80%) rename src/test/java/io/stargate/sgv2/jsonapi/{api/v1/vectorize => testbench}/lifecycle/TestPlanLifecycle.java (70%) rename src/test/java/io/stargate/sgv2/jsonapi/{api/v1/vectorize => testbench}/messaging/APIRequest.java (95%) rename src/test/java/io/stargate/sgv2/jsonapi/{api/v1/vectorize => testbench}/messaging/APIResponse.java (69%) rename src/test/java/io/stargate/sgv2/jsonapi/{api/v1/vectorize => testbench}/reporting/DynamicTreeListener.java (98%) rename src/test/java/io/stargate/sgv2/jsonapi/{api/v1/vectorize => testbench}/reporting/TestBenchConsoleWriter.java (98%) rename src/test/java/io/stargate/sgv2/jsonapi/{api/v1/vectorize => testbench}/targets/AstraBackend.java (58%) create mode 100644 src/test/java/io/stargate/sgv2/jsonapi/testbench/targets/Backend.java rename src/test/java/io/stargate/sgv2/jsonapi/{api/v1/vectorize => testbench}/targets/CassandraBackend.java (79%) rename src/test/java/io/stargate/sgv2/jsonapi/{api/v1/vectorize => testbench}/targets/Connection.java (82%) rename src/test/java/io/stargate/sgv2/jsonapi/{api/v1/vectorize => testbench}/targets/Target.java (76%) rename src/test/java/io/stargate/sgv2/jsonapi/{api/v1/vectorize => testbench}/testrun/DynamicTestExecutable.java (97%) rename src/test/java/io/stargate/sgv2/jsonapi/{api/v1/vectorize => testbench}/testrun/TestExecutionCondition.java (96%) rename src/test/java/io/stargate/sgv2/jsonapi/{api/v1/vectorize => testbench}/testrun/TestNodeFactory.java (93%) rename src/test/java/io/stargate/sgv2/jsonapi/{api/v1/vectorize => testbench}/testrun/TestRunEnv.java (96%) rename src/test/java/io/stargate/sgv2/jsonapi/{api/v1/vectorize => testbench}/testrun/TestRunRequest.java (87%) rename src/test/java/io/stargate/sgv2/jsonapi/{api/v1/vectorize => testbench}/testrun/TestRunResponse.java (82%) rename src/test/java/io/stargate/sgv2/jsonapi/{api/v1/vectorize => testbench}/testrun/TestUri.java (98%) rename src/test/java/io/stargate/sgv2/jsonapi/{api/v1/vectorize => testbench}/testspec/AssertionTemplateSpec.java (84%) rename src/test/java/io/stargate/sgv2/jsonapi/{api/v1/vectorize => testbench}/testspec/Job.java (89%) rename src/test/java/io/stargate/sgv2/jsonapi/{api/v1/vectorize => testbench}/testspec/SpecFile.java (87%) rename src/test/java/io/stargate/sgv2/jsonapi/{api/v1/vectorize => testbench}/testspec/SpecFiles.java (98%) create mode 100644 src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/TargetConfiguration.java rename src/test/java/io/stargate/sgv2/jsonapi/{api/v1/vectorize => testbench}/testspec/TargetsSpec.java (97%) rename src/test/java/io/stargate/sgv2/jsonapi/{api/v1/vectorize => testbench}/testspec/TestCase.java (82%) rename src/test/java/io/stargate/sgv2/jsonapi/{api/v1/vectorize => testbench}/testspec/TestCommand.java (96%) rename src/test/java/io/stargate/sgv2/jsonapi/{api/v1/vectorize => testbench}/testspec/TestEnvAccess.java (89%) rename src/test/java/io/stargate/sgv2/jsonapi/{api/v1/vectorize => testbench}/testspec/TestSpec.java (87%) rename src/test/java/io/stargate/sgv2/jsonapi/{api/v1/vectorize => testbench}/testspec/TestSpecKind.java (89%) rename src/test/java/io/stargate/sgv2/jsonapi/{api/v1/vectorize => testbench}/testspec/TestSpecMeta.java (63%) rename src/test/java/io/stargate/sgv2/jsonapi/{api/v1/vectorize => testbench}/testspec/TestSuiteSpec.java (95%) rename src/test/java/io/stargate/sgv2/jsonapi/{api/v1/vectorize => testbench}/testspec/WorkflowSpec.java (81%) rename src/test/resources/{integration-tests => testbench}/assertions/assertion-templates.json (100%) rename src/test/resources/{integration-tests => testbench}/targets/targets.json (100%) rename src/test/resources/{integration-tests/test-plans => testbench/testplans}/test-plan-astra-vectorize.yaml (100%) rename src/test/resources/{integration-tests/vectorize => testbench/testsuites}/findEmbeddingProviders.json (100%) rename src/test/resources/{integration-tests/vectorize => testbench/testsuites}/vectorize-base.json (100%) rename src/test/resources/{integration-tests/vectorize => testbench/testsuites}/vectorize-header-auth.json (100%) rename src/test/resources/{integration-tests/vectorize => testbench/testsuites}/vectorize-shared-auth.json (100%) rename src/test/resources/{integration-tests/vectorize => testbench/workflows}/findEmbeddingProviders-workflow.json (100%) rename src/test/resources/{integration-tests/vectorize => testbench/workflows}/vectorize-header-workflow.json (100%) rename src/test/resources/{integration-tests/vectorize => testbench/workflows}/vectorize-shared-workflow.json (100%) diff --git a/.github/workflows/test-bench-vectorize.yaml b/.github/workflows/test-bench-vectorize.yaml index e66f2b6f7b..176bf2ac86 100644 --- a/.github/workflows/test-bench-vectorize.yaml +++ b/.github/workflows/test-bench-vectorize.yaml @@ -129,7 +129,7 @@ jobs: - name: Build & Test env: - TEST_PLAN_FILE: classpath:integration-tests/test-plans/test-plan-astra-vectorize.yaml + TEST_PLAN_FILE: classpath:testbench/testplans/test-plan-astra-vectorize.yaml DATA_API_TOKEN: ${{ secrets.DATA_API_TOKEN }} TARGET_NAME: ${{ matrix.name }} ENDPOINT: ${{ matrix.endpoint }} @@ -149,7 +149,7 @@ jobs: -Dorg.fusesource.jansi.Ansi.disable=true \ -Dfailsafe.printSummary=false \ -Dfailsafe.trimStackTrace=true \ - -Dit.test=TestBenchTests \ + -Dit.test=TestBenchByTestPlan \ -Dtest-bench-report-path=$TEST_BENCH_REPORT - name: Write summary diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/targets/Backend.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/targets/Backend.java deleted file mode 100644 index a2d40bb290..0000000000 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/targets/Backend.java +++ /dev/null @@ -1,9 +0,0 @@ -package io.stargate.sgv2.jsonapi.api.v1.vectorize.targets; - -import io.stargate.sgv2.jsonapi.api.v1.vectorize.lifecycle.TestPlanLifecycle; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.Job; - -public abstract class Backend implements TestPlanLifecycle { - - public void updateJobForTarget(Job job) {} -} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/TargetConfiguration.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/TargetConfiguration.java deleted file mode 100644 index e9974df284..0000000000 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/TargetConfiguration.java +++ /dev/null @@ -1,5 +0,0 @@ -package io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec; - -import io.stargate.sgv2.jsonapi.api.v1.vectorize.targets.Connection; - -public record TargetConfiguration(String name, String backend, Connection connection) {} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestBenchTests.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/TestBenchByTestPlan.java similarity index 52% rename from src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestBenchTests.java rename to src/test/java/io/stargate/sgv2/jsonapi/testbench/TestBenchByTestPlan.java index 639afbce72..810ef4c24c 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestBenchTests.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/TestBenchByTestPlan.java @@ -1,18 +1,24 @@ -package io.stargate.sgv2.jsonapi.api.v1.vectorize; +package io.stargate.sgv2.jsonapi.testbench; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.TargetsSpec; +import io.stargate.sgv2.jsonapi.testbench.testspec.TargetsSpec; import java.nio.file.Path; import java.util.stream.Stream; -import org.junit.jupiter.api.DynamicContainer; + import org.junit.jupiter.api.DynamicNode; -import org.junit.jupiter.api.DynamicTest; import org.junit.jupiter.api.TestFactory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public class TestBenchTests { +/** + * Entry point for running a Test Bench from a Test Plan file. + *

+ * Put the name of the test plan file in the TEST_PLAN_FILE env var, this can + * set the target to hit, and the workflows to run. See {@link TestPlan.TestPlanFile} + *

+ */ +public class TestBenchByTestPlan { - private static final Logger LOGGER = LoggerFactory.getLogger(TestBenchTests.class); + private static final Logger LOGGER = LoggerFactory.getLogger(TestBenchByTestPlan.class); @TestFactory public Stream runTestPlanFile() { @@ -33,23 +39,4 @@ public Stream runTestPlanFile() { System.setProperty("testbench.test.count", String.valueOf(testPlanNodeTree.totalNodeCount())); return Stream.of(testPlanNodeTree.root()); } - - private static int countAllNodes(int accum, Stream nodes) { - return nodes.reduce(accum, (acc, node) -> switch (node) { - case DynamicTest t -> acc + 1; - case DynamicContainer c -> countAllNodes(acc, c.getChildren().map(n -> (DynamicNode) n)); - default -> acc; - }, Integer::sum); - } - - - // - // public static void main(String[] args) { - // LauncherDiscoveryRequest request = LauncherDiscoveryRequestBuilder.request() - // .selectors(DiscoverySelectors.selectClass(TestBenchTests.class)) - // .build(); - // - // Launcher launcher = LauncherFactory.create(); - // launcher.execute(request, new DynamicTreeListener()); - // } } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestPlan.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/TestPlan.java similarity index 81% rename from src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestPlan.java rename to src/test/java/io/stargate/sgv2/jsonapi/testbench/TestPlan.java index 98bb657c6d..68b23b8406 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/TestPlan.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/TestPlan.java @@ -1,4 +1,4 @@ -package io.stargate.sgv2.jsonapi.api.v1.vectorize; +package io.stargate.sgv2.jsonapi.testbench; @@ -7,29 +7,26 @@ import com.fasterxml.jackson.databind.MapperFeature; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.targets.Target; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestNodeFactory; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestRunEnv; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestUri; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.*; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.Job; +import io.stargate.sgv2.jsonapi.testbench.targets.Target; +import io.stargate.sgv2.jsonapi.testbench.testrun.TestNodeFactory; +import io.stargate.sgv2.jsonapi.testbench.testrun.TestUri; +import io.stargate.sgv2.jsonapi.testbench.testspec.*; + import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.Set; -import java.util.function.Supplier; import java.util.stream.Stream; + import org.apache.commons.text.StringSubstitutor; -import org.junit.jupiter.api.DynamicContainer; import org.junit.jupiter.api.DynamicNode; import org.slf4j.Logger; import org.slf4j.LoggerFactory; public record TestPlan( - Target target, SpecFiles specFiles, Set workflows, boolean ignoreDisabled) { + Target target, SpecFiles specFiles, Set workflows, boolean ignoreDisabled) { private static final Logger LOGGER = LoggerFactory.getLogger(TestPlan.class); private static final ObjectMapper JSON_MAPPER = @@ -72,19 +69,21 @@ public static TestPlan create(String targetName, List workflows) { } public static TestPlan create(String targetName, List workflows, Boolean ignoreDisabled) { - var targetConfigs = TargetsSpec.loadAll("integration-tests/targets/targets.json"); + var targetConfigs = TargetsSpec.loadAll("testbench/targets/targets.json"); return create(targetConfigs.configuration(targetName), workflows, ignoreDisabled); } public static TestPlan create( - TargetConfiguration targetConfiguration, List workflows, Boolean ignoreDisabled) { + TargetConfiguration targetConfiguration, List workflows, Boolean ignoreDisabled) { var target = new Target(targetConfiguration); var specFiles = SpecFiles.loadAll( List.of( - "integration-tests/vectorize", - "integration-tests/assertions/assertion-templates.json")); + "testbench/assertions", + "testbench/testsuites", + "testbench/workflows" + )); return new TestPlan( target, diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/VectorizeIT.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/VectorizeIT.java similarity index 74% rename from src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/VectorizeIT.java rename to src/test/java/io/stargate/sgv2/jsonapi/testbench/VectorizeIT.java index 472d2e4148..d006575ac6 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/VectorizeIT.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/VectorizeIT.java @@ -1,13 +1,9 @@ -package io.stargate.sgv2.jsonapi.api.v1.vectorize; +package io.stargate.sgv2.jsonapi.testbench; import io.quarkus.test.common.WithTestResource; import io.quarkus.test.junit.QuarkusIntegrationTest; import io.stargate.sgv2.jsonapi.api.v1.AbstractCollectionIntegrationTestBase; import io.stargate.sgv2.jsonapi.testresource.DseTestResource; -import java.util.List; -import java.util.stream.Stream; -import org.junit.jupiter.api.DynamicContainer; -import org.junit.jupiter.api.TestFactory; @QuarkusIntegrationTest @WithTestResource(value = DseTestResource.class) diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/AssertionFactory.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/AssertionFactory.java similarity index 94% rename from src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/AssertionFactory.java rename to src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/AssertionFactory.java index f4ae01255c..fcbc76b5e0 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/AssertionFactory.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/AssertionFactory.java @@ -1,8 +1,8 @@ -package io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions; +package io.stargate.sgv2.jsonapi.testbench.assertions; import com.fasterxml.jackson.databind.JsonNode; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.TestPlan; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.TestCommand; +import io.stargate.sgv2.jsonapi.testbench.TestPlan; +import io.stargate.sgv2.jsonapi.testbench.testspec.TestCommand; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.lang.reflect.Modifier; diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/AssertionFactoryRegistry.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/AssertionFactoryRegistry.java similarity index 96% rename from src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/AssertionFactoryRegistry.java rename to src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/AssertionFactoryRegistry.java index e369202d35..045caec2c2 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/AssertionFactoryRegistry.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/AssertionFactoryRegistry.java @@ -1,4 +1,4 @@ -package io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions; +package io.stargate.sgv2.jsonapi.testbench.assertions; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/AssertionMatcher.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/AssertionMatcher.java similarity index 70% rename from src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/AssertionMatcher.java rename to src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/AssertionMatcher.java index a00898cf93..38e616b98f 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/AssertionMatcher.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/AssertionMatcher.java @@ -1,6 +1,6 @@ -package io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions; +package io.stargate.sgv2.jsonapi.testbench.assertions; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.messaging.APIResponse; +import io.stargate.sgv2.jsonapi.testbench.messaging.APIResponse; /** Contract for running an assertion on the response from the API. */ @FunctionalInterface diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/AssertionName.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/AssertionName.java similarity index 91% rename from src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/AssertionName.java rename to src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/AssertionName.java index bf5ae37e13..5a4ccdd055 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/AssertionName.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/AssertionName.java @@ -1,10 +1,10 @@ -package io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions; +package io.stargate.sgv2.jsonapi.testbench.assertions; import java.lang.reflect.Method; public record AssertionName(String typeName, String funcName) { - private static final String PACKAGE = "io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions"; + private static final String PACKAGE = "io.stargate.sgv2.jsonapi.testbench.assertions"; public AssertionName { typeName = typeName.toLowerCase(); diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/BodyAssertion.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/BodyAssertion.java similarity index 85% rename from src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/BodyAssertion.java rename to src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/BodyAssertion.java index be069a1a42..643bf2130b 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/BodyAssertion.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/BodyAssertion.java @@ -1,6 +1,6 @@ -package io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions; +package io.stargate.sgv2.jsonapi.testbench.assertions; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.messaging.APIResponse; +import io.stargate.sgv2.jsonapi.testbench.messaging.APIResponse; import org.hamcrest.Matcher; import org.hamcrest.StringDescription; diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Describable.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/Describable.java similarity index 55% rename from src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Describable.java rename to src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/Describable.java index fc1a0a5f67..afce9b7061 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Describable.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/Describable.java @@ -1,4 +1,4 @@ -package io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions; +package io.stargate.sgv2.jsonapi.testbench.assertions; @FunctionalInterface public interface Describable { diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/DescribableAssertionMatcher.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/DescribableAssertionMatcher.java similarity index 81% rename from src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/DescribableAssertionMatcher.java rename to src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/DescribableAssertionMatcher.java index 370569abc7..c8b1ba2f66 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/DescribableAssertionMatcher.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/DescribableAssertionMatcher.java @@ -1,6 +1,6 @@ -package io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions; +package io.stargate.sgv2.jsonapi.testbench.assertions; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.messaging.APIResponse; +import io.stargate.sgv2.jsonapi.testbench.messaging.APIResponse; import org.jspecify.annotations.NonNull; public record DescribableAssertionMatcher(String description, AssertionMatcher matcher) diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Documents.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/Documents.java similarity index 84% rename from src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Documents.java rename to src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/Documents.java index 7dd711bbd6..758a960e0b 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Documents.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/Documents.java @@ -1,10 +1,10 @@ -package io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions; +package io.stargate.sgv2.jsonapi.testbench.assertions; import static net.javacrumbs.jsonunit.JsonMatchers.jsonEquals; import static org.hamcrest.Matchers.hasSize; import com.fasterxml.jackson.databind.JsonNode; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.TestCommand; +import io.stargate.sgv2.jsonapi.testbench.testspec.TestCommand; /** * Assertions that check the `document` or `documents` in the `data` field of the API result. diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Http.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/Http.java similarity index 63% rename from src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Http.java rename to src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/Http.java index 943b5e95f6..fc1d237e07 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Http.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/Http.java @@ -1,9 +1,9 @@ -package io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions; +package io.stargate.sgv2.jsonapi.testbench.assertions; -import static io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions.DescribableAssertionMatcher.described; +import static io.stargate.sgv2.jsonapi.testbench.assertions.DescribableAssertionMatcher.described; import com.fasterxml.jackson.databind.JsonNode; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.TestCommand; +import io.stargate.sgv2.jsonapi.testbench.testspec.TestCommand; import org.apache.http.HttpStatus; public class Http { diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Response.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/Response.java similarity index 93% rename from src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Response.java rename to src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/Response.java index 48e462124e..d296b12a45 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Response.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/Response.java @@ -1,9 +1,9 @@ -package io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions; +package io.stargate.sgv2.jsonapi.testbench.assertions; import static io.stargate.sgv2.jsonapi.api.v1.ResponseAssertions.*; import com.fasterxml.jackson.databind.JsonNode; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.TestCommand; +import io.stargate.sgv2.jsonapi.testbench.testspec.TestCommand; /** * Assertions that check the structure of the API response, e.g. should it have a `data` field diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/SingleTestAssertion.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/SingleTestAssertion.java similarity index 92% rename from src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/SingleTestAssertion.java rename to src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/SingleTestAssertion.java index 3c228b9b8e..4f12768fd8 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/SingleTestAssertion.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/SingleTestAssertion.java @@ -1,9 +1,10 @@ -package io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions; +package io.stargate.sgv2.jsonapi.testbench.assertions; import com.fasterxml.jackson.databind.JsonNode; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.*; import java.util.concurrent.atomic.AtomicReference; + +import io.stargate.sgv2.jsonapi.testbench.testrun.*; import org.junit.jupiter.api.DynamicNode; public record SingleTestAssertion(String name, JsonNode args, AssertionMatcher matcher) diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Status.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/Status.java similarity index 72% rename from src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Status.java rename to src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/Status.java index 4797035d77..5b1d24299c 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Status.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/Status.java @@ -1,9 +1,9 @@ -package io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions; +package io.stargate.sgv2.jsonapi.testbench.assertions; import static net.javacrumbs.jsonunit.JsonMatchers.jsonEquals; import com.fasterxml.jackson.databind.JsonNode; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.TestCommand; +import io.stargate.sgv2.jsonapi.testbench.testspec.TestCommand; public class Status { diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Templated.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/Templated.java similarity index 85% rename from src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Templated.java rename to src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/Templated.java index 45da77c935..fe54cb08ea 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/Templated.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/Templated.java @@ -1,9 +1,9 @@ -package io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions; +package io.stargate.sgv2.jsonapi.testbench.assertions; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.TestPlan; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.TestCommand; +import io.stargate.sgv2.jsonapi.testbench.TestPlan; +import io.stargate.sgv2.jsonapi.testbench.testspec.TestCommand; import java.util.List; public class Templated { diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/TestAssertion.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/TestAssertion.java similarity index 83% rename from src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/TestAssertion.java rename to src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/TestAssertion.java index aa3e250bdd..ae1cecf204 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/TestAssertion.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/TestAssertion.java @@ -1,14 +1,14 @@ -package io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions; +package io.stargate.sgv2.jsonapi.testbench.assertions; import com.fasterxml.jackson.databind.JsonNode; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.TestPlan; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestExecutionCondition; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestNodeFactory; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestRunResponse; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestUri; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.AssertionTemplateSpec; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.TestCase; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.TestCommand; +import io.stargate.sgv2.jsonapi.testbench.TestPlan; +import io.stargate.sgv2.jsonapi.testbench.testrun.TestExecutionCondition; +import io.stargate.sgv2.jsonapi.testbench.testrun.TestNodeFactory; +import io.stargate.sgv2.jsonapi.testbench.testrun.TestRunResponse; +import io.stargate.sgv2.jsonapi.testbench.testrun.TestUri; +import io.stargate.sgv2.jsonapi.testbench.testspec.AssertionTemplateSpec; +import io.stargate.sgv2.jsonapi.testbench.testspec.TestCase; +import io.stargate.sgv2.jsonapi.testbench.testspec.TestCommand; import java.util.List; import java.util.Map; import java.util.concurrent.atomic.AtomicReference; diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/TestAssertionContainer.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/TestAssertionContainer.java similarity index 80% rename from src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/TestAssertionContainer.java rename to src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/TestAssertionContainer.java index b851884e0e..42ffd13920 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/assertions/TestAssertionContainer.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/TestAssertionContainer.java @@ -1,12 +1,12 @@ -package io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions; +package io.stargate.sgv2.jsonapi.testbench.assertions; import com.fasterxml.jackson.databind.JsonNode; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestExecutionCondition; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestNodeFactory; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestRunResponse; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestUri; +import io.stargate.sgv2.jsonapi.testbench.testrun.TestExecutionCondition; +import io.stargate.sgv2.jsonapi.testbench.testrun.TestNodeFactory; +import io.stargate.sgv2.jsonapi.testbench.testrun.TestRunResponse; +import io.stargate.sgv2.jsonapi.testbench.testrun.TestUri; import java.util.List; import java.util.concurrent.atomic.AtomicReference; import org.junit.jupiter.api.DynamicNode; diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/lifecycle/TestPlanLifecycle.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/lifecycle/TestPlanLifecycle.java similarity index 70% rename from src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/lifecycle/TestPlanLifecycle.java rename to src/test/java/io/stargate/sgv2/jsonapi/testbench/lifecycle/TestPlanLifecycle.java index 53133be634..1f450317b9 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/lifecycle/TestPlanLifecycle.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/lifecycle/TestPlanLifecycle.java @@ -1,11 +1,11 @@ -package io.stargate.sgv2.jsonapi.api.v1.vectorize.lifecycle; - -import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestNodeFactory; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestRunEnv; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestUri; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.Job; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.TestSuiteSpec; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.WorkflowSpec; +package io.stargate.sgv2.jsonapi.testbench.lifecycle; + +import io.stargate.sgv2.jsonapi.testbench.testrun.TestNodeFactory; +import io.stargate.sgv2.jsonapi.testbench.testrun.TestRunEnv; +import io.stargate.sgv2.jsonapi.testbench.testrun.TestUri; +import io.stargate.sgv2.jsonapi.testbench.testspec.Job; +import io.stargate.sgv2.jsonapi.testbench.testspec.TestSuiteSpec; +import io.stargate.sgv2.jsonapi.testbench.testspec.WorkflowSpec; import java.util.Optional; import org.junit.jupiter.api.DynamicNode; diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/messaging/APIRequest.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/messaging/APIRequest.java similarity index 95% rename from src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/messaging/APIRequest.java rename to src/test/java/io/stargate/sgv2/jsonapi/testbench/messaging/APIRequest.java index 4c36039d40..b2b06e35a6 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/messaging/APIRequest.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/messaging/APIRequest.java @@ -1,4 +1,4 @@ -package io.stargate.sgv2.jsonapi.api.v1.vectorize.messaging; +package io.stargate.sgv2.jsonapi.testbench.messaging; import static io.restassured.RestAssured.given; @@ -11,9 +11,9 @@ import io.restassured.response.ValidatableResponse; import io.restassured.specification.RequestSpecification; import io.stargate.sgv2.jsonapi.api.model.command.CommandTarget; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.targets.Connection; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestRunEnv; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.TestCommand; +import io.stargate.sgv2.jsonapi.testbench.targets.Connection; +import io.stargate.sgv2.jsonapi.testbench.testrun.TestRunEnv; +import io.stargate.sgv2.jsonapi.testbench.testspec.TestCommand; import io.stargate.sgv2.jsonapi.config.constants.HttpConstants; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/messaging/APIResponse.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/messaging/APIResponse.java similarity index 69% rename from src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/messaging/APIResponse.java rename to src/test/java/io/stargate/sgv2/jsonapi/testbench/messaging/APIResponse.java index c05feaa974..05d0023658 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/messaging/APIResponse.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/messaging/APIResponse.java @@ -1,4 +1,4 @@ -package io.stargate.sgv2.jsonapi.api.v1.vectorize.messaging; +package io.stargate.sgv2.jsonapi.testbench.messaging; import io.restassured.response.ValidatableResponse; diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/reporting/DynamicTreeListener.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/reporting/DynamicTreeListener.java similarity index 98% rename from src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/reporting/DynamicTreeListener.java rename to src/test/java/io/stargate/sgv2/jsonapi/testbench/reporting/DynamicTreeListener.java index 28d9c1cb40..750e1bbdc6 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/reporting/DynamicTreeListener.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/reporting/DynamicTreeListener.java @@ -1,6 +1,6 @@ -package io.stargate.sgv2.jsonapi.api.v1.vectorize.reporting; +package io.stargate.sgv2.jsonapi.testbench.reporting; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestUri; +import io.stargate.sgv2.jsonapi.testbench.testrun.TestUri; import java.util.*; import java.util.concurrent.ConcurrentHashMap; import org.apache.maven.plugin.surefire.report.*; diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/reporting/TestBenchConsoleWriter.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/reporting/TestBenchConsoleWriter.java similarity index 98% rename from src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/reporting/TestBenchConsoleWriter.java rename to src/test/java/io/stargate/sgv2/jsonapi/testbench/reporting/TestBenchConsoleWriter.java index 1dedea4001..cc5d84e2e1 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/reporting/TestBenchConsoleWriter.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/reporting/TestBenchConsoleWriter.java @@ -1,8 +1,8 @@ -package io.stargate.sgv2.jsonapi.api.v1.vectorize.reporting; +package io.stargate.sgv2.jsonapi.testbench.reporting; import static org.apache.maven.surefire.shared.utils.logging.MessageUtils.buffer; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestUri; +import io.stargate.sgv2.jsonapi.testbench.testrun.TestUri; import java.io.IOException; import java.nio.file.Files; diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/targets/AstraBackend.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/targets/AstraBackend.java similarity index 58% rename from src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/targets/AstraBackend.java rename to src/test/java/io/stargate/sgv2/jsonapi/testbench/targets/AstraBackend.java index 7c2e58f046..6fc7aa2e13 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/targets/AstraBackend.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/targets/AstraBackend.java @@ -1,6 +1,6 @@ -package io.stargate.sgv2.jsonapi.api.v1.vectorize.targets; +package io.stargate.sgv2.jsonapi.testbench.targets; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.Job; +import io.stargate.sgv2.jsonapi.testbench.testspec.Job; public class AstraBackend extends Backend { diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/targets/Backend.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/targets/Backend.java new file mode 100644 index 0000000000..845c208b6e --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/targets/Backend.java @@ -0,0 +1,9 @@ +package io.stargate.sgv2.jsonapi.testbench.targets; + +import io.stargate.sgv2.jsonapi.testbench.lifecycle.TestPlanLifecycle; +import io.stargate.sgv2.jsonapi.testbench.testspec.Job; + +public abstract class Backend implements TestPlanLifecycle { + + public void updateJobForTarget(Job job) {} +} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/targets/CassandraBackend.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/targets/CassandraBackend.java similarity index 79% rename from src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/targets/CassandraBackend.java rename to src/test/java/io/stargate/sgv2/jsonapi/testbench/targets/CassandraBackend.java index 39c5f3f0cb..3a7505ca55 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/targets/CassandraBackend.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/targets/CassandraBackend.java @@ -1,15 +1,15 @@ -package io.stargate.sgv2.jsonapi.api.v1.vectorize.targets; +package io.stargate.sgv2.jsonapi.testbench.targets; -import static io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestRunEnv.toSafeSchemaIdentifier; +import static io.stargate.sgv2.jsonapi.testbench.testrun.TestRunEnv.toSafeSchemaIdentifier; import com.fasterxml.jackson.databind.ObjectMapper; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions.TestAssertion; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestExecutionCondition; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestNodeFactory; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestRunRequest; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestUri; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.Job; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.TestCommand; +import io.stargate.sgv2.jsonapi.testbench.assertions.TestAssertion; +import io.stargate.sgv2.jsonapi.testbench.testrun.TestExecutionCondition; +import io.stargate.sgv2.jsonapi.testbench.testrun.TestNodeFactory; +import io.stargate.sgv2.jsonapi.testbench.testrun.TestRunRequest; +import io.stargate.sgv2.jsonapi.testbench.testrun.TestUri; +import io.stargate.sgv2.jsonapi.testbench.testspec.Job; +import io.stargate.sgv2.jsonapi.testbench.testspec.TestCommand; import java.util.Optional; import org.apache.commons.lang3.RandomStringUtils; import org.junit.jupiter.api.DynamicNode; diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/targets/Connection.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/targets/Connection.java similarity index 82% rename from src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/targets/Connection.java rename to src/test/java/io/stargate/sgv2/jsonapi/testbench/targets/Connection.java index 29803a2cd8..0db4b1bd3f 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/targets/Connection.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/targets/Connection.java @@ -1,4 +1,4 @@ -package io.stargate.sgv2.jsonapi.api.v1.vectorize.targets; +package io.stargate.sgv2.jsonapi.testbench.targets; public record Connection(String domain, Integer port, String basePath) { public Connection { diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/targets/Target.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/targets/Target.java similarity index 76% rename from src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/targets/Target.java rename to src/test/java/io/stargate/sgv2/jsonapi/testbench/targets/Target.java index 27a9168f5a..48549f3036 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/targets/Target.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/targets/Target.java @@ -1,15 +1,15 @@ -package io.stargate.sgv2.jsonapi.api.v1.vectorize.targets; +package io.stargate.sgv2.jsonapi.testbench.targets; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.lifecycle.TestPlanLifecycle; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.messaging.APIRequest; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestNodeFactory; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestRunEnv; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestUri; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.Job; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.TargetConfiguration; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.TestCommand; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.TestSuiteSpec; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.WorkflowSpec; +import io.stargate.sgv2.jsonapi.testbench.lifecycle.TestPlanLifecycle; +import io.stargate.sgv2.jsonapi.testbench.messaging.APIRequest; +import io.stargate.sgv2.jsonapi.testbench.testrun.TestNodeFactory; +import io.stargate.sgv2.jsonapi.testbench.testrun.TestRunEnv; +import io.stargate.sgv2.jsonapi.testbench.testrun.TestUri; +import io.stargate.sgv2.jsonapi.testbench.testspec.Job; +import io.stargate.sgv2.jsonapi.testbench.testspec.TargetConfiguration; +import io.stargate.sgv2.jsonapi.testbench.testspec.TestCommand; +import io.stargate.sgv2.jsonapi.testbench.testspec.TestSuiteSpec; +import io.stargate.sgv2.jsonapi.testbench.testspec.WorkflowSpec; import java.util.HashMap; import java.util.Optional; import org.junit.jupiter.api.DynamicNode; diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testrun/DynamicTestExecutable.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testrun/DynamicTestExecutable.java similarity index 97% rename from src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testrun/DynamicTestExecutable.java rename to src/test/java/io/stargate/sgv2/jsonapi/testbench/testrun/DynamicTestExecutable.java index c436e1617c..c18c81ecd4 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testrun/DynamicTestExecutable.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testrun/DynamicTestExecutable.java @@ -1,4 +1,4 @@ -package io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun; +package io.stargate.sgv2.jsonapi.testbench.testrun; import org.junit.jupiter.api.Assumptions; import org.junit.jupiter.api.DynamicTest; diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testrun/TestExecutionCondition.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testrun/TestExecutionCondition.java similarity index 96% rename from src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testrun/TestExecutionCondition.java rename to src/test/java/io/stargate/sgv2/jsonapi/testbench/testrun/TestExecutionCondition.java index 70b242ad4a..55d42ff2cd 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testrun/TestExecutionCondition.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testrun/TestExecutionCondition.java @@ -1,4 +1,4 @@ -package io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun; +package io.stargate.sgv2.jsonapi.testbench.testrun; import java.util.function.BooleanSupplier; diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testrun/TestNodeFactory.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testrun/TestNodeFactory.java similarity index 93% rename from src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testrun/TestNodeFactory.java rename to src/test/java/io/stargate/sgv2/jsonapi/testbench/testrun/TestNodeFactory.java index 26cff5ed77..799651bdfc 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testrun/TestNodeFactory.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testrun/TestNodeFactory.java @@ -1,10 +1,10 @@ -package io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun; +package io.stargate.sgv2.jsonapi.testbench.testrun; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.TestPlan; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.Job; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.TestSpecMeta; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.TestSuiteSpec; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.WorkflowSpec; +import io.stargate.sgv2.jsonapi.testbench.TestPlan; +import io.stargate.sgv2.jsonapi.testbench.testspec.Job; +import io.stargate.sgv2.jsonapi.testbench.testspec.TestSpecMeta; +import io.stargate.sgv2.jsonapi.testbench.testspec.TestSuiteSpec; +import io.stargate.sgv2.jsonapi.testbench.testspec.WorkflowSpec; import org.junit.jupiter.api.DynamicContainer; import org.junit.jupiter.api.DynamicNode; import org.junit.jupiter.api.DynamicTest; @@ -16,7 +16,6 @@ import java.util.List; import java.util.Optional; import java.util.function.Supplier; -import java.util.stream.Stream; /** * A container we can pass around when building the TestNodes that has the test plan diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testrun/TestRunEnv.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testrun/TestRunEnv.java similarity index 96% rename from src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testrun/TestRunEnv.java rename to src/test/java/io/stargate/sgv2/jsonapi/testbench/testrun/TestRunEnv.java index 42f6aef769..a4c42e6472 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testrun/TestRunEnv.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testrun/TestRunEnv.java @@ -1,6 +1,6 @@ -package io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun; +package io.stargate.sgv2.jsonapi.testbench.testrun; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.TestSuiteSpec; +import io.stargate.sgv2.jsonapi.testbench.testspec.TestSuiteSpec; import java.util.HashMap; import java.util.Map; import java.util.Set; diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testrun/TestRunRequest.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testrun/TestRunRequest.java similarity index 87% rename from src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testrun/TestRunRequest.java rename to src/test/java/io/stargate/sgv2/jsonapi/testbench/testrun/TestRunRequest.java index 4dc8cf98c3..c2a8cf2647 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testrun/TestRunRequest.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testrun/TestRunRequest.java @@ -1,15 +1,15 @@ -package io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun; +package io.stargate.sgv2.jsonapi.testbench.testrun; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions.TestAssertion; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.targets.Target; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec.TestCommand; +import io.stargate.sgv2.jsonapi.testbench.assertions.TestAssertion; +import io.stargate.sgv2.jsonapi.testbench.targets.Target; +import io.stargate.sgv2.jsonapi.testbench.testspec.TestCommand; import java.util.ArrayList; import java.util.List; import java.util.concurrent.atomic.AtomicReference; -import java.util.stream.Stream; + import org.junit.jupiter.api.DynamicContainer; import org.junit.jupiter.api.DynamicNode; diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testrun/TestRunResponse.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testrun/TestRunResponse.java similarity index 82% rename from src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testrun/TestRunResponse.java rename to src/test/java/io/stargate/sgv2/jsonapi/testbench/testrun/TestRunResponse.java index ea7e869fb9..d92a34e684 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testrun/TestRunResponse.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testrun/TestRunResponse.java @@ -1,7 +1,7 @@ -package io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun; +package io.stargate.sgv2.jsonapi.testbench.testrun; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.messaging.APIRequest; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.messaging.APIResponse; +import io.stargate.sgv2.jsonapi.testbench.messaging.APIRequest; +import io.stargate.sgv2.jsonapi.testbench.messaging.APIResponse; public record TestRunResponse( TestRunRequest testRequest, APIRequest apiRequest, APIResponse apiResponse) { diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testrun/TestUri.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testrun/TestUri.java similarity index 98% rename from src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testrun/TestUri.java rename to src/test/java/io/stargate/sgv2/jsonapi/testbench/testrun/TestUri.java index a6fbada0d1..0b211d24d3 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testrun/TestUri.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testrun/TestUri.java @@ -1,4 +1,4 @@ -package io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun; +package io.stargate.sgv2.jsonapi.testbench.testrun; import static java.util.stream.Collectors.joining; diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/AssertionTemplateSpec.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/AssertionTemplateSpec.java similarity index 84% rename from src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/AssertionTemplateSpec.java rename to src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/AssertionTemplateSpec.java index 0d714e7280..b8dd269cd2 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/AssertionTemplateSpec.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/AssertionTemplateSpec.java @@ -1,4 +1,4 @@ -package io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec; +package io.stargate.sgv2.jsonapi.testbench.testspec; import com.fasterxml.jackson.databind.JsonNode; import java.util.Map; diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/Job.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/Job.java similarity index 89% rename from src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/Job.java rename to src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/Job.java index 6be04dec71..af92e66e4e 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/Job.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/Job.java @@ -1,11 +1,11 @@ -package io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec; +package io.stargate.sgv2.jsonapi.testbench.testspec; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.TestPlan; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestNodeFactory; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestRunEnv; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestUri; +import io.stargate.sgv2.jsonapi.testbench.TestPlan; +import io.stargate.sgv2.jsonapi.testbench.testrun.TestNodeFactory; +import io.stargate.sgv2.jsonapi.testbench.testrun.TestRunEnv; +import io.stargate.sgv2.jsonapi.testbench.testrun.TestUri; import java.util.ArrayList; import java.util.List; import java.util.Map; diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/SpecFile.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/SpecFile.java similarity index 87% rename from src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/SpecFile.java rename to src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/SpecFile.java index f240bb6d01..9d5ce5f158 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/SpecFile.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/SpecFile.java @@ -1,4 +1,4 @@ -package io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec; +package io.stargate.sgv2.jsonapi.testbench.testspec; import com.fasterxml.jackson.databind.JsonNode; import java.io.File; diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/SpecFiles.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/SpecFiles.java similarity index 98% rename from src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/SpecFiles.java rename to src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/SpecFiles.java index 27b7fff883..822ea4cf3f 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/SpecFiles.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/SpecFiles.java @@ -1,4 +1,4 @@ -package io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec; +package io.stargate.sgv2.jsonapi.testbench.testspec; import com.fasterxml.jackson.databind.MapperFeature; import com.fasterxml.jackson.databind.ObjectMapper; diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/TargetConfiguration.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/TargetConfiguration.java new file mode 100644 index 0000000000..0099d65a5a --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/TargetConfiguration.java @@ -0,0 +1,5 @@ +package io.stargate.sgv2.jsonapi.testbench.testspec; + +import io.stargate.sgv2.jsonapi.testbench.targets.Connection; + +public record TargetConfiguration(String name, String backend, Connection connection) {} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/TargetsSpec.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/TargetsSpec.java similarity index 97% rename from src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/TargetsSpec.java rename to src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/TargetsSpec.java index a0b8188ed9..a26ae3f8b2 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/TargetsSpec.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/TargetsSpec.java @@ -1,4 +1,4 @@ -package io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec; +package io.stargate.sgv2.jsonapi.testbench.testspec; import com.fasterxml.jackson.databind.MapperFeature; import com.fasterxml.jackson.databind.ObjectMapper; diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/TestCase.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/TestCase.java similarity index 82% rename from src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/TestCase.java rename to src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/TestCase.java index 91aa2aa092..6e8bcf3417 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/TestCase.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/TestCase.java @@ -1,9 +1,9 @@ -package io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec; +package io.stargate.sgv2.jsonapi.testbench.testspec; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.databind.node.ObjectNode; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions.TestAssertion; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.*; +import io.stargate.sgv2.jsonapi.testbench.assertions.TestAssertion; +import io.stargate.sgv2.jsonapi.testbench.testrun.*; import org.junit.jupiter.api.DynamicContainer; public record TestCase( diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/TestCommand.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/TestCommand.java similarity index 96% rename from src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/TestCommand.java rename to src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/TestCommand.java index b8661633fe..aec0f1749f 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/TestCommand.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/TestCommand.java @@ -1,4 +1,4 @@ -package io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec; +package io.stargate.sgv2.jsonapi.testbench.testspec; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.core.JsonProcessingException; @@ -7,7 +7,7 @@ import com.fasterxml.jackson.databind.node.ObjectNode; import com.fasterxml.jackson.databind.node.TextNode; import io.stargate.sgv2.jsonapi.api.model.command.CommandName; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestRunEnv; +import io.stargate.sgv2.jsonapi.testbench.testrun.TestRunEnv; import org.apache.commons.text.StringSubstitutor; public class TestCommand { diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/TestEnvAccess.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/TestEnvAccess.java similarity index 89% rename from src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/TestEnvAccess.java rename to src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/TestEnvAccess.java index 8d1e174a09..889d552f67 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/TestEnvAccess.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/TestEnvAccess.java @@ -1,4 +1,4 @@ -package io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec; +package io.stargate.sgv2.jsonapi.testbench.testspec; public abstract class TestEnvAccess { diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/TestSpec.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/TestSpec.java similarity index 87% rename from src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/TestSpec.java rename to src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/TestSpec.java index ff1ce62d4d..1caecf49cf 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/TestSpec.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/TestSpec.java @@ -1,4 +1,4 @@ -package io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec; +package io.stargate.sgv2.jsonapi.testbench.testspec; public sealed interface TestSpec permits AssertionTemplateSpec, TargetsSpec, TestSuiteSpec, WorkflowSpec { diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/TestSpecKind.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/TestSpecKind.java similarity index 89% rename from src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/TestSpecKind.java rename to src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/TestSpecKind.java index cae0d8d361..55e248f62e 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/TestSpecKind.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/TestSpecKind.java @@ -1,4 +1,4 @@ -package io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec; +package io.stargate.sgv2.jsonapi.testbench.testspec; public enum TestSpecKind { ASSERTION_TEMPLATE, diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/TestSpecMeta.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/TestSpecMeta.java similarity index 63% rename from src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/TestSpecMeta.java rename to src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/TestSpecMeta.java index bebc224a36..fbbf869fe2 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/TestSpecMeta.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/TestSpecMeta.java @@ -1,4 +1,4 @@ -package io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec; +package io.stargate.sgv2.jsonapi.testbench.testspec; import java.util.List; diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/TestSuiteSpec.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/TestSuiteSpec.java similarity index 95% rename from src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/TestSuiteSpec.java rename to src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/TestSuiteSpec.java index f7be335f5a..3c0961bd3c 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/TestSuiteSpec.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/TestSuiteSpec.java @@ -1,13 +1,13 @@ -package io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec; +package io.stargate.sgv2.jsonapi.testbench.testspec; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.assertions.TestAssertion; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.*; +import io.stargate.sgv2.jsonapi.testbench.assertions.TestAssertion; import java.util.ArrayList; -import java.util.Collection; import java.util.List; + +import io.stargate.sgv2.jsonapi.testbench.testrun.*; import org.junit.jupiter.api.DynamicContainer; import org.junit.jupiter.api.DynamicNode; diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/WorkflowSpec.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/WorkflowSpec.java similarity index 81% rename from src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/WorkflowSpec.java rename to src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/WorkflowSpec.java index 190dc71fe7..79a24b8b5f 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/vectorize/testspec/WorkflowSpec.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/WorkflowSpec.java @@ -1,9 +1,9 @@ -package io.stargate.sgv2.jsonapi.api.v1.vectorize.testspec; +package io.stargate.sgv2.jsonapi.testbench.testspec; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestNodeFactory; -import io.stargate.sgv2.jsonapi.api.v1.vectorize.testrun.TestUri; +import io.stargate.sgv2.jsonapi.testbench.testrun.TestNodeFactory; +import io.stargate.sgv2.jsonapi.testbench.testrun.TestUri; import java.util.List; import org.junit.jupiter.api.DynamicNode; diff --git a/src/test/resources/META-INF/services/org.junit.platform.launcher.TestExecutionListener b/src/test/resources/META-INF/services/org.junit.platform.launcher.TestExecutionListener index 2017f464d7..031e53e9c4 100644 --- a/src/test/resources/META-INF/services/org.junit.platform.launcher.TestExecutionListener +++ b/src/test/resources/META-INF/services/org.junit.platform.launcher.TestExecutionListener @@ -1 +1 @@ -io.stargate.sgv2.jsonapi.api.v1.vectorize.reporting.DynamicTreeListener \ No newline at end of file +io.stargate.sgv2.jsonapi.testbench.reporting.DynamicTreeListener \ No newline at end of file diff --git a/src/test/resources/integration-tests/assertions/assertion-templates.json b/src/test/resources/testbench/assertions/assertion-templates.json similarity index 100% rename from src/test/resources/integration-tests/assertions/assertion-templates.json rename to src/test/resources/testbench/assertions/assertion-templates.json diff --git a/src/test/resources/integration-tests/targets/targets.json b/src/test/resources/testbench/targets/targets.json similarity index 100% rename from src/test/resources/integration-tests/targets/targets.json rename to src/test/resources/testbench/targets/targets.json diff --git a/src/test/resources/integration-tests/test-plans/test-plan-astra-vectorize.yaml b/src/test/resources/testbench/testplans/test-plan-astra-vectorize.yaml similarity index 100% rename from src/test/resources/integration-tests/test-plans/test-plan-astra-vectorize.yaml rename to src/test/resources/testbench/testplans/test-plan-astra-vectorize.yaml diff --git a/src/test/resources/integration-tests/vectorize/findEmbeddingProviders.json b/src/test/resources/testbench/testsuites/findEmbeddingProviders.json similarity index 100% rename from src/test/resources/integration-tests/vectorize/findEmbeddingProviders.json rename to src/test/resources/testbench/testsuites/findEmbeddingProviders.json diff --git a/src/test/resources/integration-tests/vectorize/vectorize-base.json b/src/test/resources/testbench/testsuites/vectorize-base.json similarity index 100% rename from src/test/resources/integration-tests/vectorize/vectorize-base.json rename to src/test/resources/testbench/testsuites/vectorize-base.json diff --git a/src/test/resources/integration-tests/vectorize/vectorize-header-auth.json b/src/test/resources/testbench/testsuites/vectorize-header-auth.json similarity index 100% rename from src/test/resources/integration-tests/vectorize/vectorize-header-auth.json rename to src/test/resources/testbench/testsuites/vectorize-header-auth.json diff --git a/src/test/resources/integration-tests/vectorize/vectorize-shared-auth.json b/src/test/resources/testbench/testsuites/vectorize-shared-auth.json similarity index 100% rename from src/test/resources/integration-tests/vectorize/vectorize-shared-auth.json rename to src/test/resources/testbench/testsuites/vectorize-shared-auth.json diff --git a/src/test/resources/integration-tests/vectorize/findEmbeddingProviders-workflow.json b/src/test/resources/testbench/workflows/findEmbeddingProviders-workflow.json similarity index 100% rename from src/test/resources/integration-tests/vectorize/findEmbeddingProviders-workflow.json rename to src/test/resources/testbench/workflows/findEmbeddingProviders-workflow.json diff --git a/src/test/resources/integration-tests/vectorize/vectorize-header-workflow.json b/src/test/resources/testbench/workflows/vectorize-header-workflow.json similarity index 100% rename from src/test/resources/integration-tests/vectorize/vectorize-header-workflow.json rename to src/test/resources/testbench/workflows/vectorize-header-workflow.json diff --git a/src/test/resources/integration-tests/vectorize/vectorize-shared-workflow.json b/src/test/resources/testbench/workflows/vectorize-shared-workflow.json similarity index 100% rename from src/test/resources/integration-tests/vectorize/vectorize-shared-workflow.json rename to src/test/resources/testbench/workflows/vectorize-shared-workflow.json From dd704a9b7462589e4320d6f8299721f06363be2e Mon Sep 17 00:00:00 2001 From: Aaron Morton Date: Mon, 4 May 2026 13:05:16 +1200 Subject: [PATCH 81/89] reduce parallel to avoid jina concurrency error --- .github/workflows/test-bench-vectorize.yaml | 2 + .../testbench/TestBenchByTestPlan.java | 2 +- .../sgv2/jsonapi/testbench/TestPlan.java | 19 ++------ .../sgv2/jsonapi/testbench/TestPlanFile.java | 48 +++++++++++++++++++ 4 files changed, 54 insertions(+), 17 deletions(-) create mode 100644 src/test/java/io/stargate/sgv2/jsonapi/testbench/TestPlanFile.java diff --git a/.github/workflows/test-bench-vectorize.yaml b/.github/workflows/test-bench-vectorize.yaml index 176bf2ac86..31da66d83d 100644 --- a/.github/workflows/test-bench-vectorize.yaml +++ b/.github/workflows/test-bench-vectorize.yaml @@ -70,6 +70,8 @@ jobs: timeout-minutes: 120 strategy: + # aaron - temp parallel 1 to avoid Jina concurrency errors + max-parallel: 1 # do not fail fast, we want to run on all the different target dbs fail-fast: false matrix: diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/TestBenchByTestPlan.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/TestBenchByTestPlan.java index 810ef4c24c..0c6d21cc1e 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/testbench/TestBenchByTestPlan.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/TestBenchByTestPlan.java @@ -13,7 +13,7 @@ * Entry point for running a Test Bench from a Test Plan file. *

* Put the name of the test plan file in the TEST_PLAN_FILE env var, this can - * set the target to hit, and the workflows to run. See {@link TestPlan.TestPlanFile} + * set the target to hit, and the workflows to run. See {@link TestPlanFile} *

*/ public class TestBenchByTestPlan { diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/TestPlan.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/TestPlan.java index 68b23b8406..238cf54a0a 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/testbench/TestPlan.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/TestPlan.java @@ -16,7 +16,6 @@ import java.nio.file.Files; import java.nio.file.Path; import java.util.List; -import java.util.Map; import java.util.Set; import java.util.stream.Stream; @@ -27,12 +26,8 @@ public record TestPlan( Target target, SpecFiles specFiles, Set workflows, boolean ignoreDisabled) { - private static final Logger LOGGER = LoggerFactory.getLogger(TestPlan.class); - private static final ObjectMapper JSON_MAPPER = - new ObjectMapper( - JsonFactory.builder().enable(StreamReadFeature.INCLUDE_SOURCE_IN_LOCATION).build()) - .configure(MapperFeature.ACCEPT_CASE_INSENSITIVE_ENUMS, true); + private static final Logger LOGGER = LoggerFactory.getLogger(TestPlan.class); private static final ObjectMapper YAML_MAPPER = new ObjectMapper( @@ -51,14 +46,14 @@ public static TestPlan fromFile(Path path) { throw new RuntimeException(e); } - if (planFile.targetName() != null && planFile.customTarget != null) { + if (planFile.targetName() != null && planFile.customTarget() != null) { throw new RuntimeException( "Both targetName and customTarget set, use only one. testPlanFile=" + path); } var testPlan = planFile.customTarget() != null - ? create(planFile.customTarget(), planFile.workflows, planFile.ignoreDisabled) + ? create(planFile.customTarget(), planFile.workflows(), planFile.ignoreDisabled()) : create(planFile.targetName(), planFile.workflows(), planFile.ignoreDisabled()); return testPlan; @@ -128,14 +123,6 @@ public void updateJobForTarget(Job job) { target.updateJobForTarget(job); } - public record TestPlanFile( - String name, - String targetName, - TargetConfiguration customTarget, - List workflows, - Boolean ignoreDisabled, - Map envVars) {} - public record TestPlanNodeTree(DynamicNode root, int totalNodeCount){} } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/TestPlanFile.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/TestPlanFile.java new file mode 100644 index 0000000000..693115c413 --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/TestPlanFile.java @@ -0,0 +1,48 @@ +package io.stargate.sgv2.jsonapi.testbench; + +import io.stargate.sgv2.jsonapi.testbench.testspec.TargetConfiguration; + +import java.nio.file.Path; +import java.util.List; +import java.util.Map; + +/** + * Structure of a test plane file stored in a jar resource or external file. These are used with the + * {@link TestBenchByTestPlan} runner to control the target and workflows to run at execution time via a config file. + *

+ * For example testbench/testplans/test-plan-astra-vectorize.yaml, testing against and Astra DB: + * + * 

+ * name: Test Plan - Astra - Vectorize Workflows
+ * customTarget:
+ *   name: ${TARGET_NAME}
+ *   backend: astra
+ *   connection:
+ *     domain: ${ENDPOINT}
+ *     port: 443
+ *     basePath: /api/json/v1
+ * workflows:
+ *   - vectorize-header-workflow
+ *   - vectorize-shared-workflow
+ * ignoreDisabled: true
+ *
+ *

+ *

+ * The {@link TestPlan#fromFile(Path)} will run Apache command style substituions using {@link System#getenv(String)} + * as the source for replacements. This allows some sensitive information to be put into the env rather than a + * file, and for the runner to make multiple calls with different env vars. + *

+ * @param name Nice human name for the test plan, only used in this file + * @param targetName Name of the target, such as the db name, used in logging etc. + * @param customTarget Defines a {@link TargetConfiguration} of how to connect (e.g. astra or cassandra backend ) + * and the connection information. + * @param workflows List of the workflows to run, leave empty or null to run all workflows in the system. + * @param ignoreDisabled If true, work flow jobs marked as "disabled" will be executed. Default is false. + */ +public record TestPlanFile( + String name, + String targetName, + TargetConfiguration customTarget, + List workflows, + Boolean ignoreDisabled) { +} From 9ed62f55e2ece963d1e1f84ffa8ec321919da1d9 Mon Sep 17 00:00:00 2001 From: Aaron Morton Date: Mon, 4 May 2026 13:51:56 +1200 Subject: [PATCH 82/89] refactor and disable jina cannot get jina to not timeout --- .../testbench/TestBenchByTestPlan.java | 3 +- .../sgv2/jsonapi/testbench/VectorizeIT.java | 19 ---- .../testspec/AssertionTemplateSpec.java | 34 ++++++- .../jsonapi/testbench/testspec/SpecFile.java | 7 ++ .../jsonapi/testbench/testspec/SpecFiles.java | 89 ++++++++++++------- .../testspec/TargetConfiguration.java | 6 ++ .../testbench/testspec/TargetsSpec.java | 19 +--- .../jsonapi/testbench/testspec/TestCase.java | 7 ++ .../jsonapi/testbench/testspec/TestSpec.java | 13 +++ .../testbench/testspec/TestSpecMeta.java | 13 ++- .../workflows/vectorize-header-workflow.json | 1 + .../workflows/vectorize-shared-workflow.json | 1 + 12 files changed, 139 insertions(+), 73 deletions(-) delete mode 100644 src/test/java/io/stargate/sgv2/jsonapi/testbench/VectorizeIT.java diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/TestBenchByTestPlan.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/TestBenchByTestPlan.java index 0c6d21cc1e..cfb025035a 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/testbench/TestBenchByTestPlan.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/TestBenchByTestPlan.java @@ -1,5 +1,6 @@ package io.stargate.sgv2.jsonapi.testbench; +import io.stargate.sgv2.jsonapi.testbench.testspec.SpecFiles; import io.stargate.sgv2.jsonapi.testbench.testspec.TargetsSpec; import java.nio.file.Path; import java.util.stream.Stream; @@ -28,7 +29,7 @@ public Stream runTestPlanFile() { var path = rawPath.startsWith("classpath:") - ? TargetsSpec.resourceDir(rawPath.substring("classpath:".length())) + ? SpecFiles.resourceDir(rawPath.substring("classpath:".length())) : Path.of(rawPath); var testPlan = TestPlan.fromFile(path); diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/VectorizeIT.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/VectorizeIT.java deleted file mode 100644 index d006575ac6..0000000000 --- a/src/test/java/io/stargate/sgv2/jsonapi/testbench/VectorizeIT.java +++ /dev/null @@ -1,19 +0,0 @@ -package io.stargate.sgv2.jsonapi.testbench; - -import io.quarkus.test.common.WithTestResource; -import io.quarkus.test.junit.QuarkusIntegrationTest; -import io.stargate.sgv2.jsonapi.api.v1.AbstractCollectionIntegrationTestBase; -import io.stargate.sgv2.jsonapi.testresource.DseTestResource; - -@QuarkusIntegrationTest -@WithTestResource(value = DseTestResource.class) -public class VectorizeIT extends AbstractCollectionIntegrationTestBase { - -// @TestFactory -// Stream jobs() { -// -// var testPlan = TestPlan.create("integration", List.of("all-vectorize-workflow")); -// -// return testPlan.testNode(); -// } -} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/AssertionTemplateSpec.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/AssertionTemplateSpec.java index b8dd269cd2..786b4ca452 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/AssertionTemplateSpec.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/AssertionTemplateSpec.java @@ -1,14 +1,42 @@ package io.stargate.sgv2.jsonapi.testbench.testspec; import com.fasterxml.jackson.databind.JsonNode; +import io.stargate.sgv2.jsonapi.testbench.assertions.AssertionFactory; + import java.util.Map; import java.util.Optional; +/** + * Spec defintions about re-usable assertions that are defined in JSON. See {@link AssertionFactory}. + *

+ * Example, showing the defintion for "isSuccess" assertion and how it is defined on a per command basis. + * 

+ * {
+ *   "meta": {
+ *     "name": "assertions-templates",
+ *     "kind": "assertion_template"
+ *   },
+ *   "templates": {
+ *     "isSuccess": {
+ *       "createCollection": {
+ *         "http.success": null,
+ *         "response.isDDLSuccess": null
+ *       },
+ *       "find": {
+ *         "http.success": null,
+ *         "response.isFindSuccess": null
+ *       },
+ *
+ * + *

+ * @param meta Metadata for the spec. + * @param templates JSON Object that is a map of assertion name to a map of implementations per API command, + * see example + */ public record AssertionTemplateSpec(TestSpecMeta meta, Map templates) implements TestSpec { - public Optional templateFor(String name) { - - return Optional.ofNullable(templates.get(name)); + public Optional templateFor(String assertionName) { + return Optional.ofNullable(templates.get(assertionName)); } } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/SpecFile.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/SpecFile.java index 9d5ce5f158..04a5733d73 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/SpecFile.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/SpecFile.java @@ -4,6 +4,13 @@ import java.io.File; import org.jspecify.annotations.NonNull; +/** + * A Spec file is any JSON file we have that is used to drive the test suite, workflows etc. + *

+ * This is a container of the file, it's raw JSON, and the {@link TestSpec} which is the common + * parsed object from the JSON. + *

+ */ public record SpecFile(File file, TestSpec spec, JsonNode root) { @Override diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/SpecFiles.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/SpecFiles.java index 822ea4cf3f..617d1d5ca8 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/SpecFiles.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/SpecFiles.java @@ -12,7 +12,11 @@ import java.util.function.Predicate; import java.util.stream.Stream; -/** Collection of all the test spec files read for this execution. */ +/** Collection of all the {@link io.stargate.sgv2.jsonapi.testbench.testspec.SpecFile} we have loaded + * from disk. + *

+ * Call {@link #loadAll(List)} to load spec files from multiple directories, + */ public class SpecFiles { private static final ObjectMapper MAPPER = @@ -25,38 +29,54 @@ private SpecFiles(List specFiles) { for (SpecFile file : specFiles) { if (file.spec() instanceof TestSuiteSpec it) { + // expand the includes it.expand(this); } } } + /** + * Loads all the spec file, paths is a list of resource dirs in the jar. + */ public static SpecFiles loadAll(List paths) { var specFiles = resourceDirs(paths).flatMap(SpecFiles::loadAll).toList(); return new SpecFiles(specFiles); } + /** + * Get all the SpecFiles by their metadata kind + */ public Stream byKind(TestSpecKind kind) { - return specFiles.stream().filter(itFile -> itFile.spec().meta().kind() == kind); + return match(kind, x -> true); } + /** + * Get all the SpecFiles by the class of the Specification, the object that + * implements {@link TestSpec} + */ public Stream byType(Class clazz) { - return byKind(TestSpecKind.fromType(clazz)).map(specFile -> specFile.spec().asSpecType(clazz)); - } - - public Stream byName(TestSpecKind kind, String name) { - return match(kind, specFiles -> specFiles.meta().name().equals(name)); + return match(TestSpecKind.fromType(clazz), x -> true) + .map(specFile -> specFile.spec().asSpecType(clazz)); } + /** + * Get all the spec files of the type matched by name, e.g. get all the test-suites called "monkey" + */ public Stream byNameAsType(Class clazz, String name) { return match(TestSpecKind.fromType(clazz), specFiles -> specFiles.meta().name().equals(name)) .map(specFile -> specFile.spec().asSpecType(clazz)); } private Stream match(TestSpecKind kind, Predicate predicate) { - return byKind(kind).filter(specFile -> predicate.test(specFile.spec())); + return specFiles.stream() + .filter(itFile -> itFile.spec().meta().kind() == kind) + .filter(specFile -> predicate.test(specFile.spec())); } + /** + * Load all the spec files in the directory at the path + */ private static Stream loadAll(Path path) { try (Stream pathStream = Files.walk(path)) { @@ -71,9 +91,18 @@ private static Stream loadAll(Path path) { } } + private static boolean isJsonFile(Path file) { + return file.getFileName().toString().endsWith(".json"); + } + + + /** + * Load a single spec file denoted by path. + */ private static SpecFile loadOne(Path path) { var file = path.toFile(); try { + // It's always JSON var root = MAPPER.readTree(file); var kindNode = root.path("meta").path("kind"); @@ -94,33 +123,31 @@ private static SpecFile loadOne(Path path) { } } - private static boolean isJsonFile(Path file) { - return file.getFileName().toString().endsWith(".json"); + + public static Stream resourceDirs(List paths) { + + return paths.stream() + .map(SpecFiles::resourceDir); } - private static Stream resourceDirs(List paths) { + public static Path resourceDir(String path) { var cl = Thread.currentThread().getContextClassLoader(); + String normalized = path.startsWith("/") ? path.substring(1) : path; - return paths.stream() - .map( - path -> { - String normalized = path.startsWith("/") ? path.substring(1) : path; - - var url = cl.getResource(normalized); - if (url == null) { - throw new IllegalArgumentException("Test resource folder not found: " + path); - } - - try { - // Works for file: URLs; if you run tests from a jar, switch to - // getResourceAsStream-based - // walking. - return Paths.get(url.toURI()); - } catch (URISyntaxException e) { - throw new IllegalArgumentException( - "Bad resource URI for: " + path + " -> " + url, e); - } - }); + var url = cl.getResource(normalized); + if (url == null) { + throw new IllegalArgumentException("Test resource folder not found: " + path); + } + + try { + // Works for file: URLs; if you run tests from a jar, switch to + // getResourceAsStream-based + // walking. + return Paths.get(url.toURI()); + } catch (URISyntaxException e) { + throw new IllegalArgumentException( + "Bad resource URI for: " + path + " -> " + url, e); + } } } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/TargetConfiguration.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/TargetConfiguration.java index 0099d65a5a..0cb33a4d17 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/TargetConfiguration.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/TargetConfiguration.java @@ -2,4 +2,10 @@ import io.stargate.sgv2.jsonapi.testbench.targets.Connection; +/** + * A target the test bench should connect to and run the tests + * @param name Friendly name of the target + * @param backend Backend name, such as astra or cassandra so we know how to run the lifecycle + * @param connection Connection information for the target. + */ public record TargetConfiguration(String name, String backend, Connection connection) {} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/TargetsSpec.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/TargetsSpec.java index a26ae3f8b2..7378a4ec84 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/TargetsSpec.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/TargetsSpec.java @@ -35,7 +35,7 @@ public TargetConfiguration configuration(String name) { } public static TargetsSpec loadAll(String path) { - final Path dir = resourceDir(path); + final Path dir = SpecFiles.resourceDir(path); try { return MAPPER.readValue(dir.toFile(), TargetsSpec.class); @@ -44,21 +44,4 @@ public static TargetsSpec loadAll(String path) { } } - public static Path resourceDir(String path) { - String normalized = path.startsWith("/") ? path.substring(1) : path; - - ClassLoader cl = Thread.currentThread().getContextClassLoader(); - URL url = cl.getResource(normalized); - if (url == null) { - throw new IllegalArgumentException("Test resource folder not found: " + path); - } - - try { - // Works for file: URLs; if you run tests from a jar, switch to getResourceAsStream-based - // walking. - return Paths.get(url.toURI()); - } catch (URISyntaxException e) { - throw new IllegalArgumentException("Bad resource URI for: " + path + " -> " + url, e); - } - } } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/TestCase.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/TestCase.java index 6e8bcf3417..46591b8881 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/TestCase.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/TestCase.java @@ -6,6 +6,13 @@ import io.stargate.sgv2.jsonapi.testbench.testrun.*; import org.junit.jupiter.api.DynamicContainer; +/** + * Spec for a single test case in a test suite, a test case is a command to run and assertions to run after. + * @param name + * @param command + * @param asserts + * @param include + */ public record TestCase( String name, TestCommand command, diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/TestSpec.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/TestSpec.java index 1caecf49cf..d09fc6afdb 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/TestSpec.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/TestSpec.java @@ -1,10 +1,23 @@ package io.stargate.sgv2.jsonapi.testbench.testspec; +/** + * A specification for objects in the TestBench world, such as a test suite. + *

+ * Implement this for any object types a {@link io.stargate.sgv2.jsonapi.testbench.TestPlan} needs + * to read from disk or know about. + *

+ */ public sealed interface TestSpec permits AssertionTemplateSpec, TargetsSpec, TestSuiteSpec, WorkflowSpec { TestSpecMeta meta(); + /** + * Gets the object as the implementing class + * @param type the implementing class of the {@link TestSpec} + * @return The implementing object, as the type. + * @param the implementing class of the {@link TestSpec} + */ default T asSpecType(Class type) { if (!type.isInstance(this)) { diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/TestSpecMeta.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/TestSpecMeta.java index fbbf869fe2..cfa5f3c453 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/TestSpecMeta.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/TestSpecMeta.java @@ -2,4 +2,15 @@ import java.util.List; -public record TestSpecMeta(String name, TestSpecKind kind, List tags) {} +/** + * Common metadata for any object of a {@link TestSpec} + * @param name Friendly name for, such as "vectorize-header-workflow" + * @param kind {@link TestSpecKind} describing the kind. + * @param tags Optional tags that can be used for filtering etc. + */ +public record TestSpecMeta(String name, TestSpecKind kind, List tags) { + + public TestSpecMeta{ + tags = tags == null ? List.of() : tags; + } +} diff --git a/src/test/resources/testbench/workflows/vectorize-header-workflow.json b/src/test/resources/testbench/workflows/vectorize-header-workflow.json index 2aa573bf6d..91d55da229 100644 --- a/src/test/resources/testbench/workflows/vectorize-header-workflow.json +++ b/src/test/resources/testbench/workflows/vectorize-header-workflow.json @@ -60,6 +60,7 @@ "meta": { "name": "jinaAI-vectorize", "tags": [ + "disabled" ] }, "fromEnvironment": { diff --git a/src/test/resources/testbench/workflows/vectorize-shared-workflow.json b/src/test/resources/testbench/workflows/vectorize-shared-workflow.json index 4e1fffdfef..0dd3cecb80 100644 --- a/src/test/resources/testbench/workflows/vectorize-shared-workflow.json +++ b/src/test/resources/testbench/workflows/vectorize-shared-workflow.json @@ -64,6 +64,7 @@ "meta": { "name": "jinaAI-vectorize", "tags": [ + "disabled" ] }, "fromEnvironment": { From 041b6f4ae60d28e18759bb741f29062d233d4903 Mon Sep 17 00:00:00 2001 From: Aaron Morton Date: Mon, 4 May 2026 15:29:31 +1200 Subject: [PATCH 83/89] back to parallel tests --- .github/workflows/test-bench-vectorize.yaml | 2 - .../sgv2/jsonapi/testbench/TestPlan.java | 3 +- .../testbench/testspec/TargetsSpec.java | 13 +-- .../testbench/testspec/TestCommand.java | 91 ++++++++++++++++--- 4 files changed, 85 insertions(+), 24 deletions(-) diff --git a/.github/workflows/test-bench-vectorize.yaml b/.github/workflows/test-bench-vectorize.yaml index 31da66d83d..176bf2ac86 100644 --- a/.github/workflows/test-bench-vectorize.yaml +++ b/.github/workflows/test-bench-vectorize.yaml @@ -70,8 +70,6 @@ jobs: timeout-minutes: 120 strategy: - # aaron - temp parallel 1 to avoid Jina concurrency errors - max-parallel: 1 # do not fail fast, we want to run on all the different target dbs fail-fast: false matrix: diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/TestPlan.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/TestPlan.java index 238cf54a0a..fd7e6a36f9 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/testbench/TestPlan.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/TestPlan.java @@ -2,7 +2,6 @@ -import com.fasterxml.jackson.core.JsonFactory; import com.fasterxml.jackson.core.StreamReadFeature; import com.fasterxml.jackson.databind.MapperFeature; import com.fasterxml.jackson.databind.ObjectMapper; @@ -65,7 +64,7 @@ public static TestPlan create(String targetName, List workflows) { public static TestPlan create(String targetName, List workflows, Boolean ignoreDisabled) { var targetConfigs = TargetsSpec.loadAll("testbench/targets/targets.json"); - return create(targetConfigs.configuration(targetName), workflows, ignoreDisabled); + return create(targetConfigs.getTarget(targetName), workflows, ignoreDisabled); } public static TestPlan create( diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/TargetsSpec.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/TargetsSpec.java index 7378a4ec84..dc82a0abcb 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/TargetsSpec.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/TargetsSpec.java @@ -3,14 +3,14 @@ import com.fasterxml.jackson.databind.MapperFeature; import com.fasterxml.jackson.databind.ObjectMapper; import java.io.IOException; -import java.net.URISyntaxException; -import java.net.URL; import java.nio.file.Path; -import java.nio.file.Paths; import java.util.HashSet; import java.util.List; import java.util.Set; +/** + * Spec file that contains targets + */ public record TargetsSpec(TestSpecMeta meta, List targets) implements TestSpec { @@ -19,6 +19,7 @@ public record TargetsSpec(TestSpecMeta meta, List targets) public TargetsSpec { Set seen = new HashSet(); + for (TargetConfiguration target : targets) { if (seen.contains(target.name())) { throw new IllegalArgumentException("target name already exists: " + target.name()); @@ -27,11 +28,11 @@ public record TargetsSpec(TestSpecMeta meta, List targets) } } - public TargetConfiguration configuration(String name) { + public TargetConfiguration getTarget(String targetName) { return targets.stream() - .filter(target -> target.name().equals(name)) + .filter(target -> target.name().equals(targetName)) .findFirst() - .orElseThrow(() -> new IllegalArgumentException("target name not found: " + name)); + .orElseThrow(() -> new IllegalArgumentException("target targetName not found: " + targetName)); } public static TargetsSpec loadAll(String path) { diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/TestCommand.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/TestCommand.java index aec0f1749f..7ac9419186 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/TestCommand.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/TestCommand.java @@ -10,6 +10,34 @@ import io.stargate.sgv2.jsonapi.testbench.testrun.TestRunEnv; import org.apache.commons.text.StringSubstitutor; +/** + * A API command to send, that may be part of a setup, test, cleanup, or lifecycle process. This is a + * basic command to send, without any checks etc. + *

+ * This class is re-used in Spec configurations where an API command is needed. For example below, the TestCommand + * is the Objects in the setup array. The structure is then a normal Data API command object, with a single top level + * member that is the name of the command. + * 

+ *   "setup": [
+ *     {
+ *       "insertOne": {
+ *         "document": {
+ *           "_id": "Inception",
+ *           "name": "Inception",
+ *           "genre": "Science Fiction",
+ *           "artist": [
+ *             "Leonardo DiCaprio"
+ *           ],
+ *           "$vectorize": "Inception is a science fiction action film about a professional thief who steals information by infiltrating the subconscious, entering people's dreams. He is offered a chance to have his criminal history erased as payment for implanting another person's idea into a target's subconscious."
+ *         }
+ *       }
+ *     },
+ *     {
+ *       "insertMany": {
+ *         "documents": [....
+ *
+ *

+ */ public class TestCommand { private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); @@ -18,10 +46,14 @@ public class TestCommand { private final CommandName commandName; private final String includeFrom; + /** + * Constructor called when this class is used in a record etc that is deserialized using Jackson. + * @param request The JSON object to deserialize from, e.g. the objects in the array above. + */ @JsonCreator(mode = JsonCreator.Mode.DELEGATING) public TestCommand(ObjectNode request) { - // if non null, that this is a point to find commands in the named test. + // if non-null, that this is a pointer to find commands in the named test-suite. var includeField = request.get("$include"); this.includeFrom = includeField == null ? null : includeField.asText(); @@ -34,26 +66,44 @@ public TestCommand(ObjectNode request) { } } - private void checkIsInclude() { - if (includeFrom != null) { - throw new IllegalStateException("TestCommand is defined to $include from: " + includeFrom); - } - } - + /** + * Get the complete request to send + */ public ObjectNode request() { - checkIsInclude(); + throwIfHasInclude(); return request; } + /** + * The name of the Data API command, extracted from the definition. + */ public CommandName commandName() { - checkIsInclude(); + throwIfHasInclude(); return commandName; } + /** + * Name of the test-suite to include command from, rather than use the definition in here. + *

+ * For example, this says to include setup commands from 'vectorize-base' + * + * 

+   *   "setup": [
+   *     {
+   *       "$include": "vectorize-base"
+   *     }
+   *   ],
+   *
+ *

+ * @return value of the '$include' from the json + */ public String includeFrom() { return includeFrom; } + /** + * Build from a string, useful for lifecycle commands that are created on the fly. + */ public static TestCommand fromJson(String json) { try { @@ -63,6 +113,12 @@ public static TestCommand fromJson(String json) { } } + /** + * Extracts the name of the API command from the request. + *

+ * Public for re-use. + *

+ */ public static CommandName commandName(ObjectNode request) { var requestCommandName = commandNameString(request); for (CommandName name : CommandName.values()) { @@ -73,6 +129,18 @@ public static CommandName commandName(ObjectNode request) { throw new IllegalArgumentException("Unknown command name: " + requestCommandName); } + public ObjectNode withEnvironment(TestRunEnv env) { + ObjectNode updated = request.deepCopy(); + walk(updated, env.substitutor()); + return updated; + } + + private void throwIfHasInclude() { + if (includeFrom != null) { + throw new IllegalStateException("TestCommand is defined to $include from: " + includeFrom); + } + } + private static String commandNameString(ObjectNode request) { var it = request.fieldNames(); if (!it.hasNext()) { @@ -85,11 +153,6 @@ private static String commandNameString(ObjectNode request) { return name; } - public ObjectNode withEnvironment(TestRunEnv env) { - ObjectNode updated = request.deepCopy(); - walk(updated, env.substitutor()); - return updated; - } private static void walk(ObjectNode obj, StringSubstitutor subs) { obj.properties() From e082e0d705a91246e40f3d79303a1dc08db2d5a1 Mon Sep 17 00:00:00 2001 From: Aaron Morton Date: Thu, 7 May 2026 14:38:29 +1200 Subject: [PATCH 84/89] TestPlanLifecycle comments --- AGENTS.md | 116 ++++++++++++++++++ .../lifecycle/TestPlanLifecycle.java | 70 +++++++++++ 2 files changed, 186 insertions(+) create mode 100644 AGENTS.md diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000000..048f05f30e --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,116 @@ +# Agent Guidelines + +## Code Comment Standards + +### Purpose of Comments + +Comments should explain **WHY** code exists or **WHY** a particular approach was chosen, NOT simply describe **WHAT** the code does. + +### What to Avoid + +- Redundant comments that restate obvious code behavior +- Comments that merely describe what a line or block of code does +- Stating the obvious (e.g., `// increment counter` above `counter++`) + +### What to Include + +- **Intent and reasoning**: Why this approach was chosen over alternatives +- **Business logic context**: Why certain rules or constraints exist +- **Edge cases**: Non-obvious scenarios the code handles +- **Non-obvious behavior**: Subtle interactions or side effects +- **Workarounds**: Why a particular workaround was necessary +- **Performance considerations**: Why certain optimizations were made +- **Security implications**: Why certain checks or patterns are used + +## Documentation Hierarchy + +### Class-Level Documentation + +Class-level Javadoc should provide comprehensive context and explain the design intent. Include: + +- **Purpose**: What problem this class/interface solves +- **Design rationale**: Why this approach was chosen +- **Usage context**: How and when this should be used +- **Constraints and rules**: What implementations should NOT do +- **Integration points**: How this fits into the larger system + +**Avoid repeating this information at the method level.** + +### Method-Level Documentation + +Documentation requirements vary based on the method's visibility and purpose: + +#### Public API, Interface, and Abstract Methods + +These require formal Javadoc with: +- **When the method is called**: Brief statement of why this should be called and what it does +- **Parameter documentation**: Standard `@param` tags for all parameters +- **Return value**: Standard `@return` tag + +**Do NOT repeat class-level context or add implementation examples at the method level.** + +#### Private Methods + +Private implementation methods typically only need a brief comment explaining **WHY** the method exists or **WHY** a particular approach was taken. Formal `@param` and `@return` tags are not required unless: +- The method is complex with non-obvious parameter usage +- The method has subtle behavior that needs explanation + +**Example (simple private method):** +```java +// Normalize user input to prevent injection attacks +private String sanitizeInput(String raw) { + return raw.replaceAll("[^a-zA-Z0-9]", ""); +} +``` + +**Example (complex private method needing params):** +```java +/** + * Merges overlapping time ranges to optimize query performance. + * Adjacent ranges within the tolerance window are combined to reduce + * the number of database queries. + * + * @param ranges List of time ranges, may contain overlaps + * @param toleranceMs Milliseconds of gap allowed between ranges to still merge them + * @return Consolidated list with overlapping ranges merged + */ +private List mergeTimeRanges(List ranges, long toleranceMs) { +``` + +### Documentation Example: Interface + +**Class-level (comprehensive):** +```java +/** + * Defines the stages in the lifecycle of a test bench run. + *

+ * Designed to be implemented by a Backend so that it can make changes + * to the data environment so tests can run in a common environment. + * For example, when we use Cassandra as a backend we need to create + * a keyspace but for Astra we use the default one. + *

+ *

+ * There should not be any test logic within the implementations, + * that should all be in the test definitions. + *

+ */ +public interface TestPlanLifecycle { +``` + +**Method-level (minimal):** +```java + /** + * Called to optionally add a node to execute before the workflow starts. + * + * @param testNodeFactory Factory to use to create test nodes + * @param uriBuilder Builder to use to create URIs + * @param workflow The workflow about to execute + * @return Optional DynamicNode to run before the workflow + */ + default Optional beforeWorkflow(...) { +``` + +**What to avoid at method level:** +- Repeating "useful for cleanup" or "allows setting up resources" (already covered at class level) +- Adding specific implementation examples (violates the "no test logic" constraint stated at class level) +- Restating the overall purpose of the interface \ No newline at end of file diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/lifecycle/TestPlanLifecycle.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/lifecycle/TestPlanLifecycle.java index 1f450317b9..1cb35c1ca6 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/testbench/lifecycle/TestPlanLifecycle.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/lifecycle/TestPlanLifecycle.java @@ -9,31 +9,101 @@ import java.util.Optional; import org.junit.jupiter.api.DynamicNode; +/** + * Defines the stages in the lifecycle of a test bench run. + *

+ * Design to be implemented by a {@link io.stargate.sgv2.jsonapi.testbench.targets.Backend} so that it can + * make changes to the data environement so tests can run in a common environement. For example, + * when we use Cassandra as a backend we need to create a keyspace but for Astra we use the default one. + *

+ *

+ * There should not be any test logic within the implementations, that should all be in the test defintoions. + *

+ *

+ * Extracted to an interface so it can be reused, such as at the higher level + * {@link io.stargate.sgv2.jsonapi.testbench.targets.Target} which includes a backend. + *

+ *

+ * All methods allow the implementation to return a JUNIT {@link DynamicNode} which will be inserted before or after + * the dynamic nodes at that level. The returned node could be a single {@link org.junit.jupiter.api.DynamicTest} or + * a {@link org.junit.jupiter.api.DynamicContainer}. + *

+ */ public interface TestPlanLifecycle { + /** + * Called to optionally add a node to execute before the start of the workflow. + * + * @param testNodeFactory Factory to use to create test nodes, see {@link TestNodeFactory} + * @param uriBuilder Builder to use to create the URI's added to dynamic nodes. + * @param workflow The workflow we are running. + * @return Optional {@link DynamicNode} to run before the workflow. + */ default Optional beforeWorkflow( TestNodeFactory testNodeFactory, TestUri.Builder uriBuilder, WorkflowSpec workflow) { return Optional.empty(); } + /** + * Called to optionally add a node to execute after the workflow completes. + * + * @param testNodeFactory Factory to use to create test nodes, see {@link TestNodeFactory} + * @param uriBuilder Builder to use to create the URI's added to dynamic nodes. + * @param workflow The workflow that just completed. + * @return Optional {@link DynamicNode} to run after the workflow. + */ default Optional afterWorkflow( TestNodeFactory testNodeFactory, TestUri.Builder uriBuilder, WorkflowSpec workflow) { return Optional.empty(); } + /** + * Called to optionally add a node to execute before a job starts. + * + * @param testNodeFactory Factory to use to create test nodes, see {@link TestNodeFactory} + * @param uriBuilder Builder to use to create the URI's added to dynamic nodes. + * @param job The job about to execute. + * @return Optional {@link DynamicNode} to run before the job. + */ default Optional beforeJob(TestNodeFactory testNodeFactory, TestUri.Builder uriBuilder, Job job) { return Optional.empty(); } + /** + * Called to optionally add a node to execute after a job completes. + * + * @param testNodeFactory Factory to use to create test nodes, see {@link TestNodeFactory} + * @param uriBuilder Builder to use to create the URI's added to dynamic nodes. + * @param job The job that just completed. + * @return Optional {@link DynamicNode} to run after the job. + */ default Optional afterJob(TestNodeFactory testNodeFactory, TestUri.Builder uriBuilder, Job job) { return Optional.empty(); } + /** + * Called to optionally add a node to execute before a test suite runs. + * + * @param testNodeFactory Factory to use to create test nodes, see {@link TestNodeFactory} + * @param uriBuilder Builder to use to create the URI's added to dynamic nodes. + * @param test The test suite about to execute. + * @param env The runtime environment for this test, including resolved variables and configuration. + * @return Optional {@link DynamicNode} to run before the test suite. + */ default Optional beforeTestSuite( TestNodeFactory testNodeFactory, TestUri.Builder uriBuilder, TestSuiteSpec test, TestRunEnv env) { return Optional.empty(); } + /** + * Called to optionally add a node to execute after a test suite completes. + * + * @param testNodeFactory Factory to use to create test nodes, see {@link TestNodeFactory} + * @param uriBuilder Builder to use to create the URI's added to dynamic nodes. + * @param test The test suite that just completed. + * @param env The runtime environment for this test, including resolved variables and configuration. + * @return Optional {@link DynamicNode} to run after the test suite. + */ default Optional afterTestSuite( TestNodeFactory testNodeFactory, TestUri.Builder uriBuilder, TestSuiteSpec test, TestRunEnv env) { return Optional.empty(); From 1238cffdfcbb0c75047bf184e092f8fa8e905740 Mon Sep 17 00:00:00 2001 From: Aaron Morton Date: Thu, 7 May 2026 15:17:58 +1200 Subject: [PATCH 85/89] code tidy --- .../sgv2/jsonapi/testbench/TestPlan.java | 5 +- .../sgv2/jsonapi/testbench/TestPlanFile.java | 3 +- .../testbench/lifecycle/JobLifeCycle.java | 28 +++++++++++ .../testbench/messaging/APIRequest.java | 6 +-- .../testbench/targets/AstraBackend.java | 6 +++ .../jsonapi/testbench/targets/Backend.java | 28 +++++++++-- .../testbench/targets/CassandraBackend.java | 22 ++++++--- .../jsonapi/testbench/targets/Connection.java | 15 ------ .../targets/ConnectionConfiguration.java | 25 ++++++++++ .../jsonapi/testbench/targets/Target.java | 47 +++++++++++++------ .../TargetConfiguration.java | 10 ++-- .../jsonapi/testbench/testrun/TestRunEnv.java | 17 ++----- .../testbench/testspec/TargetsSpec.java | 2 + 13 files changed, 150 insertions(+), 64 deletions(-) create mode 100644 src/test/java/io/stargate/sgv2/jsonapi/testbench/lifecycle/JobLifeCycle.java delete mode 100644 src/test/java/io/stargate/sgv2/jsonapi/testbench/targets/Connection.java create mode 100644 src/test/java/io/stargate/sgv2/jsonapi/testbench/targets/ConnectionConfiguration.java rename src/test/java/io/stargate/sgv2/jsonapi/testbench/{testspec => targets}/TargetConfiguration.java (57%) diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/TestPlan.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/TestPlan.java index fd7e6a36f9..725ec5cca4 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/testbench/TestPlan.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/TestPlan.java @@ -6,7 +6,9 @@ import com.fasterxml.jackson.databind.MapperFeature; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; +import io.stargate.sgv2.jsonapi.testbench.lifecycle.JobLifeCycle; import io.stargate.sgv2.jsonapi.testbench.targets.Target; +import io.stargate.sgv2.jsonapi.testbench.targets.TargetConfiguration; import io.stargate.sgv2.jsonapi.testbench.testrun.TestNodeFactory; import io.stargate.sgv2.jsonapi.testbench.testrun.TestUri; import io.stargate.sgv2.jsonapi.testbench.testspec.*; @@ -24,7 +26,7 @@ import org.slf4j.LoggerFactory; public record TestPlan( - Target target, SpecFiles specFiles, Set workflows, boolean ignoreDisabled) { + Target target, SpecFiles specFiles, Set workflows, boolean ignoreDisabled) implements JobLifeCycle { private static final Logger LOGGER = LoggerFactory.getLogger(TestPlan.class); @@ -118,6 +120,7 @@ public TestPlanNodeTree testNode() { return new TestPlanNodeTree(root, testNodeFactory.testNodeCount()); } + @Override public void updateJobForTarget(Job job) { target.updateJobForTarget(job); } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/TestPlanFile.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/TestPlanFile.java index 693115c413..7bbc67e5aa 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/testbench/TestPlanFile.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/TestPlanFile.java @@ -1,10 +1,9 @@ package io.stargate.sgv2.jsonapi.testbench; -import io.stargate.sgv2.jsonapi.testbench.testspec.TargetConfiguration; +import io.stargate.sgv2.jsonapi.testbench.targets.TargetConfiguration; import java.nio.file.Path; import java.util.List; -import java.util.Map; /** * Structure of a test plane file stored in a jar resource or external file. These are used with the diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/lifecycle/JobLifeCycle.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/lifecycle/JobLifeCycle.java new file mode 100644 index 0000000000..b1f87261f2 --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/lifecycle/JobLifeCycle.java @@ -0,0 +1,28 @@ +package io.stargate.sgv2.jsonapi.testbench.lifecycle; + +import io.stargate.sgv2.jsonapi.testbench.testspec.Job; + +/** + * Defines the stages a {link @Job} may go through as it is executing. + *

+ * It's limited now to just the one method, may never get bigger, but there are a couple of + * implementations so extracted. + *

+ */ +public interface JobLifeCycle { + + /** + * Called to update the Job with any information it needs to run against the + * particular target. + *

+ * Typcially this means setting or updating the {@link Job#variables()} for things like a default keyspace + * name or implementing naming restrictions. + *

+ *

+ * NOTE: The target or Backend must set the KEYSPACE_NAME job variable + *

+ * @param job The job to update + */ + void updateJobForTarget(Job job); + +} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/messaging/APIRequest.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/messaging/APIRequest.java index b2b06e35a6..3316e31641 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/testbench/messaging/APIRequest.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/messaging/APIRequest.java @@ -11,7 +11,7 @@ import io.restassured.response.ValidatableResponse; import io.restassured.specification.RequestSpecification; import io.stargate.sgv2.jsonapi.api.model.command.CommandTarget; -import io.stargate.sgv2.jsonapi.testbench.targets.Connection; +import io.stargate.sgv2.jsonapi.testbench.targets.ConnectionConfiguration; import io.stargate.sgv2.jsonapi.testbench.testrun.TestRunEnv; import io.stargate.sgv2.jsonapi.testbench.testspec.TestCommand; import io.stargate.sgv2.jsonapi.config.constants.HttpConstants; @@ -32,11 +32,11 @@ public class APIRequest { private static String KEYSPACE_PATH = "/{keyspace}"; private static String DB_PATH = "/"; - private final Connection connection; + private final ConnectionConfiguration connection; private final TestRunEnv integrationEnv; private final ObjectNode request; - public APIRequest(Connection connection, TestRunEnv integrationEnv, ObjectNode request) { + public APIRequest(ConnectionConfiguration connection, TestRunEnv integrationEnv, ObjectNode request) { this.connection = connection; this.integrationEnv = integrationEnv; diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/targets/AstraBackend.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/targets/AstraBackend.java index 6fc7aa2e13..ae9b96ac32 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/testbench/targets/AstraBackend.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/targets/AstraBackend.java @@ -2,11 +2,17 @@ import io.stargate.sgv2.jsonapi.testbench.testspec.Job; +/** + * DataStax / IBM Astra + */ public class AstraBackend extends Backend { + public static final String NAME = "astra"; + @Override public void updateJobForTarget(Job job) { + // always using the `default_keyspace` for astra, because they cannot be made using the API job.variables().put("KEYSPACE_NAME", "default_keyspace"); } } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/targets/Backend.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/targets/Backend.java index 845c208b6e..f27e52b069 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/testbench/targets/Backend.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/targets/Backend.java @@ -1,9 +1,31 @@ package io.stargate.sgv2.jsonapi.testbench.targets; +import io.stargate.sgv2.jsonapi.testbench.lifecycle.JobLifeCycle; import io.stargate.sgv2.jsonapi.testbench.lifecycle.TestPlanLifecycle; -import io.stargate.sgv2.jsonapi.testbench.testspec.Job; -public abstract class Backend implements TestPlanLifecycle { +import java.util.regex.Pattern; - public void updateJobForTarget(Job job) {} +/** + * A class of backend database the tests will be run against. This is not a partcular insance of a DB to run + * against, for that see {@link Target}. + *

+ * There is some different behavior between C* and Astra, and potentially in any future versions of + * those products. + *

+ */ +public abstract class Backend implements TestPlanLifecycle, JobLifeCycle { + + private static final Pattern PATTERN_NOT_WORD_CHARS = Pattern.compile("\\W+"); + + /** + * Sanitizes a schema name to be valid in all C* derived platforms. + */ + public static String toSafeSchemaIdentifier(String name) { + + var newValue = PATTERN_NOT_WORD_CHARS.matcher(name).replaceAll("_"); + if (newValue.length() > 48) { + return newValue.substring(0, 47); + } + return newValue; + } } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/targets/CassandraBackend.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/targets/CassandraBackend.java index 3a7505ca55..4b46b50265 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/testbench/targets/CassandraBackend.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/targets/CassandraBackend.java @@ -1,8 +1,5 @@ package io.stargate.sgv2.jsonapi.testbench.targets; -import static io.stargate.sgv2.jsonapi.testbench.testrun.TestRunEnv.toSafeSchemaIdentifier; - -import com.fasterxml.jackson.databind.ObjectMapper; import io.stargate.sgv2.jsonapi.testbench.assertions.TestAssertion; import io.stargate.sgv2.jsonapi.testbench.testrun.TestExecutionCondition; import io.stargate.sgv2.jsonapi.testbench.testrun.TestNodeFactory; @@ -10,18 +7,23 @@ import io.stargate.sgv2.jsonapi.testbench.testrun.TestUri; import io.stargate.sgv2.jsonapi.testbench.testspec.Job; import io.stargate.sgv2.jsonapi.testbench.testspec.TestCommand; -import java.util.Optional; import org.apache.commons.lang3.RandomStringUtils; import org.junit.jupiter.api.DynamicNode; -/** Cassandra:Y2Fzc2FuZHJh:Y2Fzc2FuZHJh */ +import java.util.Optional; + +/** + * A classic Cassandra backend, running locally or elsewhere. + * */ public class CassandraBackend extends Backend { - private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + public static final String NAME = "cassandra"; @Override public void updateJobForTarget(Job job) { - // max length for keyspace is 48 chars + + // We are going to create the keyspace for every job, so making a new name here based on the name + // of the job, and some random because the name will be trunc'd var keyspaceName = toSafeSchemaIdentifier( "job_" @@ -31,6 +33,9 @@ public void updateJobForTarget(Job job) { job.variables().put("KEYSPACE_NAME", keyspaceName); } + /** + * Need to create a keyspace, because C* does not have a default one. + */ @Override public Optional beforeJob(TestNodeFactory testNodeFactory, TestUri.Builder uriBuilder, Job job) { @@ -57,6 +62,9 @@ public Optional beforeJob(TestNodeFactory testNodeFactory, TestUri. return Optional.of(setupRequest.testNodes(testNodeFactory, uriBuilder)); } + /** + * Drop the keyspace we made for this job. + */ @Override public Optional afterJob(TestNodeFactory testNodeFactory, TestUri.Builder uriBuilder, Job job) { var command = diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/targets/Connection.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/targets/Connection.java deleted file mode 100644 index 0db4b1bd3f..0000000000 --- a/src/test/java/io/stargate/sgv2/jsonapi/testbench/targets/Connection.java +++ /dev/null @@ -1,15 +0,0 @@ -package io.stargate.sgv2.jsonapi.testbench.targets; - -public record Connection(String domain, Integer port, String basePath) { - public Connection { - if (domain == null) { - domain = "localhost"; - } - if (port == null) { - port = 8181; - } - if (basePath == null) { - basePath = "/v1"; - } - } -} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/targets/ConnectionConfiguration.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/targets/ConnectionConfiguration.java new file mode 100644 index 0000000000..5dfd8213bf --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/targets/ConnectionConfiguration.java @@ -0,0 +1,25 @@ +package io.stargate.sgv2.jsonapi.testbench.targets; + +/** + * Configuraiton record of how to connect for a {@link TargetConfiguration}. + *

---

+ * + * @param domain domain and protocol, e.g. http://localhost + * @param port port to connect to, typically 8181 ore 443 but may be different when running integration tests. + * @param basePath base path to the API, for Astra this is /api/json/v1 when running locally is normally + * /v1 +*/ + +public record ConnectionConfiguration(String domain, Integer port, String basePath) { + public ConnectionConfiguration { + if (domain == null) { + domain = "localhost"; + } + if (port == null) { + port = 8181; + } + if (basePath == null) { + basePath = "/v1"; + } + } +} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/targets/Target.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/targets/Target.java index 48549f3036..6f719047a1 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/testbench/targets/Target.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/targets/Target.java @@ -1,33 +1,46 @@ package io.stargate.sgv2.jsonapi.testbench.targets; +import io.stargate.sgv2.jsonapi.testbench.lifecycle.JobLifeCycle; import io.stargate.sgv2.jsonapi.testbench.lifecycle.TestPlanLifecycle; import io.stargate.sgv2.jsonapi.testbench.messaging.APIRequest; import io.stargate.sgv2.jsonapi.testbench.testrun.TestNodeFactory; import io.stargate.sgv2.jsonapi.testbench.testrun.TestRunEnv; import io.stargate.sgv2.jsonapi.testbench.testrun.TestUri; import io.stargate.sgv2.jsonapi.testbench.testspec.Job; -import io.stargate.sgv2.jsonapi.testbench.testspec.TargetConfiguration; import io.stargate.sgv2.jsonapi.testbench.testspec.TestCommand; import io.stargate.sgv2.jsonapi.testbench.testspec.TestSuiteSpec; import io.stargate.sgv2.jsonapi.testbench.testspec.WorkflowSpec; -import java.util.HashMap; -import java.util.Optional; import org.junit.jupiter.api.DynamicNode; -public class Target implements TestPlanLifecycle { +import java.util.Optional; + +/** + * A particular instance of a {@link Backend} we are going to run the test against, + * so includes connection information etc. + *

+ * A run of the Test Bench is run against a Target, e.g. cassandra on localhost, or + * an astra db called monkeys. + *

+ *

+ * This is the important entry point for the lifecycle interfaces, because the life cycle is there + * to handle different target / backends that tests run against. + *

+ *

+ * Because this holds the connection information, it can also make a request, see {@link #apiRequest(TestCommand, TestRunEnv)} + *

+ */ +public class Target implements TestPlanLifecycle, JobLifeCycle { private final TargetConfiguration targetConfiguration; private final Backend backend; - private final TestRunEnv env; public Target(TargetConfiguration targetConfiguration) { this.targetConfiguration = targetConfiguration; - this.env = new TestRunEnv(new HashMap<>()); this.backend = switch (targetConfiguration.backend()) { - case "cassandra" -> new CassandraBackend(); - case "astra" -> new AstraBackend(); + case CassandraBackend.NAME -> new CassandraBackend(); + case AstraBackend.NAME -> new AstraBackend(); default -> throw new IllegalArgumentException( "Unknown backend: " + targetConfiguration.backend()); @@ -38,18 +51,24 @@ public TargetConfiguration configuration() { return targetConfiguration; } - public Connection connection() { - return targetConfiguration.connection(); + /** + * Call this to get a new {@link APIRequest} that is configured to talk to the target + * this class represents. + * @param testCommand The command the request will send, this is needed to get the actual + * DataAPI request we want to send. + * @param env Environment the commands will be run in, used to make the replacements in the + * command to execute for this particular test run. + * @return Configured {@link APIRequest} + */ + public APIRequest apiRequest(TestCommand testCommand, TestRunEnv env) { + return new APIRequest(targetConfiguration.connection(), env, testCommand.withEnvironment(env)); } + @Override public void updateJobForTarget(Job job) { backend.updateJobForTarget(job); } - public APIRequest apiRequest(TestCommand testCommand, TestRunEnv env) { - return new APIRequest(targetConfiguration.connection(), env, testCommand.withEnvironment(env)); - } - @Override public Optional beforeWorkflow( TestNodeFactory testNodeFactory, TestUri.Builder uriBuilder, WorkflowSpec workflow) { diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/TargetConfiguration.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/targets/TargetConfiguration.java similarity index 57% rename from src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/TargetConfiguration.java rename to src/test/java/io/stargate/sgv2/jsonapi/testbench/targets/TargetConfiguration.java index 0cb33a4d17..f88f2a835e 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/TargetConfiguration.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/targets/TargetConfiguration.java @@ -1,11 +1,11 @@ -package io.stargate.sgv2.jsonapi.testbench.testspec; - -import io.stargate.sgv2.jsonapi.testbench.targets.Connection; +package io.stargate.sgv2.jsonapi.testbench.targets; /** - * A target the test bench should connect to and run the tests + * Configuration record for a {@link Target}. + *

---

+ * * @param name Friendly name of the target * @param backend Backend name, such as astra or cassandra so we know how to run the lifecycle * @param connection Connection information for the target. */ -public record TargetConfiguration(String name, String backend, Connection connection) {} +public record TargetConfiguration(String name, String backend, ConnectionConfiguration connection) {} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/testrun/TestRunEnv.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testrun/TestRunEnv.java index a4c42e6472..b2cac6b1d1 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/testbench/testrun/TestRunEnv.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testrun/TestRunEnv.java @@ -1,5 +1,6 @@ package io.stargate.sgv2.jsonapi.testbench.testrun; +import io.stargate.sgv2.jsonapi.testbench.targets.Backend; import io.stargate.sgv2.jsonapi.testbench.testspec.TestSuiteSpec; import java.util.HashMap; import java.util.Map; @@ -14,7 +15,6 @@ public class TestRunEnv { private static final Logger LOGGER = LoggerFactory.getLogger(TestRunEnv.class); - private static final Pattern PATTERN_NOT_WORD_CHARS = Pattern.compile("\\W+"); private static final Set SCHEMA_IDENTIFIER = Set.of("KEYSPACE_NAME", "COLLECTION_NAME"); @@ -104,23 +104,12 @@ public String get(String name) { var substituted = substitutor().replace(value); var cleaned = - SCHEMA_IDENTIFIER.contains(name) ? toSafeSchemaIdentifier(substituted) : substituted; + SCHEMA_IDENTIFIER.contains(name) ? Backend.toSafeSchemaIdentifier(substituted) : substituted; return cleaned; } - public static String toSafeSchemaIdentifier(String name) { - - var newValue = PATTERN_NOT_WORD_CHARS.matcher(name).replaceAll("_"); - if (newValue.length() > 48) { - return newValue.substring(0, 47); - // throw new RuntimeException("Schema Identifier longer than 48 characters - // orginalName=%s, afterNormalisation==%s".formatted(name,newValue)); - } - return newValue; - } - - @Override + @Override public String toString() { return vars.toString(); } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/TargetsSpec.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/TargetsSpec.java index dc82a0abcb..72bfbf201f 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/TargetsSpec.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/TargetsSpec.java @@ -2,6 +2,8 @@ import com.fasterxml.jackson.databind.MapperFeature; import com.fasterxml.jackson.databind.ObjectMapper; +import io.stargate.sgv2.jsonapi.testbench.targets.TargetConfiguration; + import java.io.IOException; import java.nio.file.Path; import java.util.HashSet; From 0e172023452a98d77188a6a1b564e493dbd47421 Mon Sep 17 00:00:00 2001 From: Aaron Morton Date: Fri, 8 May 2026 08:15:55 +1200 Subject: [PATCH 86/89] code tidy --- .../java/io/stargate/sgv2/jsonapi/testbench/TestPlan.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/TestPlan.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/TestPlan.java index 725ec5cca4..427b2f08fa 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/testbench/TestPlan.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/TestPlan.java @@ -25,6 +25,13 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +/** + * The + * @param target + * @param specFiles + * @param workflows + * @param ignoreDisabled + */ public record TestPlan( Target target, SpecFiles specFiles, Set workflows, boolean ignoreDisabled) implements JobLifeCycle { From 423382a9d3d602c37371e5e2fa9458defeff16bd Mon Sep 17 00:00:00 2001 From: Aaron Morton Date: Mon, 11 May 2026 10:03:31 +1200 Subject: [PATCH 87/89] code tidy --- .../testbench/messaging/APIRequest.java | 150 ++++++++-------- .../testbench/messaging/APIResponse.java | 3 + .../testbench/messaging/APIRetryPolicy.java | 161 ++++++++++++++++++ .../jsonapi/testbench/testrun/TestRunEnv.java | 22 ++- 4 files changed, 258 insertions(+), 78 deletions(-) create mode 100644 src/test/java/io/stargate/sgv2/jsonapi/testbench/messaging/APIRetryPolicy.java diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/messaging/APIRequest.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/messaging/APIRequest.java index 3316e31641..1977b1064a 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/testbench/messaging/APIRequest.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/messaging/APIRequest.java @@ -19,39 +19,61 @@ import org.slf4j.LoggerFactory; import java.util.HashMap; -import java.util.List; import java.util.Map; -import java.util.concurrent.ThreadLocalRandom; +/** + * The lowest level to represent a request sent to the API, that is only concerned with the + * mechanics of sending the request. + *

+ * Handles retries based on detecting substrings in the response body, these occur before + * {@link #execute()} returns. + *

+ */ public class APIRequest { private static final Logger LOGGER = LoggerFactory.getLogger(APIRequest.class); public static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); - private static String COLLECTION_PATH = "/{keyspace}/{collection}"; + private static int RETRY_MAX_ATTEMPTS = 5; + + // API paths based on the target of the command. + private static String COLLECTION_TABLE_PATH = "/{keyspace}/{collection}"; private static String KEYSPACE_PATH = "/{keyspace}"; private static String DB_PATH = "/"; private final ConnectionConfiguration connection; - private final TestRunEnv integrationEnv; + private final TestRunEnv testRunEnv; private final ObjectNode request; + private final APIRetryPolicy retryPolicy; - public APIRequest(ConnectionConfiguration connection, TestRunEnv integrationEnv, ObjectNode request) { + /** + * Initializes a new instance of the class. + * @param connection Connection info for the API instance to use. + * @param testRunEnv Environment for this test run, used to find schema names to use in the URL path. + * @param request the complete API request to send, NOTE: any substitutions into the body of the + * request must also be done. + */ + public APIRequest(ConnectionConfiguration connection, TestRunEnv testRunEnv, ObjectNode request) { this.connection = connection; - this.integrationEnv = integrationEnv; + this.testRunEnv = testRunEnv; this.request = request; + this.retryPolicy = APIRetryPolicy.createRetryPolicy(testRunEnv); } + /** + * Executes the request, including any retries. + *

+ * No validation of the response is performed, that is left for assertions to handle later. + *

+ * @return {@link APIResponse} holding the response of the request. + */ public APIResponse execute() { - boolean retry = true; - int maxAttempts = 6; // 6 attempts, means 5 retries - int attempt = 1; ValidatableResponse lastValidatableResponse = null; - while (retry && attempt <=maxAttempts){ - retry = false; + var retryDecision = retryPolicy.firstAttempt(); + while (retryDecision.retry()){ // Create a new request spec, there is some state that is left in it when a request is run // for path params @@ -59,41 +81,18 @@ public APIResponse execute() { var rawResponse = executeRequest(requestSpec); lastValidatableResponse = rawResponse.then(); - // logg even if we retry + // log even if we retry, the request will be logged and it makes sense to see the respose that + // caused the retry lastValidatableResponse.log().status().and().log().body(); - - if (attempt < maxAttempts) { - var body = rawResponse.body().asString(); - // TODO: XXXX: put this in the test plan - var retryMatch = List.of("EMBEDDING_PROVIDER_RATE_LIMITED", "EMBEDDING_PROVIDER_TIMEOUT"); - - for (var match : retryMatch) { - if (body.contains(match)) { - retry = true; - // Service has a concurrency limit and retrying runners can collide regardless of jitter. - // Base backoff is long enough to wait out an in-flight request (5s, 10s, 20s...), - // plus a small random offset to avoid re-synchronising after the wait. - long baseMs = (long) (5000 * Math.pow(2, attempt)); - long jitterMs = ThreadLocalRandom.current().nextLong(2000); - long sleepMs = baseMs + jitterMs; - - LOGGER.info("executeRequest() - Retrying, found retry string in response. match={}, sleepMs={} ms, attemptCount={}", match, sleepMs, attempt); - try { - Thread.sleep(sleepMs); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - throw new RuntimeException("Interrupted during retry sleep", e); - } - break; - } - } - } - attempt++; + retryDecision = retryPolicy.decide(retryDecision, rawResponse); } return new APIResponse(this, lastValidatableResponse); - } + /** + * Get the request ready to send our request. + * @return + */ private RequestSpecification requestSpec() { String requestString; @@ -102,23 +101,57 @@ private RequestSpecification requestSpec() { } catch (JsonProcessingException e) { throw new RuntimeException(e); } + return requestForTaget() + .headers(getHeaders()) + .body(requestString).when(); + } + + /** + * Create a new RequestSpecification for JSON to send to the target + */ + private RequestSpecification requestForTaget() { + + return given() + .log() + .uri() + .log() + .body() + .baseUri(connection.domain()) + .port(connection.port()) + .basePath(connection.basePath()) + .contentType(ContentType.JSON); + } - return jsonRequest().body(requestString).when(); + /** + * Get the wellknown headers we need to send with the request. + */ + protected Map getHeaders() { + + var headers = new HashMap(); + headers.put(HttpConstants.AUTHENTICATION_TOKEN_HEADER_NAME, testRunEnv.requiredValue(HttpConstants.AUTHENTICATION_TOKEN_HEADER_NAME)); + + var embeddingApiKey = testRunEnv.get(HttpConstants.EMBEDDING_AUTHENTICATION_TOKEN_HEADER_NAME); + if (!Strings.isNullOrEmpty(embeddingApiKey)){ + headers.put(HttpConstants.EMBEDDING_AUTHENTICATION_TOKEN_HEADER_NAME, embeddingApiKey); + } + return headers; } private Response executeRequest(RequestSpecification requestSpec) { var commandName = TestCommand.commandName(request); Response rawRresponse; + + // URL to send to depends on the target of the command. if (commandName.getTargets().contains(CommandTarget.COLLECTION) || commandName.getTargets().contains(CommandTarget.TABLE)) { rawRresponse = requestSpec.post( - COLLECTION_PATH, - integrationEnv.requiredValue("KEYSPACE_NAME"), - integrationEnv.requiredValue("COLLECTION_NAME")); + COLLECTION_TABLE_PATH, + testRunEnv.requiredValue(TestRunEnv.ENV_KEYSPACE_NAME), + testRunEnv.requiredValue(TestRunEnv.ENV_COLLECTION_NAME)); } else if (commandName.getTargets().contains(CommandTarget.KEYSPACE)) { - rawRresponse = requestSpec.post(KEYSPACE_PATH, integrationEnv.requiredValue("KEYSPACE_NAME")); + rawRresponse = requestSpec.post(KEYSPACE_PATH, testRunEnv.requiredValue(TestRunEnv.ENV_KEYSPACE_NAME)); } else if (commandName.getTargets().contains(CommandTarget.DATABASE)) { rawRresponse = requestSpec.post(DB_PATH); } else { @@ -128,29 +161,4 @@ private Response executeRequest(RequestSpecification requestSpec) { return rawRresponse; } - protected Map getHeaders() { - - var headers = new HashMap(); - headers.put(HttpConstants.AUTHENTICATION_TOKEN_HEADER_NAME, integrationEnv.requiredValue(HttpConstants.AUTHENTICATION_TOKEN_HEADER_NAME)); - - var embeddingApiKey = integrationEnv.get(HttpConstants.EMBEDDING_AUTHENTICATION_TOKEN_HEADER_NAME); - if (!Strings.isNullOrEmpty(embeddingApiKey)){ - headers.put(HttpConstants.EMBEDDING_AUTHENTICATION_TOKEN_HEADER_NAME, embeddingApiKey); - } - return headers; - } - - public RequestSpecification jsonRequest() { - - return given() - .log() - .uri() - .log() - .body() - .baseUri(connection.domain()) - .port(connection.port()) - .basePath(connection.basePath()) - .headers(getHeaders()) - .contentType(ContentType.JSON); - } } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/messaging/APIResponse.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/messaging/APIResponse.java index 05d0023658..eec21c1cad 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/testbench/messaging/APIResponse.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/messaging/APIResponse.java @@ -2,4 +2,7 @@ import io.restassured.response.ValidatableResponse; +/** + * Basic holder for the response, so we can tie it back to the request that created it + */ public record APIResponse(APIRequest apiRequest, ValidatableResponse validatable) {} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/messaging/APIRetryPolicy.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/messaging/APIRetryPolicy.java new file mode 100644 index 0000000000..d29f68347b --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/messaging/APIRetryPolicy.java @@ -0,0 +1,161 @@ +package io.stargate.sgv2.jsonapi.testbench.messaging; + +import io.restassured.response.Response; +import io.stargate.sgv2.jsonapi.testbench.testrun.TestRunEnv; + +import java.util.List; +import java.util.concurrent.ThreadLocalRandom; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + *

---

+ */ +interface APIRetryPolicy { + + Logger LOGGER = LoggerFactory.getLogger(APIRetryPolicy.class); + + String RETRY_MATCH_STRING_DELIM = "\t"; + String ENV_RETRY_MATCH_STRING = "RETRY_MATCH_STRING"; + + String DEFAULT_MAX_ATTEMPTS = "5"; + String ENV_RETRY_MAX_ATTEMPTS = "END_RETRY_MAX_ATTEMPTS"; + + String DEFAULT_BASE_SLEEP_MS = "5000"; + String ENV_RETRY_BASE_SLEEP_MS = "RETRY_BASE_SLEEP_MS"; + + String DEFAULT_JITTER_MS = "2000"; + String ENV_RETRY_JITTER_MS = "RETRY_JITTER_MS"; + + /** + * Retries are driven by the presence of the following keys in the {@link TestRunEnv}. When a retry is + * performed the response for that request is lost, the response of the call to execute will + * be the response from the last attempt. + *
    + *
  • {@link #ENV_RETRY_MATCH_STRING} Enables retry if present and non blank string, treated as a tab + * {@link #RETRY_MATCH_STRING_DELIM} delimtered string. If any of the tokens is present in the response body + * the request is retried. + * Example: "EMBEDDING_PROVIDER_RATE_LIMITED\tEMBEDDING_PROVIDER_TIMEOUT"
    • + *
  • {@link #ENV_RETRY_MAX_ATTEMPTS} total number of attempts to make before returning, + * must be above 1, default is {@link #ENV_RETRY_MAX_ATTEMPTS}
    • + *
  • {@link #ENV_RETRY_BASE_SLEEP_MS} base number of milliseconds between attempts, this is multiplied by + * 2 ^ attempt, so base of 5000 means sleep of 5, 10, 20 seconds
    • + *
  • {@link #ENV_RETRY_JITTER_MS} Upper bound on the random number of milliseconds to add to each attempt.
    • + *
+ */ + static APIRetryPolicy createRetryPolicy(TestRunEnv testRunEnv) { + + var retryMatch = testRunEnv.get(ENV_RETRY_MATCH_STRING); + if (retryMatch == null || retryMatch.isBlank()) { + return new NoAPIRetryPolicy(); + } + + var maxAttempts = Integer.parseInt(testRunEnv.get(ENV_RETRY_MAX_ATTEMPTS, DEFAULT_MAX_ATTEMPTS)); + var baseSleepMs = Long.parseLong(testRunEnv.get(ENV_RETRY_BASE_SLEEP_MS, DEFAULT_BASE_SLEEP_MS)); + var jitterMs = Long.parseLong(testRunEnv.get(ENV_RETRY_JITTER_MS, DEFAULT_JITTER_MS)); + + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("createRetryPolicy() - retryMatch={}, maxAttempts={}, baseSleepMs={}, jitterMs={}", retryMatch, maxAttempts, baseSleepMs, jitterMs); + } + + return new ConfiguredAPIRetryPolicy(List.of(retryMatch.split(RETRY_MATCH_STRING_DELIM)), maxAttempts, baseSleepMs, jitterMs); + } + + /** + * Call to get the first decision, this is used to count the number of attempts. + */ + default RetryDecision firstAttempt() { + return RetryDecision.FIRST_ATTEMPT; + } + + /** + * Call to decide if we should retry or not, implementations will sleep if needed. + * @param lastDecision The last decision made, used to count the number of attempts + * @param response The response from the last attempt + * @return A decision to retry or not, along with the attempt count, use for the next call to this method. + */ + RetryDecision decide(RetryDecision lastDecision, Response response); + + + /** + * A decision to retry or not, along with the attempt count. This is where we count the + * number of attempts. + */ + record RetryDecision(boolean retry, + int attempt){ + + static final RetryDecision FIRST_ATTEMPT = new RetryDecision(true, 1); + + RetryDecision stopAttempts(){ + // do not increase the attempt count, we won't make another. + return new RetryDecision(false, attempt); + } + } + + /** + * A retry policy that never retries, makes a single attempt. + */ + record NoAPIRetryPolicy() implements APIRetryPolicy { + private static final RetryDecision NO_RETRY = new RetryDecision(false, 1); + + @Override + public RetryDecision decide(RetryDecision lastDecision, Response response) { + return NO_RETRY; + } + } + + /** + * Configurable retry policy with customizable retry conditions, maximum attempts, and backoff strategy. + */ + record ConfiguredAPIRetryPolicy( + List retryMatch, + int maxAttempts, + long baseSleepMs, + long jitterMs + ) implements APIRetryPolicy { + + public ConfiguredAPIRetryPolicy { + + if (retryMatch == null || retryMatch.isEmpty()) { + throw new IllegalArgumentException("retryMatch is null or empty"); + } + if (maxAttempts < 2) { + throw new IllegalArgumentException("maxAttempts must be greater than 1, got: " + maxAttempts); + } + } + + @Override + public RetryDecision decide(RetryDecision lastDecision, Response response) { + + if (lastDecision.attempt == maxAttempts) { + return lastDecision.stopAttempts(); + } + + var body = response.body().asString(); + + for (var match : retryMatch) { + if (body.contains(match)) { + // Service has a concurrency limit and retrying runners can collide regardless of jitter. + // Base backoff is long enough to wait out an in-flight request (5s, 10s, 20s...), + // plus a small random offset to avoid re-synchronising after the wait. + long baseMs = (long) (5000 * Math.pow(2, lastDecision.attempt)); + long jitterMs = ThreadLocalRandom.current().nextLong(2000); + long sleepMs = baseMs + jitterMs; + + LOGGER.info("executeRequest() - Retrying, found retry string in response. match={}, sleepMs={} ms, lastDecision.attempt={}", match, sleepMs, lastDecision.attempt); + try { + Thread.sleep(sleepMs); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException("Interrupted during retry sleep", e); + } + + // try again, increase the attempt counter. + return new RetryDecision(true, lastDecision.attempt +1); + } + } + return lastDecision.stopAttempts(); + } + } +} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/testrun/TestRunEnv.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testrun/TestRunEnv.java index b2cac6b1d1..bb3031c18b 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/testbench/testrun/TestRunEnv.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testrun/TestRunEnv.java @@ -16,7 +16,15 @@ public class TestRunEnv { private static final Logger LOGGER = LoggerFactory.getLogger(TestRunEnv.class); - private static final Set SCHEMA_IDENTIFIER = Set.of("KEYSPACE_NAME", "COLLECTION_NAME"); + // Wellknown environment variables use this because we know they are schema identifiers + public static final String ENV_KEYSPACE_NAME = "KEYSPACE_NAME"; + // Also for tables + public static final String ENV_COLLECTION_NAME = "COLLECTION_NAME"; + + private static final Set SCHEMA_IDENTIFIER = Set.of( + ENV_KEYSPACE_NAME, + ENV_COLLECTION_NAME + ); private final Map vars = new HashMap<>(); @@ -95,18 +103,18 @@ public StringSubstitutor substitutor() { .setEnableUndefinedVariableException(true); } - public String get(String name) { + public String get(String name){ + return get(name, ""); + } + public String get(String name, String defaultValue) { var value = vars.get(name); if (value == null) { - return ""; + value = defaultValue; } var substituted = substitutor().replace(value); - var cleaned = - SCHEMA_IDENTIFIER.contains(name) ? Backend.toSafeSchemaIdentifier(substituted) : substituted; - - return cleaned; + return SCHEMA_IDENTIFIER.contains(name) ? Backend.toSafeSchemaIdentifier(substituted) : substituted; } @Override From e121f852305daae7190642a699d546f94afe595b Mon Sep 17 00:00:00 2001 From: Aaron Morton Date: Mon, 11 May 2026 13:11:17 +1200 Subject: [PATCH 88/89] Code tidy --- .../testbench/TestBenchByTestPlan.java | 6 +- .../{TestPlan.java => TestBenchPlan.java} | 22 +- .../sgv2/jsonapi/testbench/TestPlanFile.java | 2 +- .../assertions/AssertionFactory.java | 8 +- .../testbench/assertions/Templated.java | 6 +- .../testbench/assertions/TestAssertion.java | 12 +- .../reporting/DynamicTreeListener.java | 176 ++++++-------- .../reporting/TestBenchConsoleWriter.java | 224 +++++++++++++----- .../testbench/testrun/TestNodeFactory.java | 8 +- .../sgv2/jsonapi/testbench/testspec/Job.java | 8 +- .../jsonapi/testbench/testspec/TestSpec.java | 4 +- src/test/resources/application.yaml | 2 +- 12 files changed, 290 insertions(+), 188 deletions(-) rename src/test/java/io/stargate/sgv2/jsonapi/testbench/{TestPlan.java => TestBenchPlan.java} (88%) diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/TestBenchByTestPlan.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/TestBenchByTestPlan.java index cfb025035a..1e08e1eac2 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/testbench/TestBenchByTestPlan.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/TestBenchByTestPlan.java @@ -1,7 +1,7 @@ package io.stargate.sgv2.jsonapi.testbench; import io.stargate.sgv2.jsonapi.testbench.testspec.SpecFiles; -import io.stargate.sgv2.jsonapi.testbench.testspec.TargetsSpec; + import java.nio.file.Path; import java.util.stream.Stream; @@ -32,12 +32,12 @@ public Stream runTestPlanFile() { ? SpecFiles.resourceDir(rawPath.substring("classpath:".length())) : Path.of(rawPath); - var testPlan = TestPlan.fromFile(path); + var testPlan = TestBenchPlan.fromFile(path); LOGGER.info("runTestPlanFile() - building test plan tree"); var testPlanNodeTree = testPlan.testNode(); LOGGER.info("runTestPlanFile() - test plan tree build, totalNodeCount={}", testPlanNodeTree.totalNodeCount()); - System.setProperty("testbench.test.count", String.valueOf(testPlanNodeTree.totalNodeCount())); + System.setProperty(TestBenchPlan.TEST_PLAN_TEST_COUNT_PROPERTY, String.valueOf(testPlanNodeTree.totalNodeCount())); return Stream.of(testPlanNodeTree.root()); } } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/TestPlan.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/TestBenchPlan.java similarity index 88% rename from src/test/java/io/stargate/sgv2/jsonapi/testbench/TestPlan.java rename to src/test/java/io/stargate/sgv2/jsonapi/testbench/TestBenchPlan.java index 427b2f08fa..b9811c3c99 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/testbench/TestPlan.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/TestBenchPlan.java @@ -26,23 +26,29 @@ import org.slf4j.LoggerFactory; /** - * The + * + *

+ * NOTE: called "TestBenchPlan" to avoid collision with "org.junit.platform.launcher.TestPlan" + *

* @param target * @param specFiles * @param workflows * @param ignoreDisabled */ -public record TestPlan( +public record +TestBenchPlan( Target target, SpecFiles specFiles, Set workflows, boolean ignoreDisabled) implements JobLifeCycle { - private static final Logger LOGGER = LoggerFactory.getLogger(TestPlan.class); + private static final Logger LOGGER = LoggerFactory.getLogger(TestBenchPlan.class); + + public static final String TEST_PLAN_TEST_COUNT_PROPERTY = "testbench.test.count"; private static final ObjectMapper YAML_MAPPER = new ObjectMapper( YAMLFactory.builder().enable(StreamReadFeature.INCLUDE_SOURCE_IN_LOCATION).build()) .configure(MapperFeature.ACCEPT_CASE_INSENSITIVE_ENUMS, true); - public static TestPlan fromFile(Path path) { + public static TestBenchPlan fromFile(Path path) { LOGGER.info("fromFile() - Loading test plan file, path={}", path); TestPlanFile planFile; @@ -67,16 +73,16 @@ public static TestPlan fromFile(Path path) { return testPlan; } - public static TestPlan create(String targetName, List workflows) { + public static TestBenchPlan create(String targetName, List workflows) { return create(targetName, workflows, true); } - public static TestPlan create(String targetName, List workflows, Boolean ignoreDisabled) { + public static TestBenchPlan create(String targetName, List workflows, Boolean ignoreDisabled) { var targetConfigs = TargetsSpec.loadAll("testbench/targets/targets.json"); return create(targetConfigs.getTarget(targetName), workflows, ignoreDisabled); } - public static TestPlan create( + public static TestBenchPlan create( TargetConfiguration targetConfiguration, List workflows, Boolean ignoreDisabled) { var target = new Target(targetConfiguration); @@ -88,7 +94,7 @@ public static TestPlan create( "testbench/workflows" )); - return new TestPlan( + return new TestBenchPlan( target, specFiles, workflows == null ? Set.of() : Set.copyOf(workflows), diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/TestPlanFile.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/TestPlanFile.java index 7bbc67e5aa..6d1139fcf5 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/testbench/TestPlanFile.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/TestPlanFile.java @@ -27,7 +27,7 @@ * *

*

- * The {@link TestPlan#fromFile(Path)} will run Apache command style substituions using {@link System#getenv(String)} + * The {@link TestBenchPlan#fromFile(Path)} will run Apache command style substituions using {@link System#getenv(String)} * as the source for replacements. This allows some sensitive information to be put into the env rather than a * file, and for the runner to make multiple calls with different env vars. *

diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/AssertionFactory.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/AssertionFactory.java index fcbc76b5e0..8d6ed04b6f 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/AssertionFactory.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/AssertionFactory.java @@ -1,7 +1,7 @@ package io.stargate.sgv2.jsonapi.testbench.assertions; import com.fasterxml.jackson.databind.JsonNode; -import io.stargate.sgv2.jsonapi.testbench.TestPlan; +import io.stargate.sgv2.jsonapi.testbench.TestBenchPlan; import io.stargate.sgv2.jsonapi.testbench.testspec.TestCommand; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; @@ -21,7 +21,7 @@ non-sealed interface AssertionMatcherFactory extends AssertionFactory { @FunctionalInterface non-sealed interface TemplatedAssertionFactory extends AssertionFactory { List create( - TestPlan testPlan, JsonNode template, TestCommand testCommand, JsonNode args); + TestBenchPlan testPlan, JsonNode template, TestCommand testCommand, JsonNode args); } static boolean isValidFactoryMethod(Method method) { @@ -34,7 +34,7 @@ static boolean isValidFactoryMethod(Method method) { if (List.class.isAssignableFrom(method.getReturnType())) { var p = method.getParameterTypes(); return p.length == 4 - && p[0] == TestPlan.class + && p[0] == TestBenchPlan.class && p[1] == JsonNode.class && p[2] == TestCommand.class && p[3] == JsonNode.class; @@ -114,7 +114,7 @@ final class WrappedTemplatedAssertionFactory extends WrappedMethod @Override public List create( - TestPlan testPlan, JsonNode template, TestCommand testCommand, JsonNode args) { + TestBenchPlan testPlan, JsonNode template, TestCommand testCommand, JsonNode args) { return invoke(testPlan, template, testCommand, args); } } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/Templated.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/Templated.java index fe54cb08ea..b4b0dde278 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/Templated.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/Templated.java @@ -2,7 +2,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; -import io.stargate.sgv2.jsonapi.testbench.TestPlan; +import io.stargate.sgv2.jsonapi.testbench.TestBenchPlan; import io.stargate.sgv2.jsonapi.testbench.testspec.TestCommand; import java.util.List; @@ -13,7 +13,7 @@ public class Templated { } public static List isSuccess( - TestPlan testPlan, JsonNode template, TestCommand testCommand, JsonNode args) { + TestBenchPlan testPlan, JsonNode template, TestCommand testCommand, JsonNode args) { var commandTemplate = template.get(testCommand.commandName().getApiName()); if (commandTemplate == null) { throw new IllegalArgumentException( @@ -24,7 +24,7 @@ public static List isSuccess( } private static List runTemplate( - TestPlan testPlan, ObjectNode template, TestCommand testCommand, JsonNode args) { + TestBenchPlan testPlan, ObjectNode template, TestCommand testCommand, JsonNode args) { return template.properties().stream() .map(entry -> new TestAssertion.AssertionDefinition(entry.getKey(), entry.getValue())) .map(def -> TestAssertion.buildAssertion(testPlan, testCommand, def)) diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/TestAssertion.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/TestAssertion.java index ae1cecf204..61f1610583 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/TestAssertion.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/TestAssertion.java @@ -1,7 +1,7 @@ package io.stargate.sgv2.jsonapi.testbench.assertions; import com.fasterxml.jackson.databind.JsonNode; -import io.stargate.sgv2.jsonapi.testbench.TestPlan; +import io.stargate.sgv2.jsonapi.testbench.TestBenchPlan; import io.stargate.sgv2.jsonapi.testbench.testrun.TestExecutionCondition; import io.stargate.sgv2.jsonapi.testbench.testrun.TestNodeFactory; import io.stargate.sgv2.jsonapi.testbench.testrun.TestRunResponse; @@ -25,7 +25,7 @@ public interface TestAssertion { DynamicNode testNodes(TestNodeFactory testNodeFactory, TestUri.Builder uriBuilder, AtomicReference testResponse, TestExecutionCondition testExecutionCondition); - static List forSuccess(TestPlan testPlan, TestCommand testCommand) { + static List forSuccess(TestBenchPlan testPlan, TestCommand testCommand) { var builder = Stream.builder() @@ -34,20 +34,20 @@ static List forSuccess(TestPlan testPlan, TestCommand testCommand return buildAssertions(testPlan, testCommand, builder.build()); } - static List buildAssertions(TestPlan testPlan, TestCase testCase) { + static List buildAssertions(TestBenchPlan testPlan, TestCase testCase) { var defs = testCase.asserts().properties().stream().map(AssertionDefinition::create); return buildAssertions(testPlan, testCase.command(), defs); } static List buildAssertions( - TestPlan testPlan, TestCommand testCommand, Stream defs) { + TestBenchPlan testPlan, TestCommand testCommand, Stream defs) { return defs.map(def -> buildAssertion(testPlan, testCommand, def)).toList(); } static TestAssertion buildAssertion( - TestPlan testPlan, TestCommand testCommand, AssertionDefinition def) { + TestBenchPlan testPlan, TestCommand testCommand, AssertionDefinition def) { return def.addFactory(AssertionFactory.REGISTRY).build(testPlan, testCommand); } @@ -71,7 +71,7 @@ AssertionDefWithFactory addFactory(AssertionFactoryRegistry registry) { /** */ record AssertionDefWithFactory(AssertionFactory.WrappedMethod method, JsonNode args) { - TestAssertion build(TestPlan testPlan, TestCommand testCommand) { + TestAssertion build(TestBenchPlan testPlan, TestCommand testCommand) { return switch (method) { case AssertionFactory.WrappedAssertionMatcherFactory factory -> diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/reporting/DynamicTreeListener.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/reporting/DynamicTreeListener.java index 750e1bbdc6..52cc31db08 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/testbench/reporting/DynamicTreeListener.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/reporting/DynamicTreeListener.java @@ -1,62 +1,70 @@ package io.stargate.sgv2.jsonapi.testbench.reporting; +import io.stargate.sgv2.jsonapi.testbench.TestBenchPlan; import io.stargate.sgv2.jsonapi.testbench.testrun.TestUri; import java.util.*; import java.util.concurrent.ConcurrentHashMap; -import org.apache.maven.plugin.surefire.report.*; + import org.junit.platform.engine.TestExecutionResult; import org.junit.platform.engine.UniqueId; import org.junit.platform.engine.support.descriptor.UriSource; import org.junit.platform.launcher.TestExecutionListener; import org.junit.platform.launcher.TestIdentifier; import org.junit.platform.launcher.TestPlan; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import javax.annotation.Nullable; +/** + * Listens to the execution of the dynammic tests created by the {@link TestPlan} and logs + * the results using the {@link TestBenchConsoleWriter}. + *

+ * The class mus tbe registeded by a text file at + * META-INF/services/org.junit.platform.launcher.TestExecutionListener that contains the + * fully qualitified path to the class. + *

+ *

+ * Type types of output are generated: + *

    + *
  • As the tests are running the name of every test node is outputted together with progress, so we can + * see how long there is to go. See {@link TestBenchConsoleWriter#testStarted(int, int, TestReportingTracker)}. + * At this point we do not know how long child nodes will take to process and what their result will be.
    • + *
  • Once complet a summary is outputted that does not include every node to brevity, + * see {@link TestBenchConsoleWriter#allTestsFinished(TestReportingTracker)}. At this point we know + * how long child nodes took to process and their result.
    • + *
+ *

+ */ public class DynamicTreeListener implements TestExecutionListener { - private static final Logger LOGGER = LoggerFactory.getLogger(DynamicTreeListener.class); - private Integer totalTestCount = null; private int startedTestCount = 0; + private TestReportingTracker rootTracker; + // Keyed on TestIdentifier.uniqueID() see {@link TestIdentifier#uniqueId()} private final Map testTrackers = new ConcurrentHashMap<>(); - private final Map startTimes = new ConcurrentHashMap<>(); - private final Map containerStats = new ConcurrentHashMap<>(); - private final TestBenchConsoleWriter writer = new TestBenchConsoleWriter(); - @Override - public void testPlanExecutionStarted(TestPlan testPlan) { - } - @Override public void testPlanExecutionFinished(TestPlan testPlan) { + // All done, write the summary. writer.allTestsFinished(rootTracker); } - @Override - public void dynamicTestRegistered(TestIdentifier id) { - } - @Override public void executionStarted(TestIdentifier id) { if (!isTestBenchNode(id)) { return; } + var tracker = getCreateTestTracker(id); if (tracker == null) { return; } - // we will not see the test count until we see the first dymamic test node we create, e.g. - //TestPlan: smoketest-aws-us-east-1 on astra workflows vectorize-header-workflow - // if we have a tracker, its one of our tests. + // Test count will not be in the system properties until we see the first dymamic test node we create, e.g. + // "TestPlan: smoketest-aws-us-east-1 on astra workflows vectorize-header-workflow" + // because the nodes will not have been created until then. if (totalTestCount == null) { - totalTestCount = Integer.parseInt(System.getProperty("testbench.test.count", "0")); + totalTestCount = Integer.parseInt(System.getProperty(TestBenchPlan.TEST_PLAN_TEST_COUNT_PROPERTY, "0")); } writer.testStarted(totalTestCount,++startedTestCount, tracker); @@ -74,6 +82,7 @@ public void executionFinished(TestIdentifier id, TestExecutionResult result) { @Override public void executionSkipped(TestIdentifier id, String reason) { + // Looks like we never get skipped, included for completeness. var tracker = getCreateTestTracker(id); if (tracker == null) { return; @@ -82,74 +91,22 @@ public void executionSkipped(TestIdentifier id, String reason) { tracker.executionSkipped(); } + /** + * Determine if tests are running that we should be tracking. + */ private static boolean isTestBenchNode(TestIdentifier testIdentifier) { + + // This is the uniqueID created by jupiter as it is traversing the code, once we get to the + // nodes that are created by the test plan that have different formatting return testIdentifier .getUniqueId() - .startsWith("[engine:junit-jupiter]/[class:io.stargate.sgv2.jsonapi."); + .startsWith("[engine:junit-jupiter]/[class:io.stargate.sgv2.jsonapi.testbench."); } - // private String buildClassName(TestIdentifier id) { - // List parts = new ArrayList<>(); - // Optional current = Optional.of(id); - // while (current.isPresent()) { - // TestIdentifier node = current.get(); - // if (isEngineOrClass(node)) { - // break; - // } - // parts.add(0, node.getDisplayName()); - // current = testPlan.getParent(node); - // } - // return rootClassName(id) + (parts.isEmpty() ? "" : "$" + String.join("$", parts)); - // } - // - // private String rootClassName(TestIdentifier id) { - // Optional current = Optional.of(id); - // while (current.isPresent()) { - // TestIdentifier node = current.get(); - // if (isClassNode(node)) { - // return node.getDisplayName(); - // } - // current = testPlan.getParent(node); - // } - // return "Unknown"; - // } - // - // private SimpleReportEntry toReportEntry(TestIdentifier id, String methodName, String - // methodDisplay) { - // return new SimpleReportEntry( - // RunMode.NORMAL_RUN, - // System.currentTimeMillis(), - // buildClassName(id), - // id.getDisplayName(), - // methodName, - // methodDisplay - // ); - // } - // - // private boolean isEngineOrClass(TestIdentifier id) { - // return id.getUniqueId().startsWith("[engine:") || isClassNode(id); - // } - // - // private boolean isClassNode(TestIdentifier id) { - // return id.getUniqueId().contains("[class:"); - // } - // - // private long elapsed(TestIdentifier id) { - // return System.currentTimeMillis() - startTimes.getOrDefault(id.getUniqueId(), - // System.currentTimeMillis()); - // } - // - // private Optional nearestContainerStats(TestIdentifier id) { - // Optional current = testPlan.getParent(id); - // while (current.isPresent()) { - // if (containerStats.containsKey(current.get().getUniqueId())) { - // return Optional.of(containerStats.get(current.get().getUniqueId())); - // } - // current = testPlan.getParent(current.get()); - // } - // return Optional.empty(); - // } - + /** + * We use a Tracker for every node in the test plan, to track the execution time + * and result of it and all of its children. + */ private TestReportingTracker getCreateTestTracker(TestIdentifier testIdentifier) { var existingTracker = testTrackers.get(testIdentifier.getUniqueIdObject()); @@ -157,7 +114,8 @@ private TestReportingTracker getCreateTestTracker(TestIdentifier testIdentifier) return existingTracker; } - // if this is not a TESTRUN:// it is not a node we care about + // The getSource() is a URI, jupiter / junit use it to identify the test file, but we dont have those. + // We use the {@link TestUri} instead. We need one to know what sort of test node this is var testUri = testIdentifier .getSource() @@ -177,7 +135,7 @@ private TestReportingTracker getCreateTestTracker(TestIdentifier testIdentifier) testUri.get().leafType() != TestUri.Segment.TARGET ? Objects.requireNonNull( testTrackers.get(testIdentifier.getParentIdObject().get()), - "parentID not found for testIdentifier: " + testIdentifier.toString()) + "parentID not found for testIdentifier: " + testIdentifier) : null; var tracker = new TestReportingTracker(testIdentifier, testUri.get(), parentTracker); @@ -189,22 +147,28 @@ private TestReportingTracker getCreateTestTracker(TestIdentifier testIdentifier) return tracker; } - /** */ + /** + * Container for tracking the execution of a test, and all of its children. + *

---

+ * */ public class TestReportingTracker { private final TestIdentifier identifier; private final TestUri runUri; private final TestReportingTracker parent; private final int depth; + private final TestContainerStats stats; private final List children = new ArrayList<>(); + // Set when we know the test completed, value on the test result is an optional private Optional throwable = Optional.empty(); + // Set when we know the test completed private TestExecutionResult.Status junitStatus; - private final TestContainerStats stats; public TestReportingTracker( TestIdentifier identifier, TestUri runUri, TestReportingTracker parent) { + this.identifier = identifier; this.runUri = runUri; this.parent = parent; @@ -220,10 +184,10 @@ public Optional throwable() { return throwable; } - @Nullable public TestExecutionResult.Status junitStatus(){ return junitStatus; } + public TestIdentifier identifier() { return identifier; } @@ -237,7 +201,7 @@ public TestReportingTracker parent() { } public List children() { - return children; + return Collections.unmodifiableList(children); } public int depth() { @@ -248,6 +212,11 @@ public TestContainerStats stats() { return stats; } + /** + * Call when the execution of the test is finished, updates tracking for the node and + * for any ancestors. + * @param result + */ public void executionFinished(TestExecutionResult result) { junitStatus = result.getStatus(); throwable = result.getThrowable(); @@ -268,6 +237,7 @@ private void descendantExecutionFinished(TestReportingTracker originalTracker, parent.descendantExecutionFinished(originalTracker, result); } } + public void executionSkipped() { if (stats != null) { stats.testSkipped(); @@ -278,20 +248,28 @@ public void executionSkipped() { } } - /** Modeled on org.apache.maven.plugin.surefire.report.TestSetStats */ + /** + * Count the tests that failed etc. + *

+ * Modeled on org.apache.maven.plugin.surefire.report.TestSetStats + *

+ * */ public class TestContainerStats { - private TestContainerStats parent; - private final long startedAtMillis; - private long lastFinishedAtMillis; + private long selfOrDescendantFinishedAtMillis; private int successful; + // Aborted happens when the test node decided not to run, normally by calling + // Assumptions.assumeTrue() which throws a TestAbortedException which is tracked + // differently by junit. private int aborted; + // An actual failure of an assertion or unexpected error thrown private int failures; + // dont think used, kept for completeness private int skipped; public TestContainerStats() { @@ -299,9 +277,9 @@ public TestContainerStats() { } public long elapsedMillis() { - return lastFinishedAtMillis == 0 + return selfOrDescendantFinishedAtMillis == 0 ? System.currentTimeMillis() - startedAtMillis - : lastFinishedAtMillis - startedAtMillis; + : selfOrDescendantFinishedAtMillis - startedAtMillis; } public int successful() { @@ -321,7 +299,7 @@ public int skipped() { } public void testCompleted(TestReportingTracker tracker, TestExecutionResult result) { - lastFinishedAtMillis = System.currentTimeMillis(); + selfOrDescendantFinishedAtMillis = System.currentTimeMillis(); // we only update the stats IF the test we are tracking is a TEST, we do not update for containers. if (tracker.identifier().isTest()) { @@ -334,7 +312,7 @@ public void testCompleted(TestReportingTracker tracker, TestExecutionResult res } public void testSkipped() { - lastFinishedAtMillis = System.currentTimeMillis(); + selfOrDescendantFinishedAtMillis = System.currentTimeMillis(); skipped++; } } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/reporting/TestBenchConsoleWriter.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/reporting/TestBenchConsoleWriter.java index cc5d84e2e1..17b414eec2 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/testbench/reporting/TestBenchConsoleWriter.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/reporting/TestBenchConsoleWriter.java @@ -15,25 +15,48 @@ import org.slf4j.LoggerFactory; /** - * Writes messages for the output of a Test Bench runs using a LOGGER. + * Writes messages for the output of Test Bench using the logging system. * - *

Works in two modes, because we need to wait for all the tests to complete so we know what was - * successful. For info on how the output will look like see {@link Theme} and {@link - * org.apache.maven.surefire.shared.utils.logging.AnsiMessageBuilder} - * - *

Call {@link #testStarted(DynamicTreeListener.TestReportingTracker)} when a test is being - * executed and it has started, will print the progress of the tests. There is no correlating test - * finished. + *

+ * NOTE: to get the best output via the logging system, we need to configure to remove the normal formatting + * that comes with logging. Below should be in the `applicaiton.yaml` in addition to the regular config + * 

+ * quarkus:
+ *   log:
+ *     console:
+ *       format: "%-5p [%t] %d{yyyy-MM-dd HH:mm:ss,SSS} %F:%L - %m%n"
+ *     handler:
+ *       console:
+ *         PLAIN_CONSOLE:
+ *           format: "%m%n"
+ *     category:
+ *       'io.stargate.sgv2.jsonapi.testbench.reporting.TestBenchConsoleWriter':
+ *         level: INFO
+ *         handlers:
+ *           - PLAIN_CONSOLE
+ *         use-parent-handlers: false
+ *
+ *

* + *

Works in two modes because we need to wait for all the tests to complete so we know what was + * successful. For info on how the output will look see {@link Theme} and {@link + * org.apache.maven.surefire.shared.utils.logging.AnsiMessageBuilder} + *

+ *

Call {@link #testStarted(int, int, DynamicTreeListener.TestReportingTracker)} when a test starts + * executing, will print the progress of the tests + *

*

Call {@link #allTestsFinished(DynamicTreeListener.TestReportingTracker)} with the root * tracker for the rest run, this should be called once all the tests have completed so we know what * failed and how long things took. This will print the full test tree, but only go down to the * request + assertion level for test scenarios that have failed. + *

*/ public class TestBenchConsoleWriter { private static final Logger LOGGER = LoggerFactory.getLogger(TestBenchConsoleWriter.class); + public static final String ENV_TEST_BENCH_REPORT_PATH = "test-bench-report-path"; + private boolean firstLine = true; private final Theme theme; @@ -49,8 +72,24 @@ public TestBenchConsoleWriter(Theme theme) { /** * Writes that a test has started, without knowing how it will end, used when the test suite is * running. - * - * @param tracker Tracker for the test that is running. + *

+ * Example output: + * 

+   * ── ── ── ── ── ── ── ── ── ── ── ── ── ── ── ── ── ── ── ──
+   * Running Test Bench, results shown at completion...
+   * ── ── ── ── ── ── ── ── ── ── ── ── ── ── ── ── ── ── ── ──
+   *  ┬─ 1 of 1465: TestPlan: smoketest-prod-aws-ap-south-1 (701031671627) on astra (node:axM)
+   * ├─ 2 of 1465: Workflow: vectorize-shared-workflow  (node:alW)
+   *    ├─ 3 of 1465: Job: open-ai-vectorize  (node:ab7)
+   *       ├─ 4 of 1465: TestSuite: vectorize-shared-auth  (node:ab6)
+   *          ├─ 5 of 1465: TestEnv: [CREDENTIAL=open-ai-key, MODEL=text-embedding-3-small, PROVIDER=openai]  (node:aaN)
+   *             ├─ 6 of 1465: Request: SetupRequest[1]: CREATE_COLLECTION (node:aaf)
+   *                ├─ 7 of 1465: Command: createCollection (node:aaa)
+   *
+ *

+ * @param totalTestCount the total number of tests that will be run + * @param startedTestCount the number of tests that have started running, including this one + * @param tracker the tracker for the test that started */ public void testStarted(int totalTestCount, int startedTestCount, DynamicTreeListener.TestReportingTracker tracker) { @@ -67,34 +106,46 @@ public void testStarted(int totalTestCount, int startedTestCount, DynamicTreeLis .newline(); } - // if there is no parent, this is the first test that is running. - // display name is the name the TestBench put on the container - if (tracker.parent() == null) { - buffer.a(theme.down()); - } else { - buffer - .a(theme.blank().repeat(tracker.depth() - 1)) - .a(theme.entry()); - } + writeTreeOutline(buffer, tracker); - buffer.a(startedTestCount).a(" of ").a(totalTestCount).a(": "); - buffer.strong(tracker.identifier().getDisplayName()); + // the progress "x of Y" and the displayName from the node set when it was created by the TestPlan. + // NOTE: the "(node:aaN)" is part of the displayName, it is so that when we print failed nodes we can + // find them in the full log easily. + buffer + .a(startedTestCount) + .a(" of ") + .a(totalTestCount) + .a(": ") + .strong(tracker.identifier().getDisplayName()); LOGGER.info(buffer.toString()); } /** - * Call when all the tests have finished running, so we can print a summary for test suites that pass, and + * Call when all the tests have finished running, so we can print a summary for test suites that pass and * details for those that fail. - * @param rootTracker + *

+ * Writes two types of output: + *

    + *
  • If {@link LOGGER#isInfoEnabled()} logs a detailed report of the test results, skips details of + * assertions that have completed successfully. See {@link #writeCompletedSummary(MessageBuilder, DynamicTreeListener.TestReportingTracker, boolean)} + *
    • + *
  • If {@link #ENV_TEST_BENCH_REPORT_PATH} is set in the path, writes a markdown file in a format + * to be included in the summary for GitHub actions via the $GITHUB_STEP_SUMMARY. + *
    • + *
+ *

+ * @param rootTracker Tracker for the root of the test tree, this is the one that has the full test tree + * under it. */ public void allTestsFinished(DynamicTreeListener.TestReportingTracker rootTracker) { var reportBuffer = buffer(); - writeCompletedSummary(reportBuffer, rootTracker, true, false); + writeCompletedSummary(reportBuffer, rootTracker, false); var testReport = reportBuffer.toString(); if (LOGGER.isInfoEnabled()){ + var logLineBuffer = buffer(); logLineBuffer .newline() @@ -108,7 +159,7 @@ public void allTestsFinished(DynamicTreeListener.TestReportingTracker rootTracke LOGGER.info(logLineBuffer.toString()); } - var reportFilePath = System.getProperty("test-bench-report-path"); + var reportFilePath = System.getProperty(ENV_TEST_BENCH_REPORT_PATH); if (reportFilePath != null) { LOGGER.info("Writing report file to: {}", reportFilePath); @@ -117,6 +168,7 @@ public void allTestsFinished(DynamicTreeListener.TestReportingTracker rootTracke writeTestDesc(testPlanNodeDesc, rootTracker); testPlanNodeDesc.newline(); + // the failed nodes and their throwable, if any String failureReport; if (rootTracker.stats().failures() == 0){ failureReport = "No failures"; @@ -126,8 +178,8 @@ public void allTestsFinished(DynamicTreeListener.TestReportingTracker rootTracke failureReport = failureReportBuffer.toString(); } - // report is a markdown - // GitHub collapsable sections + // report is a markdown file + // Using GitHub collapsable sections // https://docs.github.com/en/get-started/writing-on-github/working-with-advanced-formatting/organizing-information-with-collapsed-sections var markdownReport = """ ## %s @@ -164,43 +216,102 @@ public void allTestsFinished(DynamicTreeListener.TestReportingTracker rootTracke } /** - * TestPlan: smoketest-aws-us-east-1 on astra workflows vectorize-header-workflow Workflow: - * vectorize-header-workflow Job: nvidia-vectorize TestSuite: vectorize-header-auth TestEnv: - * [MODEL=NV-Embed-QA, PROVIDER=nvidia] RESULTS.... TestEnv: [MODEL=nvidia/nv-embedqa-e5-v5, - * PROVIDER=nvidia] RESULTS... + * Walks the test tree, outputs the results of running each node now that we know the completed results. + *

+ * Example (the header is put in th by the caller) : + * 

+   * ── ── ── ── ── ── ── ── ── ── ── ── ── ── ── ── ── ── ── ──
+   * Test Bench Results
+   * ── ── ── ── ── ── ── ── ── ── ── ── ── ── ── ── ── ── ── ──
+   * ┬─ ✔ TestPlan: smoketest-prod-aws-ap-south-1 (701031671627) on astra (node:axM) - 835 s   Successful: 756, Failures: 0, Aborted: 0
+   * ├─ ✔ Workflow: vectorize-shared-workflow  (node:alW) - 417 s   Successful: 378, Failures: 0, Aborted: 0
+   *    ├─ ✔ Job: open-ai-vectorize  (node:ab7) - 76 s   Successful: 63, Failures: 0, Aborted: 0
+   *       ├─ ✔ TestSuite: vectorize-shared-auth  (node:ab6) - 76 s   Successful: 63, Failures: 0, Aborted: 0
+   *          ├─ ✔ TestEnv: [CREDENTIAL=open-ai-key, MODEL=text-embedding-3-small, PROVIDER=openai]  (node:aaN) - 30 s   Successful: 21, Failures: 0, Aborted: 0
+   *          ├─ ✔ TestEnv: [CREDENTIAL=open-ai-key, MODEL=text-embedding-3-large, PROVIDER=openai]  (node:abr) - 23 s   Successful: 21, Failures: 0, Aborted: 0
+   *          ├─ ✔ TestEnv: [CREDENTIAL=open-ai-key, MODEL=text-embedding-ada-002, PROVIDER=openai]  (node:ab5) - 22 s   Successful: 21, Failures: 0, Aborted: 0
+   *
+ *

+ *

+ * NOTE: Because there can be 1,000s of test nodes, we do not below the TestEnv nodes unless there is a failure. + * Below those nodes are where the assertions that will have failed exist. We traverse down to the first failed test node, + * and then want to show which sibling (or cousin) nodes aborted because of the failure. For example: + * 

+   * ┬─ ✘ TestPlan: smoketest-prod-aws-eu-central-1 (105817733955) on astra (node:axM) - 413 s   Successful: 436, Failures: 20, Aborted: 300
+   * ├─ ✘ Workflow: vectorize-shared-workflow  (node:alW) - 41 s   Successful: 90, Failures: 18, Aborted: 270
+   *    ├─ ✘ Job: open-ai-vectorize  (node:ab7) - 10 s   Successful: 15, Failures: 3, Aborted: 45
+   *       ├─ ✘ TestSuite: vectorize-shared-auth  (node:ab6) - 10 s   Successful: 15, Failures: 3, Aborted: 45
+   *          ├─ ✘ TestEnv: [CREDENTIAL=open-ai-key, MODEL=text-embedding-3-small, PROVIDER=openai]  (node:aaN) - 5 s   Successful: 5, Failures: 1, Aborted: 15
+   *             ├─ ✘ Request: SetupRequest[1]: CREATE_COLLECTION (node:aaf) - 4 s   Successful: 2, Failures: 1, Aborted: 0
+   *                ├─ ✔ Command: createCollection (node:aaa)
+   *                ├─ ✘ Assertions (node:aae) - 0 s   Successful: 1, Failures: 1, Aborted: 0
+   *                   ├─ ✘ isSuccess (node:aad) - 0 s   Successful: 1, Failures: 1, Aborted: 0
+   *                      ├─ ✔ success [http status is 200] (node:aab)
+   *                      ├─ ✘ isDDLSuccess [body('$') - responseIsDDLSuccess: REQUIRED:[status], OPTIONAL:[], FORBIDDEN:[data, errors]] (node:aac)
+   *             ├─ ✘ Request: SetupRequest[2]: INSERT_ONE (node:aal) - 0 s   Successful: 0, Failures: 0, Aborted: 3
+   *             ├─ ✘ Request: SetupRequest[3]: INSERT_MANY (node:aar) - 0 s   Successful: 0, Failures: 0, Aborted: 3
+   *             ├─ ✘ Request: TestCase: name=basic findMany (node:aay) - 0 s   Successful: 0, Failures: 0, Aborted: 4
+   *             ├─ ✘ Request: TestCase: name=findOneAndUpdate (node:aaG) - 0 s   Successful: 0, Failures: 0, Aborted: 5
+   *             ├─ ✔ Request: CleanupRequest[4]: DELETE_COLLECTION (node:aaM) - 0 s   Successful: 3, Failures: 0, Aborted: 0
+   *
+ * + * In the first example we do not go down to the Request nodes because they did not fail or abort. + *

* - * @param buffer - * @param tracker - * @param isRoot + * @param buffer Buffer to append the message to. + * @param tracker The tracker we are writing out the summary for, and may then traverse its children. + * @param parentFailures Set true if any parent nodes have failures, this will cause us to traverse down to the children */ private void writeCompletedSummary( MessageBuilder buffer, DynamicTreeListener.TestReportingTracker tracker, - boolean isRoot, boolean parentFailures) { - // the tree part of the line - if (isRoot) { - buffer.a(theme.down()); - } else { - buffer.a(theme.blank().repeat(tracker.depth() - 1)).a(theme.entry()); - } - + writeTreeOutline(buffer, tracker); writeTestDesc(buffer, tracker); buffer.newline(); // If we have a TestEnv then we want to write out the summary of results for it, otherwise // descend until we get one - // OF if there are FAILURES then we descend, these are tests that ran but assertion failed. + // OR if there are FAILURES then we descend, these are tests that ran but assertion failed. // we do not descend for aborted, these are tests that did not run because of previous failure. for (var child : tracker.children()){ var hasFailures = child.stats() != null && child.stats().failures() > 0; + if (!child.runUri().leafType().descendantOf(TestUri.Segment.ENV) || hasFailures || parentFailures) { - writeCompletedSummary(buffer, child, false, hasFailures); + writeCompletedSummary(buffer, child, hasFailures); } } } + /** + * Writes the test node name, and it's throwable if it Failed, as in the assertions failed or it threw + * an exception. + *

+ * Example: + * 

+   * ✘ isDDLSuccess [body('$') - responseIsDDLSuccess: REQUIRED:[status], OPTIONAL:[], FORBIDDEN:[data, errors]] (node:aac)
+   *
+   * [responseIsDDLSuccess: REQUIRED:[status], OPTIONAL:[], FORBIDDEN:[data, errors]]
+   * Expecting actual:
+   *   {"errors"=[{"errorCode"="VECTORIZE_CREDENTIAL_INVALID", "family"="REQUEST", "id"="95b651e3-815b-4be3-8cad-3074fcd86cd9", "message"="Invalid credential name for vectorize, with error: Embedding Gateway unable to resolve authentication type.
+   * Underlying problem: Sync service has internal server error. Error Code: 500; response description: Internal Server Error.      .", "scope"="SCHEMA", "title"="Invalid credential name for vectorize"}]}
+   * to contain key:
+   *   "status"
+   * -----
+   *
+   * ✘ isDDLSuccess [body('$') - responseIsDDLSuccess: REQUIRED:[status], OPTIONAL:[], FORBIDDEN:[data, errors]] (node:aaQ)
+   *
+   * [responseIsDDLSuccess: REQUIRED:[status], OPTIONAL:[], FORBIDDEN:[data, errors]]
+   * Expecting actual:
+   *   {"errors"=[{"errorCode"="VECTORIZE_CREDENTIAL_INVALID", "family"="REQUEST", "id"="9d512a57-ca08-491b-8440-a5abc14b99a4", "message"="Invalid credential name for vectorize, with error: Embedding Gateway unable to resolve authentication type.
+   * Underlying problem: Sync service has internal server error. Error Code: 500; response description: Internal Server Error.      .", "scope"="SCHEMA", "title"="Invalid credential name for vectorize"}]}
+   * to contain key:
+   *   "status"
+   * -----
+   *
+ *

+ */ private void writeFailureMessages(MessageBuilder buffer, DynamicTreeListener.TestReportingTracker tracker) { // if we have a throwable, write out the tree node test and the error it generated. @@ -214,6 +325,7 @@ private void writeFailureMessages(MessageBuilder buffer, DynamicTreeListener.Tes } tracker.children().forEach(child -> writeFailureMessages(buffer, child)); } + private void writeTestDesc(MessageBuilder buffer, DynamicTreeListener.TestReportingTracker tracker) { // Icon for success for failure, @@ -235,18 +347,8 @@ private void writeTestDesc(MessageBuilder buffer, DynamicTreeListener.TestReport // The name of the container or test buffer.strong(tracker.identifier().getDisplayName()); - // timing info if available - if (tracker.stats() != null){ - buffer.a(" - %s s".formatted(tracker.stats().elapsedMillis() / 1000)); - } - // if we have stats write a stats line, these are aggregate for all things below. - // alternative is to only print them for Test Environment these are lines like - // TestEnv: [MODEL=text-embedding-3-small, PROVIDER=openai] - // to do that do this test: (tracker.runUri().leafType() == TestUri.Segment.ENV) - if (tracker.stats() != null) { - buffer.a(theme.blank()); writeTestStats(buffer, tracker); } @@ -255,8 +357,22 @@ private void writeTestDesc(MessageBuilder buffer, DynamicTreeListener.TestReport private void writeTestStats(MessageBuilder buffer, DynamicTreeListener.TestReportingTracker tracker) { buffer + .a(" - %s s".formatted(tracker.stats().elapsedMillis() / 1000)) + .a(theme.blank()) .a("Successful: ").a( tracker.stats().successful()).a(", ") .a("Failures: ").a( tracker.stats().failures()).a(", ") .a("Aborted: ").a( tracker.stats().aborted()); } + + private void writeTreeOutline(MessageBuilder buffer, DynamicTreeListener.TestReportingTracker tracker) { + + // Indenting and tree building + if (tracker.parent() == null) { + buffer.a(theme.down()); + } else { + buffer + .a(theme.blank().repeat(tracker.depth() - 1)) + .a(theme.entry()); + } + } } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/testrun/TestNodeFactory.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testrun/TestNodeFactory.java index 799651bdfc..ff820e1b81 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/testbench/testrun/TestNodeFactory.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testrun/TestNodeFactory.java @@ -1,6 +1,6 @@ package io.stargate.sgv2.jsonapi.testbench.testrun; -import io.stargate.sgv2.jsonapi.testbench.TestPlan; +import io.stargate.sgv2.jsonapi.testbench.TestBenchPlan; import io.stargate.sgv2.jsonapi.testbench.testspec.Job; import io.stargate.sgv2.jsonapi.testbench.testspec.TestSpecMeta; import io.stargate.sgv2.jsonapi.testbench.testspec.TestSuiteSpec; @@ -28,14 +28,14 @@ public class TestNodeFactory { private final NodeCode nodeCode = new NodeCode(); - private final TestPlan testPlan; + private final TestBenchPlan testPlan; private int totalNodeCount = 0; - public TestNodeFactory(TestPlan testPlan) { + public TestNodeFactory(TestBenchPlan testPlan) { this.testPlan = testPlan; } - public TestPlan testPlan() { + public TestBenchPlan testPlan() { return testPlan; } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/Job.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/Job.java index af92e66e4e..0c2ec128ac 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/Job.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/Job.java @@ -2,7 +2,7 @@ -import io.stargate.sgv2.jsonapi.testbench.TestPlan; +import io.stargate.sgv2.jsonapi.testbench.TestBenchPlan; import io.stargate.sgv2.jsonapi.testbench.testrun.TestNodeFactory; import io.stargate.sgv2.jsonapi.testbench.testrun.TestRunEnv; import io.stargate.sgv2.jsonapi.testbench.testrun.TestUri; @@ -41,7 +41,7 @@ public DynamicContainer testNode(TestNodeFactory testNodeFactory, TestUri.Builde testNodeFactory.addLifecycle(uriBuilder.clone(), this, testSuiteNodes)); } - public Stream testSuites(TestPlan testPlan) { + public Stream testSuites(TestBenchPlan testPlan) { Stream.Builder allTests = Stream.builder(); tests() .forEach( @@ -52,7 +52,7 @@ public Stream testSuites(TestPlan testPlan) { return allTests.build(); } - public TestRunEnv withoutMatrix(TestPlan testPlan) { + public TestRunEnv withoutMatrix(TestBenchPlan testPlan) { var fromEnv = new TestRunEnv(); @@ -65,7 +65,7 @@ public TestRunEnv withoutMatrix(TestPlan testPlan) { return fromEnv.clone().put(fromVariables); } - public List allEnvironments(TestPlan testPlan) { + public List allEnvironments(TestBenchPlan testPlan) { var fromEnv = new TestRunEnv(); diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/TestSpec.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/TestSpec.java index d09fc6afdb..ed8f81874d 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/TestSpec.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/TestSpec.java @@ -1,9 +1,11 @@ package io.stargate.sgv2.jsonapi.testbench.testspec; +import io.stargate.sgv2.jsonapi.testbench.TestBenchPlan; + /** * A specification for objects in the TestBench world, such as a test suite. *

- * Implement this for any object types a {@link io.stargate.sgv2.jsonapi.testbench.TestPlan} needs + * Implement this for any object types a {@link TestBenchPlan} needs * to read from disk or know about. *

*/ diff --git a/src/test/resources/application.yaml b/src/test/resources/application.yaml index 44839370f2..5fd20660f0 100644 --- a/src/test/resources/application.yaml +++ b/src/test/resources/application.yaml @@ -26,7 +26,7 @@ quarkus: # Fine for JVM mode; if building native images, use index-dependency instead 'io.quarkus.deployment.steps.ReflectiveHierarchyStep': level: ERROR - 'io.stargate.sgv2.jsonapi.api.v1.vectorize.reporting.TestBenchConsoleWriter': + 'io.stargate.sgv2.jsonapi.testbench.reporting.TestBenchConsoleWriter': level: INFO handlers: - PLAIN_CONSOLE From efe736a5dd0f0e127cf821d038f87b236a468c85 Mon Sep 17 00:00:00 2001 From: Aaron Morton Date: Mon, 11 May 2026 16:20:03 +1200 Subject: [PATCH 89/89] Code tidy --- .../testbench/TestBenchByTestPlan.java | 20 +- .../sgv2/jsonapi/testbench/TestBenchPlan.java | 41 +-- .../sgv2/jsonapi/testbench/TestPlanFile.java | 42 +-- .../assertions/AssertionFactory.java | 106 +++++- .../assertions/AssertionFactoryRegistry.java | 50 ++- .../assertions/AssertionMatcher.java | 6 +- .../testbench/assertions/AssertionName.java | 25 +- .../testbench/assertions/BodyAssertion.java | 16 +- .../testbench/assertions/Describable.java | 4 + .../DescribableAssertionMatcher.java | 4 + .../testbench/assertions/Documents.java | 10 + .../jsonapi/testbench/assertions/Http.java | 2 + .../testbench/assertions/Response.java | 16 +- .../assertions/SingleTestAssertion.java | 29 +- .../jsonapi/testbench/assertions/Status.java | 2 + .../testbench/assertions/Templated.java | 44 ++- .../testbench/assertions/TestAssertion.java | 154 ++++++++- .../assertions/TestAssertionContainer.java | 45 +-- .../testbench/lifecycle/JobLifeCycle.java | 30 +- .../lifecycle/TestPlanLifecycle.java | 58 ++-- .../testbench/messaging/APIRequest.java | 71 ++-- .../testbench/messaging/APIResponse.java | 4 +- .../testbench/messaging/APIRetryPolicy.java | 281 +++++++-------- .../reporting/DynamicTreeListener.java | 81 ++--- .../reporting/TestBenchConsoleWriter.java | 225 ++++++------ .../testbench/targets/AstraBackend.java | 4 +- .../jsonapi/testbench/targets/Backend.java | 16 +- .../testbench/targets/CassandraBackend.java | 26 +- .../targets/ConnectionConfiguration.java | 13 +- .../jsonapi/testbench/targets/Target.java | 58 ++-- .../targets/TargetConfiguration.java | 6 +- .../testrun/DynamicTestExecutable.java | 17 +- .../testrun/TestExecutionCondition.java | 87 ++--- .../testbench/testrun/TestNodeFactory.java | 321 +++++++++--------- .../jsonapi/testbench/testrun/TestRunEnv.java | 24 +- .../testbench/testrun/TestRunRequest.java | 12 +- .../testbench/testrun/TestRunResponse.java | 28 +- .../jsonapi/testbench/testrun/TestUri.java | 2 +- .../testspec/AssertionTemplateSpec.java | 15 +- .../sgv2/jsonapi/testbench/testspec/Job.java | 5 +- .../jsonapi/testbench/testspec/SpecFile.java | 5 +- .../jsonapi/testbench/testspec/SpecFiles.java | 44 +-- .../testbench/testspec/TargetsSpec.java | 9 +- .../jsonapi/testbench/testspec/TestCase.java | 9 +- .../testbench/testspec/TestCommand.java | 39 +-- .../jsonapi/testbench/testspec/TestSpec.java | 10 +- .../testbench/testspec/TestSpecMeta.java | 7 +- .../testbench/testspec/TestSuiteSpec.java | 35 +- .../testbench/testspec/WorkflowSpec.java | 14 +- 49 files changed, 1254 insertions(+), 918 deletions(-) diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/TestBenchByTestPlan.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/TestBenchByTestPlan.java index 1e08e1eac2..e7516743b8 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/testbench/TestBenchByTestPlan.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/TestBenchByTestPlan.java @@ -1,10 +1,8 @@ package io.stargate.sgv2.jsonapi.testbench; import io.stargate.sgv2.jsonapi.testbench.testspec.SpecFiles; - import java.nio.file.Path; import java.util.stream.Stream; - import org.junit.jupiter.api.DynamicNode; import org.junit.jupiter.api.TestFactory; import org.slf4j.Logger; @@ -12,9 +10,12 @@ /** * Entry point for running a Test Bench from a Test Plan file. + * + *

Put the name of the test plan file in the TEST_PLAN_FILE env var, this can set + * the target to hit, and the workflows to run. See {@link TestPlanFile} + *

*

- * Put the name of the test plan file in the TEST_PLAN_FILE env var, this can - * set the target to hit, and the workflows to run. See {@link TestPlanFile} + * This will look like a unit test, so we only run when the env var is set. *

*/ public class TestBenchByTestPlan { @@ -25,6 +26,9 @@ public class TestBenchByTestPlan { public Stream runTestPlanFile() { var rawPath = System.getenv("TEST_PLAN_FILE"); + if (rawPath == null) { + return Stream.empty(); + } LOGGER.info("runTestPlanFile() - getting TEST_PLAN_FILE from ENV, rawPath={}", rawPath); var path = @@ -36,8 +40,12 @@ public Stream runTestPlanFile() { LOGGER.info("runTestPlanFile() - building test plan tree"); var testPlanNodeTree = testPlan.testNode(); - LOGGER.info("runTestPlanFile() - test plan tree build, totalNodeCount={}", testPlanNodeTree.totalNodeCount()); - System.setProperty(TestBenchPlan.TEST_PLAN_TEST_COUNT_PROPERTY, String.valueOf(testPlanNodeTree.totalNodeCount())); + LOGGER.info( + "runTestPlanFile() - test plan tree build, totalNodeCount={}", + testPlanNodeTree.totalNodeCount()); + System.setProperty( + TestBenchPlan.TEST_PLAN_TEST_COUNT_PROPERTY, + String.valueOf(testPlanNodeTree.totalNodeCount())); return Stream.of(testPlanNodeTree.root()); } } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/TestBenchPlan.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/TestBenchPlan.java index b9811c3c99..59012516f7 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/testbench/TestBenchPlan.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/TestBenchPlan.java @@ -1,7 +1,5 @@ package io.stargate.sgv2.jsonapi.testbench; - - import com.fasterxml.jackson.core.StreamReadFeature; import com.fasterxml.jackson.databind.MapperFeature; import com.fasterxml.jackson.databind.ObjectMapper; @@ -12,32 +10,28 @@ import io.stargate.sgv2.jsonapi.testbench.testrun.TestNodeFactory; import io.stargate.sgv2.jsonapi.testbench.testrun.TestUri; import io.stargate.sgv2.jsonapi.testbench.testspec.*; - import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.util.List; import java.util.Set; import java.util.stream.Stream; - import org.apache.commons.text.StringSubstitutor; import org.junit.jupiter.api.DynamicNode; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** - * - *

* NOTE: called "TestBenchPlan" to avoid collision with "org.junit.platform.launcher.TestPlan" - *

+ * * @param target * @param specFiles * @param workflows * @param ignoreDisabled */ -public record -TestBenchPlan( - Target target, SpecFiles specFiles, Set workflows, boolean ignoreDisabled) implements JobLifeCycle { +public record TestBenchPlan( + Target target, SpecFiles specFiles, Set workflows, boolean ignoreDisabled) + implements JobLifeCycle { private static final Logger LOGGER = LoggerFactory.getLogger(TestBenchPlan.class); @@ -77,22 +71,19 @@ public static TestBenchPlan create(String targetName, List workflows) { return create(targetName, workflows, true); } - public static TestBenchPlan create(String targetName, List workflows, Boolean ignoreDisabled) { + public static TestBenchPlan create( + String targetName, List workflows, Boolean ignoreDisabled) { var targetConfigs = TargetsSpec.loadAll("testbench/targets/targets.json"); return create(targetConfigs.getTarget(targetName), workflows, ignoreDisabled); } public static TestBenchPlan create( - TargetConfiguration targetConfiguration, List workflows, Boolean ignoreDisabled) { + TargetConfiguration targetConfiguration, List workflows, Boolean ignoreDisabled) { var target = new Target(targetConfiguration); var specFiles = SpecFiles.loadAll( - List.of( - "testbench/assertions", - "testbench/testsuites", - "testbench/workflows" - )); + List.of("testbench/assertions", "testbench/testsuites", "testbench/workflows")); return new TestBenchPlan( target, @@ -114,9 +105,7 @@ public TestPlanNodeTree testNode() { var desc = "TestPlan: %s on %s" - .formatted( - target.configuration().name(), - target.configuration().backend()); + .formatted(target.configuration().name(), target.configuration().backend()); var uriBuilder = TestUri.builder(TestUri.Scheme.DATAAPI) @@ -124,12 +113,15 @@ public TestPlanNodeTree testNode() { var testNodeFactory = new TestNodeFactory(this); - var root = testNodeFactory.testPlanContainer( + var root = + testNodeFactory.testPlanContainer( desc, uriBuilder.build().uri(), selectedWorkflows() - .map(workflow -> workflow.testNode(testNodeFactory, uriBuilder.clone(), ignoreDisabled)) - .toList()); + .map( + workflow -> + workflow.testNode(testNodeFactory, uriBuilder.clone(), ignoreDisabled)) + .toList()); return new TestPlanNodeTree(root, testNodeFactory.testNodeCount()); } @@ -138,6 +130,5 @@ public void updateJobForTarget(Job job) { target.updateJobForTarget(job); } - public record TestPlanNodeTree(DynamicNode root, - int totalNodeCount){} + public record TestPlanNodeTree(DynamicNode root, int totalNodeCount) {} } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/TestPlanFile.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/TestPlanFile.java index 6d1139fcf5..9ba1b37df9 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/testbench/TestPlanFile.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/TestPlanFile.java @@ -1,15 +1,16 @@ package io.stargate.sgv2.jsonapi.testbench; import io.stargate.sgv2.jsonapi.testbench.targets.TargetConfiguration; - import java.nio.file.Path; import java.util.List; /** * Structure of a test plane file stored in a jar resource or external file. These are used with the - * {@link TestBenchByTestPlan} runner to control the target and workflows to run at execution time via a config file. - *

- * For example testbench/testplans/test-plan-astra-vectorize.yaml, testing against and Astra DB: + * {@link TestBenchByTestPlan} runner to control the target and workflows to run at execution time + * via a config file. + * + *

For example testbench/testplans/test-plan-astra-vectorize.yaml, testing against + * and Astra DB: * * 

  * name: Test Plan - Astra - Vectorize Workflows
@@ -25,23 +26,24 @@
  *   - vectorize-shared-workflow
  * ignoreDisabled: true
  *
- *

- *

- * The {@link TestBenchPlan#fromFile(Path)} will run Apache command style substituions using {@link System#getenv(String)} - * as the source for replacements. This allows some sensitive information to be put into the env rather than a - * file, and for the runner to make multiple calls with different env vars. - *

+ * + *

The {@link TestBenchPlan#fromFile(Path)} will run Apache command style substituions using + * {@link System#getenv(String)} as the source for replacements. This allows some sensitive + * information to be put into the env rather than a file, and for the runner to make multiple calls + * with different env vars. + * * @param name Nice human name for the test plan, only used in this file * @param targetName Name of the target, such as the db name, used in logging etc. - * @param customTarget Defines a {@link TargetConfiguration} of how to connect (e.g. astra or cassandra backend ) - * and the connection information. - * @param workflows List of the workflows to run, leave empty or null to run all workflows in the system. - * @param ignoreDisabled If true, work flow jobs marked as "disabled" will be executed. Default is false. + * @param customTarget Defines a {@link TargetConfiguration} of how to connect (e.g. astra or + * cassandra backend ) and the connection information. + * @param workflows List of the workflows to run, leave empty or null to run all workflows in the + * system. + * @param ignoreDisabled If true, work flow jobs marked as "disabled" will be executed. Default is + * false. */ public record TestPlanFile( - String name, - String targetName, - TargetConfiguration customTarget, - List workflows, - Boolean ignoreDisabled) { -} + String name, + String targetName, + TargetConfiguration customTarget, + List workflows, + Boolean ignoreDisabled) {} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/AssertionFactory.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/AssertionFactory.java index 8d6ed04b6f..dc1932db3a 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/AssertionFactory.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/AssertionFactory.java @@ -8,29 +8,81 @@ import java.lang.reflect.Modifier; import java.util.List; -/** */ +/** + * Describes functions that can be used to create instances of an assertion, and finds the factory + */ public sealed interface AssertionFactory { - public static final AssertionFactoryRegistry REGISTRY = new AssertionFactoryRegistry(); - + /** + * Registry of the factories that can be called to create assertions, see {@link + * AssertionFactoryRegistry} + */ + AssertionFactoryRegistry REGISTRY = new AssertionFactoryRegistry(); + + /** + * A factory that returns a single AssertionMatcher, this is a single match of the response. + * + *

This is what we use with basic assertions like: + * + * 

+   *     {
+   *        "Status.isExactly" : {
+   *           "matchedCount": 1,
+   *           "modifiedCount": 1
+   *         }
+   *     }
+   *
+ */ @FunctionalInterface non-sealed interface AssertionMatcherFactory extends AssertionFactory { + /** + * Create an assertion matcher that can be used to match the response. + * + * @param testCommand The command the assertion will be run against. + * @param args The arguments defined in the test suite, e.g. the number of documents in a + * collection. + * @return AssertionMatcher that can be used to match the response. + */ AssertionMatcher create(TestCommand testCommand, JsonNode args); } + /** + * A factory that returns a list of assertions, this is a list of matches of the response. Used + * with templated assertions. + * + *

This returns the {@link TestAssertion} which is higher up the stack than the {@link + * AssertionMatcher} because a templated assertion is a list of assertions and only the template + * factory knows how to describe them because it makes them. + */ @FunctionalInterface non-sealed interface TemplatedAssertionFactory extends AssertionFactory { + + /** + * Create a list of assertions that can be used to match the response. + * + * @param testPlan The test plan that is being created holds context of what we are doing. + * @param template JSON pulled from the {@link + * io.stargate.sgv2.jsonapi.testbench.testspec.TestSpecKind#ASSERTION_TEMPLATE} that matched + * the assertion name. + * @param testCommand The command the assertion will be run against. + * @param args The arguments defined in the test suite, e.g. the number of documents in a + * collection. + * @return + */ List create( - TestBenchPlan testPlan, JsonNode template, TestCommand testCommand, JsonNode args); + TestBenchPlan testPlan, JsonNode template, TestCommand testCommand, JsonNode args); } + /** Returns true if the function matches *either* of the assertion factory functions. */ static boolean isValidFactoryMethod(Method method) { + // must be static if (!Modifier.isStatic(method.getModifiers())) { return false; } - // NOTE: not checked the type of the list, lazy + // Checking for TemplatedAssertionFactory + // NOTE: not checked the generic type of the list, lazy and this is a contrained world if (List.class.isAssignableFrom(method.getReturnType())) { var p = method.getParameterTypes(); return p.length == 4 @@ -40,6 +92,7 @@ static boolean isValidFactoryMethod(Method method) { && p[3] == JsonNode.class; } + // Checking for AssertionMatcherFactory if (method.getReturnType() == AssertionMatcher.class) { var p = method.getParameterTypes(); return p.length == 2 && p[0] == TestCommand.class && p[1] == JsonNode.class; @@ -47,6 +100,23 @@ static boolean isValidFactoryMethod(Method method) { return false; } + /** + * There are two situations we handle the factory methods with: first when registering them so we + * know what we can call, second when processing a test suite config we need to call them to get + * the assertion. + * + *

For the first part, the factory methods are defined as static functions on a class, and + * there can be two different types of factory. This class wraps the raw factory, so we have a + * common class we can use when registering all the factories we know. It also constructs a {@link + * AssertionName} for the java function that can be used to match against the name used in the + * test suite config. + * + *

For the second path the subclasses provide an adapter functionality: they implement the + * factory interface so we get strong type checking for calling, and then transform that call into + * an untyped call to the raw factory method via {@link Method#invoke(Object, Object...)}. + * + *

NOTE: Construct instances using {@link WrappedMethod#of(Class, Method)} + */ abstract sealed class WrappedMethod permits WrappedAssertionMatcherFactory, WrappedTemplatedAssertionFactory { @@ -61,23 +131,22 @@ protected WrappedMethod(Class clazz, Method method) { } static WrappedMethod of(Class clazz, Method method) { + + // a little sloppy, but we already know this should be a factory method return (method.getReturnType() == AssertionMatcher.class) ? new WrappedAssertionMatcherFactory(clazz, method) : new WrappedTemplatedAssertionFactory(clazz, method); } - public Class clazz() { - return clazz; - } - - public Method method() { - return method; - } - + /** Gets the name of the factory function, without the class name. */ public String properName() { return AssertionName.properName(method); } + /** + * The identifier for the factory function that can be used to match agains the name used in the + * test suite config. e.g. "Documents.count" + */ public AssertionName assertionName() { return assertionName; } @@ -85,6 +154,7 @@ public AssertionName assertionName() { @SuppressWarnings("unchecked") protected T invoke(Object... args) { try { + // pass null for the object reference, all factories are static return (T) method.invoke(null, args); } catch (IllegalAccessException | InvocationTargetException e) { throw new RuntimeException(e); @@ -92,6 +162,10 @@ protected T invoke(Object... args) { } } + /** + * Adapter class for the {@link AssertionMatcherFactory}, translates strong typed called to the + * factory method into untypes method invocation. + */ final class WrappedAssertionMatcherFactory extends WrappedMethod implements AssertionMatcherFactory { @@ -105,6 +179,10 @@ public AssertionMatcher create(TestCommand testCommand, JsonNode args) { } } + /** + * Adapter class for the {@link TemplatedAssertionFactory}, translates strong typed called to the + * factory method into untypes method invocation. + */ final class WrappedTemplatedAssertionFactory extends WrappedMethod implements TemplatedAssertionFactory { @@ -114,7 +192,7 @@ final class WrappedTemplatedAssertionFactory extends WrappedMethod @Override public List create( - TestBenchPlan testPlan, JsonNode template, TestCommand testCommand, JsonNode args) { + TestBenchPlan testPlan, JsonNode template, TestCommand testCommand, JsonNode args) { return invoke(testPlan, template, testCommand, args); } } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/AssertionFactoryRegistry.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/AssertionFactoryRegistry.java index 045caec2c2..5b8620c122 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/AssertionFactoryRegistry.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/AssertionFactoryRegistry.java @@ -3,12 +3,37 @@ import java.util.Map; import java.util.concurrent.ConcurrentHashMap; +/** + * Registry of assertion factories that is built by the static constructors of the assertion + * classes registering themselves. + * + *

Every assertion class must register itself with the registry, so we know what factories exist + * and can match them at test execution time. Do that in a static constructor as below, the regsitry + * will walk the class and registry functions that match either of the two {@link AssertionFactory} + * interfaces. + * + * 

+ * public class Documents {
+ *
+ *   static {
+ *     AssertionFactory.REGISTRY.register(Documents.class);
+ *   }
+ *
+ * + *

Use the singleton instance at {@link AssertionFactory#REGISTRY} to access the registry. + */ public class AssertionFactoryRegistry { + // Map of the factory functions and the name they should be matched to in test suite config private final Map factoryMethods = new ConcurrentHashMap<>(); + /** + * Registers all the static factory functions on the class that match the signature of either + * {@link AssertionFactory} interface. + */ public void register(Class cls) { + for (var method : cls.getMethods()) { if (AssertionFactory.isValidFactoryMethod(method)) { var wrapped = AssertionFactory.WrappedMethod.of(cls, method); @@ -17,19 +42,30 @@ public void register(Class cls) { } } - public AssertionFactory.WrappedMethod getWrapped(String fullKey) { - var normalisedName = AssertionName.from(fullKey); + /** + * Gets the factory function for the assertion with the given name. + * + * @param rawAssertionName the string name of the assertion from the config, e.g. + * "Documents.count" + * @return A wrapper for the factory function that can be used to create an assertion instance. + */ + public AssertionFactory.WrappedMethod getWrappedAssertionFactory(String rawAssertionName) { + + // need to take the raw name from the config into the internal representation that + // matches to what we used in the registry + var assertionName = AssertionName.from(rawAssertionName); - var factoryMethod = factoryMethods.get(normalisedName); + var factoryMethod = factoryMethods.get(assertionName); if (factoryMethod == null) { - loadClassFor(normalisedName); + // sometimes the class is not loaded yet, so let's just give nature a helping hand. Shhh + loadClassFor(assertionName); } - factoryMethod = factoryMethods.get(normalisedName); + factoryMethod = factoryMethods.get(assertionName); if (factoryMethod == null) { throw new IllegalArgumentException( - "Unknown assertion factory. (normalised)name: %s known=%s" - .formatted(normalisedName, factoryMethods.keySet())); + "Unknown assertion factory. (parsed) assertionName: %s factoryMethods.keySet:%s" + .formatted(assertionName, factoryMethods.keySet())); } return factoryMethod; } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/AssertionMatcher.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/AssertionMatcher.java index 38e616b98f..09e738b478 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/AssertionMatcher.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/AssertionMatcher.java @@ -2,7 +2,11 @@ import io.stargate.sgv2.jsonapi.testbench.messaging.APIResponse; -/** Contract for running an assertion on the response from the API. */ +/** + * Contract for matching the result of an API call to an assertion. + * + *

This is the raw function to do the work, without any descriptive elements around it. + */ @FunctionalInterface public interface AssertionMatcher { diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/AssertionName.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/AssertionName.java index 5a4ccdd055..883db5ed08 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/AssertionName.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/AssertionName.java @@ -2,15 +2,31 @@ import java.lang.reflect.Method; +/** + * A normalized way to represent an assertion name, used in both directions: used when registering + * assertion factories, so we map from the Java method name to this; and then to parse the assertion + * name from the test spec the user provided. + * + *

Once in this common middle ground, we can map between test spec and java method names. + * NOTE: the matching is case-insensitive, and we normalize to lower case. So "isSuccess" and + * "issuccess" are the same. + * + * @param typeName Class name of the method, or the first part of the assertion config e.g. + * "Documents" + * @param funcName The name of the method, or the second part of the assertion config e.g. "count" + */ public record AssertionName(String typeName, String funcName) { + // A bit of a hack, buit this is used when we force the static load. see below. private static final String PACKAGE = "io.stargate.sgv2.jsonapi.testbench.assertions"; public AssertionName { + // we normalize to match on case-insensitive typeName = typeName.toLowerCase(); funcName = funcName.toLowerCase(); } + /** Create from the string name provided in the test spec config, e.g. "Documents.count" */ public static AssertionName from(String fullKey) { int pos = fullKey.indexOf('.'); @@ -22,10 +38,7 @@ public static AssertionName from(String fullKey) { return new AssertionName(type, func); } - public static String properName(Class clazz, Method method) { - return clazz.getSimpleName() + '.' + method.getName(); - } - + /** Gets the name of the factory function, without the class name. */ public static String properName(Method method) { return method.getName(); } @@ -33,8 +46,4 @@ public static String properName(Method method) { public String properClassName() { return PACKAGE + "." + Character.toUpperCase(typeName.charAt(0)) + typeName.substring(1); } - - public String normalisedKey() { - return typeName + "." + funcName; - } } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/BodyAssertion.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/BodyAssertion.java index 643bf2130b..07e51fbcaf 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/BodyAssertion.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/BodyAssertion.java @@ -5,13 +5,22 @@ import org.hamcrest.StringDescription; /** - * Assertions that check the body of the response using a {@link Matcher} form hamcrest. + * Assertions that check the body of the response using a {@link Matcher} from hamcrest. * *

Example: * * 

  *   return new BodyAssertion("data.documents", hasSize(expectedCount));
  *
+ * + *

So this is a very re-usable record, as most assertions we want to create run a matcher against + * the body of the response by calling {@link + * io.restassured.response.ValidatableResponse#body(String, Matcher)}. This record makes it easy to + * make a matcher from hamcrest, work out the description, and plug it into the {@link + * TestAssertion} structure so we can run it in the dynamic tests we build + * + * @param bodyPath Path to the body to check, e.g. "data.documents" + * @param matcher Matcher to use to check the body, e.g. hasSize(expectedCount) */ public record BodyAssertion(String bodyPath, Matcher matcher) implements Describable, AssertionMatcher { @@ -21,12 +30,15 @@ public void match(APIResponse apiResponse) { apiResponse.validatable().body(bodyPath(), matcher()); } + /** Describes the assertion, based on the path and the matcher. */ @Override public String describe() { var describable = new StringDescription(); + // get hamcrest to describe the matcher it will run matcher.describeTo(describable); - // called should truncate if it wants to limit it + // caller should truncate if it wants to limit it + // example: "body('data.documents') - a collection with size <3>" return "body('%s') - %s".formatted(bodyPath(), describable.toString()); } } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/Describable.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/Describable.java index afce9b7061..68f1730582 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/Describable.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/Describable.java @@ -1,5 +1,9 @@ package io.stargate.sgv2.jsonapi.testbench.assertions; +/** + * Functional interface to call so an assertion can describe itself outside of its string + * representation. Typically based on how the assertion was described in the test configuration. + */ @FunctionalInterface public interface Describable { diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/DescribableAssertionMatcher.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/DescribableAssertionMatcher.java index c8b1ba2f66..38025cf47f 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/DescribableAssertionMatcher.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/DescribableAssertionMatcher.java @@ -3,6 +3,10 @@ import io.stargate.sgv2.jsonapi.testbench.messaging.APIResponse; import org.jspecify.annotations.NonNull; +/** + * Hold an assertion matcher and a description for it, used with very simple assertions that are + * just a function. + */ public record DescribableAssertionMatcher(String description, AssertionMatcher matcher) implements Describable, AssertionMatcher { diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/Documents.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/Documents.java index 758a960e0b..6050bcf350 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/Documents.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/Documents.java @@ -17,11 +17,21 @@ public class Documents { AssertionFactory.REGISTRY.register(Documents.class); } + /** + * Check the count of the `data.documents` array. + * + *

Assertion factory, see {@link AssertionFactory.AssertionMatcherFactory} + */ public static AssertionMatcher count(TestCommand testCommand, JsonNode args) { var expectedCount = args.asInt(); return new BodyAssertion("data.documents", hasSize(expectedCount)); } + /** + * Check that the `data.document` matches the given JSON. + * + *

Assertion factory, see {@link AssertionFactory.AssertionMatcherFactory} + */ public static AssertionMatcher isExactly(TestCommand testCommand, JsonNode args) { return new BodyAssertion("data.document", jsonEquals(args)); } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/Http.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/Http.java index fc1d237e07..2d99e7be3c 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/Http.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/Http.java @@ -6,12 +6,14 @@ import io.stargate.sgv2.jsonapi.testbench.testspec.TestCommand; import org.apache.http.HttpStatus; +/** Assertions that check the structure of the HTTP Response code */ public class Http { static { AssertionFactory.REGISTRY.register(Http.class); } + /** Assertion factory, see {@link AssertionFactory.AssertionMatcherFactory} */ public static AssertionMatcher success(TestCommand testCommand, JsonNode args) { return described( "http status is " + HttpStatus.SC_OK, diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/Response.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/Response.java index d296b12a45..b2b4ef4db4 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/Response.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/Response.java @@ -5,18 +5,18 @@ import com.fasterxml.jackson.databind.JsonNode; import io.stargate.sgv2.jsonapi.testbench.testspec.TestCommand; -/** - * Assertions that check the structure of the API response, e.g. should it have a `data` field - * - *

See {@link TestAssertion} - */ +/** Assertions that check the structure of the API response, e.g. should it have a `data` field */ public class Response { static { AssertionFactory.REGISTRY.register(Response.class); } - /** Checks the hTTP status AND the shape of the response doc */ + /** + * Checks the HTTP status AND the shape of the response doc + * + *

Assertion factory, see {@link AssertionFactory.AssertionMatcherFactory} + */ public static AssertionMatcher isSuccess(TestCommand testCommand, JsonNode args) { var commandName = testCommand.commandName(); @@ -24,9 +24,9 @@ public static AssertionMatcher isSuccess(TestCommand testCommand, JsonNode args) case DDL -> isDDLSuccess(testCommand, args); case DML -> switch (commandName) { - case FIND_ONE, FIND -> Response.isFindSuccess(testCommand, args); + case FIND_ONE, FIND -> isFindSuccess(testCommand, args); case FIND_ONE_AND_DELETE, FIND_ONE_AND_REPLACE, FIND_ONE_AND_UPDATE -> - Response.isFindAndSuccess(testCommand, args); + isFindAndSuccess(testCommand, args); case INSERT_ONE, INSERT_MANY -> Response.isWriteSuccess(testCommand, args); default -> throw new IllegalStateException( diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/SingleTestAssertion.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/SingleTestAssertion.java index 4f12768fd8..da1371ca9d 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/SingleTestAssertion.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/SingleTestAssertion.java @@ -1,31 +1,32 @@ package io.stargate.sgv2.jsonapi.testbench.assertions; import com.fasterxml.jackson.databind.JsonNode; - -import java.util.concurrent.atomic.AtomicReference; - import io.stargate.sgv2.jsonapi.testbench.testrun.*; +import java.util.concurrent.atomic.AtomicReference; import org.junit.jupiter.api.DynamicNode; +/** + * A single test assertion that performs a single test of the results of a request. + * + * @param name Name of the assertion, e.g. "Documents.count" + * @param args Raw arguments passed into the assertion factory, from the test case definition + * @param matcher The actual logic that will be run to assert the result of the request. + */ public record SingleTestAssertion(String name, JsonNode args, AssertionMatcher matcher) implements TestAssertion { + @Override public void run(TestRunResponse testResponse) { - - try { - matcher.match(testResponse.apiResponse()); - } catch (AssertionError e) { -// System.out.printf("Failed Assertion: name=%s, args=%s", name, args); - throw e; - } catch (Exception e) { -// System.out.printf("Error In Assertion: name=%s, args=%s", name, args); - throw e; - } + // exceptions can bubble out, that's how the frameworks know the assertion result. + matcher.match(testResponse.apiResponse()); } @Override public DynamicNode testNodes( - TestNodeFactory testNodeFactory, TestUri.Builder uriBuilder, AtomicReference testResponse, TestExecutionCondition testExecutionCondition) { + TestNodeFactory testNodeFactory, + TestUri.Builder uriBuilder, + AtomicReference testResponse, + TestExecutionCondition testExecutionCondition) { var matcherDesc = (matcher instanceof Describable d) ? d.describe() : ""; diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/Status.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/Status.java index 5b1d24299c..8f1f3364fb 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/Status.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/Status.java @@ -5,12 +5,14 @@ import com.fasterxml.jackson.databind.JsonNode; import io.stargate.sgv2.jsonapi.testbench.testspec.TestCommand; +/** Assertions that match Data API status top level response field. */ public class Status { static { AssertionFactory.REGISTRY.register(Status.class); } + /** Assertion factory, see {@link AssertionFactory.AssertionMatcherFactory} */ public static AssertionMatcher isExactly(TestCommand testCommand, JsonNode args) { return new BodyAssertion("status", jsonEquals(args)); } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/Templated.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/Templated.java index b4b0dde278..e80700e1b6 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/Templated.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/Templated.java @@ -6,14 +6,53 @@ import io.stargate.sgv2.jsonapi.testbench.testspec.TestCommand; import java.util.List; +/** + * Assertions that are defined by a JSON template. + * + *

There are two parts to using a template, first the template must be defined in a {@link + * io.stargate.sgv2.jsonapi.testbench.testspec.TestSpecKind#ASSERTION_TEMPLATE} such as + * + * 

+ * {
+ *   "meta": {
+ *     "name": "assertions-templates",
+ *     "kind": "assertion_template"
+ *   },
+ *   "templates": {
+ *     "isSuccess": {
+ *       "createCollection": {
+ *         "http.success": null,
+ *         "response.isDDLSuccess": null
+ *       },
+ *       "createKeyspace": {
+ *         "http.success": null,
+ *         "response.isDDLSuccess": null
+ *       }
+ *   }
+ * }
+ *
+ * + * NOTE: The format of a template is fixed: the members under "templates" as the names of the + * template, the value is a JSON object that is passed to its factory below. Each template can then + * have its own style of definition. + * + *

Strongly encouraged to use the same structure as isSucces: whose members are the names of API + * commands, and those values are the assertions that should be run as you would define them + * normally. We use this structure because the idea of a template like "isSuccess" is that it is + * saying "whatever the API command we just ran it should be successful". + * + *

So, every template name from a config file like above must have a function here to create + */ public class Templated { static { AssertionFactory.REGISTRY.register(Templated.class); } + /** Assertion factory, see {@link AssertionFactory.TemplatedAssertionFactory} */ public static List isSuccess( - TestBenchPlan testPlan, JsonNode template, TestCommand testCommand, JsonNode args) { + TestBenchPlan testPlan, JsonNode template, TestCommand testCommand, JsonNode args) { + var commandTemplate = template.get(testCommand.commandName().getApiName()); if (commandTemplate == null) { throw new IllegalArgumentException( @@ -24,7 +63,8 @@ public static List isSuccess( } private static List runTemplate( - TestBenchPlan testPlan, ObjectNode template, TestCommand testCommand, JsonNode args) { + TestBenchPlan testPlan, ObjectNode template, TestCommand testCommand, JsonNode args) { + return template.properties().stream() .map(entry -> new TestAssertion.AssertionDefinition(entry.getKey(), entry.getValue())) .map(def -> TestAssertion.buildAssertion(testPlan, testCommand, def)) diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/TestAssertion.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/TestAssertion.java index 61f1610583..7ed0fac282 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/TestAssertion.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/TestAssertion.java @@ -2,10 +2,7 @@ import com.fasterxml.jackson.databind.JsonNode; import io.stargate.sgv2.jsonapi.testbench.TestBenchPlan; -import io.stargate.sgv2.jsonapi.testbench.testrun.TestExecutionCondition; -import io.stargate.sgv2.jsonapi.testbench.testrun.TestNodeFactory; -import io.stargate.sgv2.jsonapi.testbench.testrun.TestRunResponse; -import io.stargate.sgv2.jsonapi.testbench.testrun.TestUri; +import io.stargate.sgv2.jsonapi.testbench.testrun.*; import io.stargate.sgv2.jsonapi.testbench.testspec.AssertionTemplateSpec; import io.stargate.sgv2.jsonapi.testbench.testspec.TestCase; import io.stargate.sgv2.jsonapi.testbench.testspec.TestCommand; @@ -15,16 +12,100 @@ import java.util.stream.Stream; import org.junit.jupiter.api.DynamicNode; -public interface TestAssertion { - +/** + * A single assertion to run on the result of a single request sent to the API. This is the + * assertion with name, description etc, and the logic of the matcher that will perform the test. + * See also {@link AssertionMatcher} + * + *

Assertions are defined in code, and then linked to the setup, test, or cleanup request in the + * test-suite. Or in the case of Target lifecycle (such as creating a keyspace) created in code + * entirely. + * + *

For example, this TestCase has two assertions. One is templated, that is made up of multiple + * other assertions, and the other is a simple matcher. + * + * 

+ *     {
+ *       "name": "basic findMany",
+ *       "command": {
+ *         "find": {
+ *           "sort": {
+ *             "$vectorize": "I love movies!"
+ *           }
+ *         }
+ *       },
+ *       "asserts": {
+ *         "Templated.isSuccess": null,
+ *         "Documents.count": 3
+ *       }
+ *     }
+ *
+ * + *

NOTE: The name of the assertion, e.g. "Documents.count" MUST to the name of a + * "Class.Method" in the "assertions" package. For example see {@link Documents#count(TestCommand, + * JsonNode)} and the {@link AssertionFactory} for the registry of assertions. The assertion object + * for a particular assertion, that is an instance with the configuration from above, is created in + * {@link AssertionDefWithFactory#build(TestBenchPlan, TestCommand)} + */ +public sealed interface TestAssertion permits SingleTestAssertion, TestAssertionContainer { + + /** Friendly name used in logs and reports. */ String name(); + /** + * Arguments that may be passed to the assertion from the test suite definition. For example, text + * node 3 will be passed to "Documents.count" in the above. Used for reporting / + * logging. + */ JsonNode args(); + /** + * Called for the assertion to run against the response of running the API request. + * + *

Three things can happen: + * + *

    + *
  1. Everything is OK, returns without exception. + *
  2. The assertion fails, throws an exception like a normal Junit test, and it will be + * recorded as a failure. + *
  3. Throw a {@link org.junit.AssumptionViolatedException} to say the assumption was not meet + * and the test is aborted. This is normally done before the assertion is called so we do + * not send requests after one has failed in the test env, see {@link + * io.stargate.sgv2.jsonapi.testbench.testrun.DynamicTestExecutable}. + *
+ * + * @param testResponse + */ void run(TestRunResponse testResponse); - DynamicNode testNodes(TestNodeFactory testNodeFactory, TestUri.Builder uriBuilder, AtomicReference testResponse, TestExecutionCondition testExecutionCondition); - + /** + * Gets {@link DynamicNode} that represents this assertion in the test tree. + * + *

An assertion is always a single test node in the test tree, not a container. See {@link + * SingleTestAssertion} for the implementation for a single assertion. Where we have a group of + * assertions, they end up going to that class as well to get the test nodes for the individual + * assertions. + * + * @param testNodeFactory Factory to build test nodes with, for common naming etc. + * @param uriBuilder Builder for the URI tp descrbie the type of node in the test tree. + * @param testResponse Atomic reference that will be updated with the response of the test + * request, that the assertion will use to perform to rune once it is time to execute. + * @param testExecutionCondition Condition that can be checked to see if the test node should be + * skipped. + * @return DynamicNode representing this assertion in the test tree, may be a container if there + * are multiple assertions. + */ + DynamicNode testNodes( + TestNodeFactory testNodeFactory, + TestUri.Builder uriBuilder, + AtomicReference testResponse, + TestExecutionCondition testExecutionCondition); + + /** + * Returns a list of assertions that can be used to determine if a command was successful. + * + *

Uses the Templated.isSuccess templated assertion. + */ static List forSuccess(TestBenchPlan testPlan, TestCommand testCommand) { var builder = @@ -34,6 +115,10 @@ static List forSuccess(TestBenchPlan testPlan, TestCommand testCo return buildAssertions(testPlan, testCommand, builder.build()); } + /** + * Creates assertions based on the configuration of the {@link TestCase}, that is what is under + * the "asserts" member in the example above. + */ static List buildAssertions(TestBenchPlan testPlan, TestCase testCase) { var defs = testCase.asserts().properties().stream().map(AssertionDefinition::create); @@ -41,17 +126,33 @@ static List buildAssertions(TestBenchPlan testPlan, TestCase test } static List buildAssertions( - TestBenchPlan testPlan, TestCommand testCommand, Stream defs) { + TestBenchPlan testPlan, TestCommand testCommand, Stream defs) { return defs.map(def -> buildAssertion(testPlan, testCommand, def)).toList(); } static TestAssertion buildAssertion( - TestBenchPlan testPlan, TestCommand testCommand, AssertionDefinition def) { + TestBenchPlan testPlan, TestCommand testCommand, AssertionDefinition def) { return def.addFactory(AssertionFactory.REGISTRY).build(testPlan, testCommand); } - /** */ + /** + * Definition of an assertion that can be used to create an instance of the assertion used that + * can be run later. + * + *

Example, the member and it's value in below: + * + * 

+   *     {
+   *         "Documents.count": 3
+   *     }
+   *
+ * + * @param name Name of the assertion, it *must* map to a class and method in the "assertions" + * package. + * @param args Arguments that will be passed to the assertion factory, see {@link + * AssertionFactory} + */ record AssertionDefinition(String name, JsonNode args) { static AssertionDefinition create(Map.Entry def) { @@ -60,7 +161,7 @@ static AssertionDefinition create(Map.Entry def) { AssertionDefWithFactory addFactory(AssertionFactoryRegistry registry) { - var factory = registry.getWrapped(name()); + var factory = registry.getWrappedAssertionFactory(name()); if (factory == null) { throw new IllegalStateException("Unknown Assertion Factory name=" + name()); } @@ -68,29 +169,54 @@ AssertionDefWithFactory addFactory(AssertionFactoryRegistry registry) { } } - /** */ + /** + * Definition of an assertion where we have the factory function that can create it with the + * arguments supplied by the test definition. + */ record AssertionDefWithFactory(AssertionFactory.WrappedMethod method, JsonNode args) { + /** + * Create an assertion instance by calling the appropriate factory function with the arguments + * from the test definition. + * + * @param testPlan the test plan that is being created holds context of what we are doing. + * @param testCommand The actual command that will be executed, that we will want to assert the + * result of + * @return A single assertion that can be run later. + */ TestAssertion build(TestBenchPlan testPlan, TestCommand testCommand) { return switch (method) { + // basic single assertion case AssertionFactory.WrappedAssertionMatcherFactory factory -> new SingleTestAssertion( method.properName(), args(), factory.create(testCommand, args())); + // templated, we need to look up what assertions should be in it. case AssertionFactory.TemplatedAssertionFactory factory -> { + // search all assertion templates to find any that have the name given in the test + // definition, there + // must be one and only one var template = testPlan .specFiles() .byType(AssertionTemplateSpec.class) .flatMap( assertTemplate -> assertTemplate.templateFor(method.properName()).stream()) - .findFirst() + .reduce( + (a, b) -> { + throw new IllegalStateException( + "Multiple Assertion Templates found for name=" + method.properName()); + }) .orElseThrow( () -> new IllegalStateException( "Unknown Assertion Template name=" + method.properName())); + // templated assertions have children, so we need an assertion that contains those + // children. + // the factory we call here will use the template to make AssertionDefinition 's and end + // up back in this method to make the childred yield new TestAssertionContainer( method.properName(), args(), factory.create(testPlan, template, testCommand, args())); } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/TestAssertionContainer.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/TestAssertionContainer.java index 42ffd13920..cb29c46ab4 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/TestAssertionContainer.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/TestAssertionContainer.java @@ -1,7 +1,5 @@ package io.stargate.sgv2.jsonapi.testbench.assertions; - - import com.fasterxml.jackson.databind.JsonNode; import io.stargate.sgv2.jsonapi.testbench.testrun.TestExecutionCondition; import io.stargate.sgv2.jsonapi.testbench.testrun.TestNodeFactory; @@ -11,38 +9,41 @@ import java.util.concurrent.atomic.AtomicReference; import org.junit.jupiter.api.DynamicNode; +/** + * An assertion made up of child assertions, for example built from a template that created multiple + * assertions + * + * @param name Name of the template, e.g. "Templated.isSuccess" + * @param args Raw arguments passed into the template factory, from the test case definition + * @param assertions Child assertions that were created by the template factory + */ public record TestAssertionContainer(String name, JsonNode args, List assertions) implements TestAssertion { @Override public void run(TestRunResponse testResponse) { - for (TestAssertion assertion : assertions) { - try { - assertion.run(testResponse); - } catch (AssertionError e) { - System.out.printf( - "Failed Assertion Container: name=%s, args=%s\n\t Caused By: name=%s, args=%s", - name, args, assertion.name(), assertion.args()); - throw e; - } catch (Exception e) { - System.out.printf( - "Error In Assertion Container: name=%s, args=%s\n\t Caused By: name=%s, args=%s", - name, args, assertion.name(), assertion.args()); - throw e; - } - } + + // we are just a container, let those exceptions bubble up + assertions.forEach(assertion -> assertion.run(testResponse)); } @Override public DynamicNode testNodes( - TestNodeFactory testNodeFactory, TestUri.Builder uriBuilder, AtomicReference testResponse, TestExecutionCondition testExecutionCondition) { + TestNodeFactory testNodeFactory, + TestUri.Builder uriBuilder, + AtomicReference testResponse, + TestExecutionCondition testExecutionCondition) { uriBuilder.addSegment(TestUri.Segment.ASSERTION, name()); - var childs = + + var children = assertions.stream() - .map(assertion -> assertion.testNodes(testNodeFactory, uriBuilder.clone(), testResponse, testExecutionCondition)) - .toList(); + .map( + assertion -> + assertion.testNodes( + testNodeFactory, uriBuilder.clone(), testResponse, testExecutionCondition)) + .toList(); - return testNodeFactory.testPlanContainer(name, uriBuilder.build().uri(), childs); + return testNodeFactory.testPlanContainer(name, uriBuilder.build().uri(), children); } } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/lifecycle/JobLifeCycle.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/lifecycle/JobLifeCycle.java index b1f87261f2..e2bd83a8f1 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/testbench/lifecycle/JobLifeCycle.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/lifecycle/JobLifeCycle.java @@ -4,25 +4,21 @@ /** * Defines the stages a {link @Job} may go through as it is executing. - *

- * It's limited now to just the one method, may never get bigger, but there are a couple of + * + *

It's limited now to just the one method, may never get bigger, but there are a couple of * implementations so extracted. - *

*/ public interface JobLifeCycle { - /** - * Called to update the Job with any information it needs to run against the - * particular target. - *

- * Typcially this means setting or updating the {@link Job#variables()} for things like a default keyspace - * name or implementing naming restrictions. - *

- *

- * NOTE: The target or Backend must set the KEYSPACE_NAME job variable - *

- * @param job The job to update - */ - void updateJobForTarget(Job job); - + /** + * Called to update the Job with any information it needs to run against the particular target. + * + *

Typcially this means setting or updating the {@link Job#variables()} for things like a + * default keyspace name or implementing naming restrictions. + * + *

NOTE: The target or Backend must set the KEYSPACE_NAME job variable + * + * @param job The job to update + */ + void updateJobForTarget(Job job); } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/lifecycle/TestPlanLifecycle.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/lifecycle/TestPlanLifecycle.java index 1cb35c1ca6..77c67c6aac 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/testbench/lifecycle/TestPlanLifecycle.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/lifecycle/TestPlanLifecycle.java @@ -11,23 +11,21 @@ /** * Defines the stages in the lifecycle of a test bench run. - *

- * Design to be implemented by a {@link io.stargate.sgv2.jsonapi.testbench.targets.Backend} so that it can - * make changes to the data environement so tests can run in a common environement. For example, - * when we use Cassandra as a backend we need to create a keyspace but for Astra we use the default one. - *

- *

- * There should not be any test logic within the implementations, that should all be in the test defintoions. - *

- *

- * Extracted to an interface so it can be reused, such as at the higher level - * {@link io.stargate.sgv2.jsonapi.testbench.targets.Target} which includes a backend. - *

- *

- * All methods allow the implementation to return a JUNIT {@link DynamicNode} which will be inserted before or after - * the dynamic nodes at that level. The returned node could be a single {@link org.junit.jupiter.api.DynamicTest} or - * a {@link org.junit.jupiter.api.DynamicContainer}. - *

+ * + *

Design to be implemented by a {@link io.stargate.sgv2.jsonapi.testbench.targets.Backend} so + * that it can make changes to the data environement so tests can run in a common environement. For + * example, when we use Cassandra as a backend we need to create a keyspace but for Astra we use the + * default one. + * + *

There should not be any test logic within the implementations, that should all be in the test + * defintoions. + * + *

Extracted to an interface so it can be reused, such as at the higher level {@link + * io.stargate.sgv2.jsonapi.testbench.targets.Target} which includes a backend. + * + *

All methods allow the implementation to return a JUNIT {@link DynamicNode} which will be + * inserted before or after the dynamic nodes at that level. The returned node could be a single + * {@link org.junit.jupiter.api.DynamicTest} or a {@link org.junit.jupiter.api.DynamicContainer}. */ public interface TestPlanLifecycle { @@ -40,7 +38,7 @@ public interface TestPlanLifecycle { * @return Optional {@link DynamicNode} to run before the workflow. */ default Optional beforeWorkflow( - TestNodeFactory testNodeFactory, TestUri.Builder uriBuilder, WorkflowSpec workflow) { + TestNodeFactory testNodeFactory, TestUri.Builder uriBuilder, WorkflowSpec workflow) { return Optional.empty(); } @@ -53,7 +51,7 @@ default Optional beforeWorkflow( * @return Optional {@link DynamicNode} to run after the workflow. */ default Optional afterWorkflow( - TestNodeFactory testNodeFactory, TestUri.Builder uriBuilder, WorkflowSpec workflow) { + TestNodeFactory testNodeFactory, TestUri.Builder uriBuilder, WorkflowSpec workflow) { return Optional.empty(); } @@ -65,7 +63,8 @@ default Optional afterWorkflow( * @param job The job about to execute. * @return Optional {@link DynamicNode} to run before the job. */ - default Optional beforeJob(TestNodeFactory testNodeFactory, TestUri.Builder uriBuilder, Job job) { + default Optional beforeJob( + TestNodeFactory testNodeFactory, TestUri.Builder uriBuilder, Job job) { return Optional.empty(); } @@ -77,7 +76,8 @@ default Optional beforeJob(TestNodeFactory testNodeFactory, TestUri * @param job The job that just completed. * @return Optional {@link DynamicNode} to run after the job. */ - default Optional afterJob(TestNodeFactory testNodeFactory, TestUri.Builder uriBuilder, Job job) { + default Optional afterJob( + TestNodeFactory testNodeFactory, TestUri.Builder uriBuilder, Job job) { return Optional.empty(); } @@ -87,11 +87,15 @@ default Optional afterJob(TestNodeFactory testNodeFactory, TestUri. * @param testNodeFactory Factory to use to create test nodes, see {@link TestNodeFactory} * @param uriBuilder Builder to use to create the URI's added to dynamic nodes. * @param test The test suite about to execute. - * @param env The runtime environment for this test, including resolved variables and configuration. + * @param env The runtime environment for this test, including resolved variables and + * configuration. * @return Optional {@link DynamicNode} to run before the test suite. */ default Optional beforeTestSuite( - TestNodeFactory testNodeFactory, TestUri.Builder uriBuilder, TestSuiteSpec test, TestRunEnv env) { + TestNodeFactory testNodeFactory, + TestUri.Builder uriBuilder, + TestSuiteSpec test, + TestRunEnv env) { return Optional.empty(); } @@ -101,11 +105,15 @@ default Optional beforeTestSuite( * @param testNodeFactory Factory to use to create test nodes, see {@link TestNodeFactory} * @param uriBuilder Builder to use to create the URI's added to dynamic nodes. * @param test The test suite that just completed. - * @param env The runtime environment for this test, including resolved variables and configuration. + * @param env The runtime environment for this test, including resolved variables and + * configuration. * @return Optional {@link DynamicNode} to run after the test suite. */ default Optional afterTestSuite( - TestNodeFactory testNodeFactory, TestUri.Builder uriBuilder, TestSuiteSpec test, TestRunEnv env) { + TestNodeFactory testNodeFactory, + TestUri.Builder uriBuilder, + TestSuiteSpec test, + TestRunEnv env) { return Optional.empty(); } } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/messaging/APIRequest.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/messaging/APIRequest.java index 1977b1064a..eb175c7a5e 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/testbench/messaging/APIRequest.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/messaging/APIRequest.java @@ -11,23 +11,21 @@ import io.restassured.response.ValidatableResponse; import io.restassured.specification.RequestSpecification; import io.stargate.sgv2.jsonapi.api.model.command.CommandTarget; +import io.stargate.sgv2.jsonapi.config.constants.HttpConstants; import io.stargate.sgv2.jsonapi.testbench.targets.ConnectionConfiguration; import io.stargate.sgv2.jsonapi.testbench.testrun.TestRunEnv; import io.stargate.sgv2.jsonapi.testbench.testspec.TestCommand; -import io.stargate.sgv2.jsonapi.config.constants.HttpConstants; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.util.HashMap; import java.util.Map; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * The lowest level to represent a request sent to the API, that is only concerned with the * mechanics of sending the request. - *

- * Handles retries based on detecting substrings in the response body, these occur before - * {@link #execute()} returns. - *

+ * + *

Handles retries based on detecting substrings in the response body, these occur before {@link + * #execute()} returns. */ public class APIRequest { private static final Logger LOGGER = LoggerFactory.getLogger(APIRequest.class); @@ -48,10 +46,12 @@ public class APIRequest { /** * Initializes a new instance of the class. + * * @param connection Connection info for the API instance to use. - * @param testRunEnv Environment for this test run, used to find schema names to use in the URL path. - * @param request the complete API request to send, NOTE: any substitutions into the body of the - * request must also be done. + * @param testRunEnv Environment for this test run, used to find schema names to use in the URL + * path. + * @param request the complete API request to send, NOTE: any substitutions into the body + * of the request must also be done. */ public APIRequest(ConnectionConfiguration connection, TestRunEnv testRunEnv, ObjectNode request) { @@ -63,9 +63,9 @@ public APIRequest(ConnectionConfiguration connection, TestRunEnv testRunEnv, Obj /** * Executes the request, including any retries. - *

- * No validation of the response is performed, that is left for assertions to handle later. - *

+ * + *

No validation of the response is performed, that is left for assertions to handle later. + * * @return {@link APIResponse} holding the response of the request. */ public APIResponse execute() { @@ -73,7 +73,7 @@ public APIResponse execute() { ValidatableResponse lastValidatableResponse = null; var retryDecision = retryPolicy.firstAttempt(); - while (retryDecision.retry()){ + while (retryDecision.retry()) { // Create a new request spec, there is some state that is left in it when a request is run // for path params @@ -91,6 +91,7 @@ public APIResponse execute() { /** * Get the request ready to send our request. + * * @return */ private RequestSpecification requestSpec() { @@ -101,37 +102,33 @@ private RequestSpecification requestSpec() { } catch (JsonProcessingException e) { throw new RuntimeException(e); } - return requestForTaget() - .headers(getHeaders()) - .body(requestString).when(); + return requestForTaget().headers(getHeaders()).body(requestString).when(); } - /** - * Create a new RequestSpecification for JSON to send to the target - */ + /** Create a new RequestSpecification for JSON to send to the target */ private RequestSpecification requestForTaget() { return given() - .log() - .uri() - .log() - .body() - .baseUri(connection.domain()) - .port(connection.port()) - .basePath(connection.basePath()) - .contentType(ContentType.JSON); + .log() + .uri() + .log() + .body() + .baseUri(connection.domain()) + .port(connection.port()) + .basePath(connection.basePath()) + .contentType(ContentType.JSON); } - /** - * Get the wellknown headers we need to send with the request. - */ + /** Get the wellknown headers we need to send with the request. */ protected Map getHeaders() { var headers = new HashMap(); - headers.put(HttpConstants.AUTHENTICATION_TOKEN_HEADER_NAME, testRunEnv.requiredValue(HttpConstants.AUTHENTICATION_TOKEN_HEADER_NAME)); + headers.put( + HttpConstants.AUTHENTICATION_TOKEN_HEADER_NAME, + testRunEnv.requiredValue(HttpConstants.AUTHENTICATION_TOKEN_HEADER_NAME)); var embeddingApiKey = testRunEnv.get(HttpConstants.EMBEDDING_AUTHENTICATION_TOKEN_HEADER_NAME); - if (!Strings.isNullOrEmpty(embeddingApiKey)){ + if (!Strings.isNullOrEmpty(embeddingApiKey)) { headers.put(HttpConstants.EMBEDDING_AUTHENTICATION_TOKEN_HEADER_NAME, embeddingApiKey); } return headers; @@ -147,11 +144,12 @@ private Response executeRequest(RequestSpecification requestSpec) { || commandName.getTargets().contains(CommandTarget.TABLE)) { rawRresponse = requestSpec.post( - COLLECTION_TABLE_PATH, + COLLECTION_TABLE_PATH, testRunEnv.requiredValue(TestRunEnv.ENV_KEYSPACE_NAME), testRunEnv.requiredValue(TestRunEnv.ENV_COLLECTION_NAME)); } else if (commandName.getTargets().contains(CommandTarget.KEYSPACE)) { - rawRresponse = requestSpec.post(KEYSPACE_PATH, testRunEnv.requiredValue(TestRunEnv.ENV_KEYSPACE_NAME)); + rawRresponse = + requestSpec.post(KEYSPACE_PATH, testRunEnv.requiredValue(TestRunEnv.ENV_KEYSPACE_NAME)); } else if (commandName.getTargets().contains(CommandTarget.DATABASE)) { rawRresponse = requestSpec.post(DB_PATH); } else { @@ -160,5 +158,4 @@ private Response executeRequest(RequestSpecification requestSpec) { return rawRresponse; } - } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/messaging/APIResponse.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/messaging/APIResponse.java index eec21c1cad..3b1a22225f 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/testbench/messaging/APIResponse.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/messaging/APIResponse.java @@ -2,7 +2,5 @@ import io.restassured.response.ValidatableResponse; -/** - * Basic holder for the response, so we can tie it back to the request that created it - */ +/** Basic holder for the response, so we can tie it back to the request that created it */ public record APIResponse(APIRequest apiRequest, ValidatableResponse validatable) {} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/messaging/APIRetryPolicy.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/messaging/APIRetryPolicy.java index d29f68347b..233d93c905 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/testbench/messaging/APIRetryPolicy.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/messaging/APIRetryPolicy.java @@ -2,160 +2,165 @@ import io.restassured.response.Response; import io.stargate.sgv2.jsonapi.testbench.testrun.TestRunEnv; - import java.util.List; import java.util.concurrent.ThreadLocalRandom; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; -/** - *

---

- */ +/** --- */ interface APIRetryPolicy { - Logger LOGGER = LoggerFactory.getLogger(APIRetryPolicy.class); - - String RETRY_MATCH_STRING_DELIM = "\t"; - String ENV_RETRY_MATCH_STRING = "RETRY_MATCH_STRING"; - - String DEFAULT_MAX_ATTEMPTS = "5"; - String ENV_RETRY_MAX_ATTEMPTS = "END_RETRY_MAX_ATTEMPTS"; - - String DEFAULT_BASE_SLEEP_MS = "5000"; - String ENV_RETRY_BASE_SLEEP_MS = "RETRY_BASE_SLEEP_MS"; - - String DEFAULT_JITTER_MS = "2000"; - String ENV_RETRY_JITTER_MS = "RETRY_JITTER_MS"; - - /** - * Retries are driven by the presence of the following keys in the {@link TestRunEnv}. When a retry is - * performed the response for that request is lost, the response of the call to execute will - * be the response from the last attempt. - *
    - *
  • {@link #ENV_RETRY_MATCH_STRING} Enables retry if present and non blank string, treated as a tab - * {@link #RETRY_MATCH_STRING_DELIM} delimtered string. If any of the tokens is present in the response body - * the request is retried. - * Example: "EMBEDDING_PROVIDER_RATE_LIMITED\tEMBEDDING_PROVIDER_TIMEOUT"
    • - *
  • {@link #ENV_RETRY_MAX_ATTEMPTS} total number of attempts to make before returning, - * must be above 1, default is {@link #ENV_RETRY_MAX_ATTEMPTS}
    • - *
  • {@link #ENV_RETRY_BASE_SLEEP_MS} base number of milliseconds between attempts, this is multiplied by - * 2 ^ attempt, so base of 5000 means sleep of 5, 10, 20 seconds
    • - *
  • {@link #ENV_RETRY_JITTER_MS} Upper bound on the random number of milliseconds to add to each attempt.
    • - *
- */ - static APIRetryPolicy createRetryPolicy(TestRunEnv testRunEnv) { - - var retryMatch = testRunEnv.get(ENV_RETRY_MATCH_STRING); - if (retryMatch == null || retryMatch.isBlank()) { - return new NoAPIRetryPolicy(); - } - - var maxAttempts = Integer.parseInt(testRunEnv.get(ENV_RETRY_MAX_ATTEMPTS, DEFAULT_MAX_ATTEMPTS)); - var baseSleepMs = Long.parseLong(testRunEnv.get(ENV_RETRY_BASE_SLEEP_MS, DEFAULT_BASE_SLEEP_MS)); - var jitterMs = Long.parseLong(testRunEnv.get(ENV_RETRY_JITTER_MS, DEFAULT_JITTER_MS)); - - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("createRetryPolicy() - retryMatch={}, maxAttempts={}, baseSleepMs={}, jitterMs={}", retryMatch, maxAttempts, baseSleepMs, jitterMs); - } - - return new ConfiguredAPIRetryPolicy(List.of(retryMatch.split(RETRY_MATCH_STRING_DELIM)), maxAttempts, baseSleepMs, jitterMs); + Logger LOGGER = LoggerFactory.getLogger(APIRetryPolicy.class); + + String RETRY_MATCH_STRING_DELIM = "\t"; + String ENV_RETRY_MATCH_STRING = "RETRY_MATCH_STRING"; + + String DEFAULT_MAX_ATTEMPTS = "5"; + String ENV_RETRY_MAX_ATTEMPTS = "END_RETRY_MAX_ATTEMPTS"; + + String DEFAULT_BASE_SLEEP_MS = "5000"; + String ENV_RETRY_BASE_SLEEP_MS = "RETRY_BASE_SLEEP_MS"; + + String DEFAULT_JITTER_MS = "2000"; + String ENV_RETRY_JITTER_MS = "RETRY_JITTER_MS"; + + /** + * Retries are driven by the presence of the following keys in the {@link TestRunEnv}. When a + * retry is performed the response for that request is lost, the response of the call to + * execute will be the response from the last attempt. + * + *
    + *
  • {@link #ENV_RETRY_MATCH_STRING} Enables retry if present and non blank string, treated as + * a tab {@link #RETRY_MATCH_STRING_DELIM} delimtered string. If any of the tokens is + * present in the response body the request is retried. Example: + * "EMBEDDING_PROVIDER_RATE_LIMITED\tEMBEDDING_PROVIDER_TIMEOUT" + *
  • {@link #ENV_RETRY_MAX_ATTEMPTS} total number of attempts to make before returning, must + * be above 1, default is {@link #ENV_RETRY_MAX_ATTEMPTS} + *
  • {@link #ENV_RETRY_BASE_SLEEP_MS} base number of milliseconds between attempts, this is + * multiplied by 2 ^ attempt, so base of 5000 means sleep of 5, 10, 20 seconds + *
  • {@link #ENV_RETRY_JITTER_MS} Upper bound on the random number of milliseconds to add to + * each attempt. + *
+ */ + static APIRetryPolicy createRetryPolicy(TestRunEnv testRunEnv) { + + var retryMatch = testRunEnv.get(ENV_RETRY_MATCH_STRING); + if (retryMatch == null || retryMatch.isBlank()) { + return new NoAPIRetryPolicy(); } - /** - * Call to get the first decision, this is used to count the number of attempts. - */ - default RetryDecision firstAttempt() { - return RetryDecision.FIRST_ATTEMPT; + var maxAttempts = + Integer.parseInt(testRunEnv.get(ENV_RETRY_MAX_ATTEMPTS, DEFAULT_MAX_ATTEMPTS)); + var baseSleepMs = + Long.parseLong(testRunEnv.get(ENV_RETRY_BASE_SLEEP_MS, DEFAULT_BASE_SLEEP_MS)); + var jitterMs = Long.parseLong(testRunEnv.get(ENV_RETRY_JITTER_MS, DEFAULT_JITTER_MS)); + + if (LOGGER.isDebugEnabled()) { + LOGGER.debug( + "createRetryPolicy() - retryMatch={}, maxAttempts={}, baseSleepMs={}, jitterMs={}", + retryMatch, + maxAttempts, + baseSleepMs, + jitterMs); } - /** - * Call to decide if we should retry or not, implementations will sleep if needed. - * @param lastDecision The last decision made, used to count the number of attempts - * @param response The response from the last attempt - * @return A decision to retry or not, along with the attempt count, use for the next call to this method. - */ - RetryDecision decide(RetryDecision lastDecision, Response response); - - - /** - * A decision to retry or not, along with the attempt count. This is where we count the - * number of attempts. - */ - record RetryDecision(boolean retry, - int attempt){ - - static final RetryDecision FIRST_ATTEMPT = new RetryDecision(true, 1); - - RetryDecision stopAttempts(){ - // do not increase the attempt count, we won't make another. - return new RetryDecision(false, attempt); - } + return new ConfiguredAPIRetryPolicy( + List.of(retryMatch.split(RETRY_MATCH_STRING_DELIM)), maxAttempts, baseSleepMs, jitterMs); + } + + /** Call to get the first decision, this is used to count the number of attempts. */ + default RetryDecision firstAttempt() { + return RetryDecision.FIRST_ATTEMPT; + } + + /** + * Call to decide if we should retry or not, implementations will sleep if needed. + * + * @param lastDecision The last decision made, used to count the number of attempts + * @param response The response from the last attempt + * @return A decision to retry or not, along with the attempt count, use for the next call to this + * method. + */ + RetryDecision decide(RetryDecision lastDecision, Response response); + + /** + * A decision to retry or not, along with the attempt count. This is where we count the number of + * attempts. + */ + record RetryDecision(boolean retry, int attempt) { + + static final RetryDecision FIRST_ATTEMPT = new RetryDecision(true, 1); + + RetryDecision stopAttempts() { + // do not increase the attempt count, we won't make another. + return new RetryDecision(false, attempt); } + } - /** - * A retry policy that never retries, makes a single attempt. - */ - record NoAPIRetryPolicy() implements APIRetryPolicy { - private static final RetryDecision NO_RETRY = new RetryDecision(false, 1); + /** A retry policy that never retries, makes a single attempt. */ + record NoAPIRetryPolicy() implements APIRetryPolicy { + private static final RetryDecision NO_RETRY = new RetryDecision(false, 1); - @Override - public RetryDecision decide(RetryDecision lastDecision, Response response) { - return NO_RETRY; - } + @Override + public RetryDecision decide(RetryDecision lastDecision, Response response) { + return NO_RETRY; + } + } + + /** + * Configurable retry policy with customizable retry conditions, maximum attempts, and backoff + * strategy. + */ + record ConfiguredAPIRetryPolicy( + List retryMatch, int maxAttempts, long baseSleepMs, long jitterMs) + implements APIRetryPolicy { + + public ConfiguredAPIRetryPolicy { + + if (retryMatch == null || retryMatch.isEmpty()) { + throw new IllegalArgumentException("retryMatch is null or empty"); + } + if (maxAttempts < 2) { + throw new IllegalArgumentException( + "maxAttempts must be greater than 1, got: " + maxAttempts); + } } - /** - * Configurable retry policy with customizable retry conditions, maximum attempts, and backoff strategy. - */ - record ConfiguredAPIRetryPolicy( - List retryMatch, - int maxAttempts, - long baseSleepMs, - long jitterMs - ) implements APIRetryPolicy { - - public ConfiguredAPIRetryPolicy { - - if (retryMatch == null || retryMatch.isEmpty()) { - throw new IllegalArgumentException("retryMatch is null or empty"); - } - if (maxAttempts < 2) { - throw new IllegalArgumentException("maxAttempts must be greater than 1, got: " + maxAttempts); - } - } - - @Override - public RetryDecision decide(RetryDecision lastDecision, Response response) { - - if (lastDecision.attempt == maxAttempts) { - return lastDecision.stopAttempts(); - } - - var body = response.body().asString(); - - for (var match : retryMatch) { - if (body.contains(match)) { - // Service has a concurrency limit and retrying runners can collide regardless of jitter. - // Base backoff is long enough to wait out an in-flight request (5s, 10s, 20s...), - // plus a small random offset to avoid re-synchronising after the wait. - long baseMs = (long) (5000 * Math.pow(2, lastDecision.attempt)); - long jitterMs = ThreadLocalRandom.current().nextLong(2000); - long sleepMs = baseMs + jitterMs; - - LOGGER.info("executeRequest() - Retrying, found retry string in response. match={}, sleepMs={} ms, lastDecision.attempt={}", match, sleepMs, lastDecision.attempt); - try { - Thread.sleep(sleepMs); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - throw new RuntimeException("Interrupted during retry sleep", e); - } - - // try again, increase the attempt counter. - return new RetryDecision(true, lastDecision.attempt +1); - } - } - return lastDecision.stopAttempts(); + @Override + public RetryDecision decide(RetryDecision lastDecision, Response response) { + + if (lastDecision.attempt == maxAttempts) { + return lastDecision.stopAttempts(); + } + + var body = response.body().asString(); + + for (var match : retryMatch) { + if (body.contains(match)) { + // Service has a concurrency limit and retrying runners can collide regardless of jitter. + // Base backoff is long enough to wait out an in-flight request (5s, 10s, 20s...), + // plus a small random offset to avoid re-synchronising after the wait. + long baseMs = (long) (5000 * Math.pow(2, lastDecision.attempt)); + long jitterMs = ThreadLocalRandom.current().nextLong(2000); + long sleepMs = baseMs + jitterMs; + + LOGGER.info( + "executeRequest() - Retrying, found retry string in response. match={}, sleepMs={} ms, lastDecision.attempt={}", + match, + sleepMs, + lastDecision.attempt); + try { + Thread.sleep(sleepMs); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException("Interrupted during retry sleep", e); + } + + // try again, increase the attempt counter. + return new RetryDecision(true, lastDecision.attempt + 1); } + } + return lastDecision.stopAttempts(); } + } } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/reporting/DynamicTreeListener.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/reporting/DynamicTreeListener.java index 52cc31db08..7e41260395 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/testbench/reporting/DynamicTreeListener.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/reporting/DynamicTreeListener.java @@ -4,7 +4,6 @@ import io.stargate.sgv2.jsonapi.testbench.testrun.TestUri; import java.util.*; import java.util.concurrent.ConcurrentHashMap; - import org.junit.platform.engine.TestExecutionResult; import org.junit.platform.engine.UniqueId; import org.junit.platform.engine.support.descriptor.UriSource; @@ -13,24 +12,24 @@ import org.junit.platform.launcher.TestPlan; /** - * Listens to the execution of the dynammic tests created by the {@link TestPlan} and logs - * the results using the {@link TestBenchConsoleWriter}. - *

- * The class mus tbe registeded by a text file at - * META-INF/services/org.junit.platform.launcher.TestExecutionListener that contains the - * fully qualitified path to the class. - *

- *

- * Type types of output are generated: + * Listens to the execution of the dynammic tests created by the {@link TestPlan} and logs the + * results using the {@link TestBenchConsoleWriter}. + * + *

The class mus tbe registeded by a text file at + * META-INF/services/org.junit.platform.launcher.TestExecutionListener that contains the + * fully qualitified path to the class. + * + *

Type types of output are generated: + * *

    - *
  • As the tests are running the name of every test node is outputted together with progress, so we can - * see how long there is to go. See {@link TestBenchConsoleWriter#testStarted(int, int, TestReportingTracker)}. - * At this point we do not know how long child nodes will take to process and what their result will be.
    • - *
  • Once complet a summary is outputted that does not include every node to brevity, - * see {@link TestBenchConsoleWriter#allTestsFinished(TestReportingTracker)}. At this point we know - * how long child nodes took to process and their result.
    • + *
  • As the tests are running the name of every test node is outputted together with progress, + * so we can see how long there is to go. See {@link TestBenchConsoleWriter#testStarted(int, + * int, TestReportingTracker)}. At this point we do not know how long child nodes will take to + * process and what their result will be. + *
  • Once complet a summary is outputted that does not include every node to brevity, see {@link + * TestBenchConsoleWriter#allTestsFinished(TestReportingTracker)}. At this point we know how + * long child nodes took to process and their result. *
- *

*/ public class DynamicTreeListener implements TestExecutionListener { @@ -60,14 +59,16 @@ public void executionStarted(TestIdentifier id) { return; } - // Test count will not be in the system properties until we see the first dymamic test node we create, e.g. + // Test count will not be in the system properties until we see the first dymamic test node we + // create, e.g. // "TestPlan: smoketest-aws-us-east-1 on astra workflows vectorize-header-workflow" // because the nodes will not have been created until then. if (totalTestCount == null) { - totalTestCount = Integer.parseInt(System.getProperty(TestBenchPlan.TEST_PLAN_TEST_COUNT_PROPERTY, "0")); + totalTestCount = + Integer.parseInt(System.getProperty(TestBenchPlan.TEST_PLAN_TEST_COUNT_PROPERTY, "0")); } - writer.testStarted(totalTestCount,++startedTestCount, tracker); + writer.testStarted(totalTestCount, ++startedTestCount, tracker); } @Override @@ -91,9 +92,7 @@ public void executionSkipped(TestIdentifier id, String reason) { tracker.executionSkipped(); } - /** - * Determine if tests are running that we should be tracking. - */ + /** Determine if tests are running that we should be tracking. */ private static boolean isTestBenchNode(TestIdentifier testIdentifier) { // This is the uniqueID created by jupiter as it is traversing the code, once we get to the @@ -104,8 +103,8 @@ private static boolean isTestBenchNode(TestIdentifier testIdentifier) { } /** - * We use a Tracker for every node in the test plan, to track the execution time - * and result of it and all of its children. + * We use a Tracker for every node in the test plan, to track the execution time and result of it + * and all of its children. */ private TestReportingTracker getCreateTestTracker(TestIdentifier testIdentifier) { @@ -114,7 +113,8 @@ private TestReportingTracker getCreateTestTracker(TestIdentifier testIdentifier) return existingTracker; } - // The getSource() is a URI, jupiter / junit use it to identify the test file, but we dont have those. + // The getSource() is a URI, jupiter / junit use it to identify the test file, but we dont have + // those. // We use the {@link TestUri} instead. We need one to know what sort of test node this is var testUri = testIdentifier @@ -149,8 +149,9 @@ private TestReportingTracker getCreateTestTracker(TestIdentifier testIdentifier) /** * Container for tracking the execution of a test, and all of its children. - *

---

- * */ + * + *

--- + */ public class TestReportingTracker { private final TestIdentifier identifier; @@ -184,7 +185,7 @@ public Optional throwable() { return throwable; } - public TestExecutionResult.Status junitStatus(){ + public TestExecutionResult.Status junitStatus() { return junitStatus; } @@ -213,8 +214,9 @@ public TestContainerStats stats() { } /** - * Call when the execution of the test is finished, updates tracking for the node and - * for any ancestors. + * Call when the execution of the test is finished, updates tracking for the node and for any + * ancestors. + * * @param result */ public void executionFinished(TestExecutionResult result) { @@ -229,7 +231,8 @@ public void executionFinished(TestExecutionResult result) { } } - private void descendantExecutionFinished(TestReportingTracker originalTracker, TestExecutionResult result) { + private void descendantExecutionFinished( + TestReportingTracker originalTracker, TestExecutionResult result) { if (stats != null) { stats.testCompleted(originalTracker, result); } @@ -250,10 +253,9 @@ public void executionSkipped() { /** * Count the tests that failed etc. - *

- * Modeled on org.apache.maven.plugin.surefire.report.TestSetStats - *

- * */ + * + *

Modeled on org.apache.maven.plugin.surefire.report.TestSetStats + */ public class TestContainerStats { private final long startedAtMillis; @@ -297,11 +299,12 @@ public int failures() { public int skipped() { return skipped; } - - public void testCompleted(TestReportingTracker tracker, TestExecutionResult result) { + + public void testCompleted(TestReportingTracker tracker, TestExecutionResult result) { selfOrDescendantFinishedAtMillis = System.currentTimeMillis(); - // we only update the stats IF the test we are tracking is a TEST, we do not update for containers. + // we only update the stats IF the test we are tracking is a TEST, we do not update for + // containers. if (tracker.identifier().isTest()) { switch (result.getStatus()) { case FAILED -> failures++; diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/reporting/TestBenchConsoleWriter.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/reporting/TestBenchConsoleWriter.java index 17b414eec2..eefdd5866e 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/testbench/reporting/TestBenchConsoleWriter.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/reporting/TestBenchConsoleWriter.java @@ -3,7 +3,6 @@ import static org.apache.maven.surefire.shared.utils.logging.MessageUtils.buffer; import io.stargate.sgv2.jsonapi.testbench.testrun.TestUri; - import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; @@ -17,9 +16,10 @@ /** * Writes messages for the output of Test Bench using the logging system. * - *

- * NOTE: to get the best output via the logging system, we need to configure to remove the normal formatting - * that comes with logging. Below should be in the `applicaiton.yaml` in addition to the regular config + *

NOTE: to get the best output via the logging system, we need to configure to remove the normal + * formatting that comes with logging. Below should be in the `applicaiton.yaml` in addition to the + * regular config + * * 

  * quarkus:
  *   log:
@@ -36,20 +36,18 @@
  *           - PLAIN_CONSOLE
  *         use-parent-handlers: false
  *
- *

* *

Works in two modes because we need to wait for all the tests to complete so we know what was * successful. For info on how the output will look see {@link Theme} and {@link * org.apache.maven.surefire.shared.utils.logging.AnsiMessageBuilder} - *

- *

Call {@link #testStarted(int, int, DynamicTreeListener.TestReportingTracker)} when a test starts - * executing, will print the progress of the tests - *

- *

Call {@link #allTestsFinished(DynamicTreeListener.TestReportingTracker)} with the root - * tracker for the rest run, this should be called once all the tests have completed so we know what - * failed and how long things took. This will print the full test tree, but only go down to the - * request + assertion level for test scenarios that have failed. - *

+ * + *

Call {@link #testStarted(int, int, DynamicTreeListener.TestReportingTracker)} when a test + * starts executing, will print the progress of the tests + * + *

Call {@link #allTestsFinished(DynamicTreeListener.TestReportingTracker)} with the root tracker + * for the rest run, this should be called once all the tests have completed so we know what failed + * and how long things took. This will print the full test tree, but only go down to the request + + * assertion level for test scenarios that have failed. */ public class TestBenchConsoleWriter { @@ -72,9 +70,10 @@ public TestBenchConsoleWriter(Theme theme) { /** * Writes that a test has started, without knowing how it will end, used when the test suite is * running. - *

- * Example output: - * 

+   *
+   * 
Example output:
+   *
+   * 
    * ── ── ── ── ── ── ── ── ── ── ── ── ── ── ── ── ── ── ── ──
    * Running Test Bench, results shown at completion...
    * ── ── ── ── ── ── ── ── ── ── ── ── ── ── ── ── ── ── ── ──
@@ -86,12 +85,13 @@ public TestBenchConsoleWriter(Theme theme) {
    *             ├─ 6 of 1465: Request: SetupRequest[1]: CREATE_COLLECTION (node:aaf)
    *                ├─ 7 of 1465: Command: createCollection (node:aaa)
    *  

-   * 

+   *
    * @param totalTestCount the total number of tests that will be run
    * @param startedTestCount the number of tests that have started running, including this one
    * @param tracker the tracker for the test that started
    */
-  public void testStarted(int totalTestCount, int startedTestCount, DynamicTreeListener.TestReportingTracker tracker) {
+  public void testStarted(
+      int totalTestCount, int startedTestCount, DynamicTreeListener.TestReportingTracker tracker) {
 
     var buffer = buffer();
     if (firstLine) {
@@ -108,53 +108,57 @@ public void testStarted(int totalTestCount, int startedTestCount, DynamicTreeLis
 
     writeTreeOutline(buffer, tracker);
 
-    // the progress "x of Y" and the displayName from the node set when it was created by the TestPlan.
-    // NOTE: the "(node:aaN)" is part of the displayName, it is so that when we print failed nodes we can
+    // the progress "x of Y" and the displayName from the node set when it was created by the
+    // TestPlan.
+    // NOTE: the "(node:aaN)" is part of the displayName, it is so that when we print failed nodes
+    // we can
     // find them in the full log easily.
     buffer
-            .a(startedTestCount)
-            .a(" of ")
-            .a(totalTestCount)
-            .a(": ")
-            .strong(tracker.identifier().getDisplayName());
+        .a(startedTestCount)
+        .a(" of ")
+        .a(totalTestCount)
+        .a(": ")
+        .strong(tracker.identifier().getDisplayName());
 
     LOGGER.info(buffer.toString());
   }
 
   /**
-   * Call when all the tests have finished running, so we can print a summary for test suites that pass and
-   * details for those that fail.
-   * 

-   * Writes two types of output:
+   * Call when all the tests have finished running, so we can print a summary for test suites that
+   * pass and details for those that fail.
+   *
+   * 
Writes two types of output:
+   *
    * 
    
-   *     
  • If {@link LOGGER#isInfoEnabled()} logs a detailed report of the test results, skips details of
-   *     assertions that have completed successfully. See {@link #writeCompletedSummary(MessageBuilder, DynamicTreeListener.TestReportingTracker, boolean)}
-   *     
    • 
-   *     
  • If {@link #ENV_TEST_BENCH_REPORT_PATH} is set in the path, writes a markdown file in a format
-   *     to be included in the summary for GitHub actions via the $GITHUB_STEP_SUMMARY.
-   *      
    • 
+   *   
  • If {@link LOGGER#isInfoEnabled()} logs a detailed report of the test results, skips
+   *       details of assertions that have completed successfully. See {@link
+   *       #writeCompletedSummary(MessageBuilder, DynamicTreeListener.TestReportingTracker,
+   *       boolean)}
+   *   
  • If {@link #ENV_TEST_BENCH_REPORT_PATH} is set in the path, writes a markdown file in a
+   *       format to be included in the summary for GitHub actions via the 
+   *       $GITHUB_STEP_SUMMARY.
    * 

-   * 

-   * @param rootTracker Tracker for the root of the test tree, this is the one that has the full test tree
-   *                    under it.
+   *
+   * @param rootTracker Tracker for the root of the test tree, this is the one that has the full
+   *     test tree under it.
    */
   public void allTestsFinished(DynamicTreeListener.TestReportingTracker rootTracker) {
 
     var reportBuffer = buffer();
-    writeCompletedSummary(reportBuffer, rootTracker,  false);
+    writeCompletedSummary(reportBuffer, rootTracker, false);
     var testReport = reportBuffer.toString();
 
-    if (LOGGER.isInfoEnabled()){
+    if (LOGGER.isInfoEnabled()) {
 
       var logLineBuffer = buffer();
       logLineBuffer
-              .newline()
-              .a(theme.dash().repeat(20))
-              .newline()
-              .a("Test Bench Results")
-              .newline()
-              .a(theme.dash().repeat(20))
-              .newline();
+          .newline()
+          .a(theme.dash().repeat(20))
+          .newline()
+          .a("Test Bench Results")
+          .newline()
+          .a(theme.dash().repeat(20))
+          .newline();
       logLineBuffer.a(testReport);
       LOGGER.info(logLineBuffer.toString());
     }
@@ -170,7 +174,7 @@ public void allTestsFinished(DynamicTreeListener.TestReportingTracker rootTracke
 
       // the failed nodes and their throwable, if any
       String failureReport;
-      if (rootTracker.stats().failures() == 0){
+      if (rootTracker.stats().failures() == 0) {
         failureReport = "No failures";
       } else {
         var failureReportBuffer = buffer();
@@ -181,44 +185,52 @@ public void allTestsFinished(DynamicTreeListener.TestReportingTracker rootTracke
       // report is a markdown file
       // Using GitHub collapsable sections
       // https://docs.github.com/en/get-started/writing-on-github/working-with-advanced-formatting/organizing-information-with-collapsed-sections
-      var markdownReport = """
+      var markdownReport =
+              """
               ## %s
-              
+
               %s
-              
+
               

-              
+
               Test Bench Summary
-              
+
               ```
               %s
               ```
-              
+
               

-              
+
               

-              
+
               Test Bench Failures
-              
+
               ```
               %s
               ```
-              
-              

-              """.formatted(rootTracker.identifier().getDisplayName(), testPlanNodeDesc.toString(), testReport, failureReport);
 
-        try {
-            Files.writeString(Path.of(reportFilePath), markdownReport);
-        } catch (IOException e) {
-            throw new RuntimeException(e);
-        }
+
+ """ + .formatted( + rootTracker.identifier().getDisplayName(), + testPlanNodeDesc.toString(), + testReport, + failureReport); + + try { + Files.writeString(Path.of(reportFilePath), markdownReport); + } catch (IOException e) { + throw new RuntimeException(e); + } } } /** - * Walks the test tree, outputs the results of running each node now that we know the completed results. - *

- * Example (the header is put in th by the caller) : + * Walks the test tree, outputs the results of running each node now that we know the completed + * results. + * + *

Example (the header is put in th by the caller) : + * * 

    * ── ── ── ── ── ── ── ── ── ── ── ── ── ── ── ── ── ── ── ──
    * Test Bench Results
@@ -231,11 +243,12 @@ public void allTestsFinished(DynamicTreeListener.TestReportingTracker rootTracke
    *          ├─ ✔ TestEnv: [CREDENTIAL=open-ai-key, MODEL=text-embedding-3-large, PROVIDER=openai]  (node:abr) - 23 s   Successful: 21, Failures: 0, Aborted: 0
    *          ├─ ✔ TestEnv: [CREDENTIAL=open-ai-key, MODEL=text-embedding-ada-002, PROVIDER=openai]  (node:ab5) - 22 s   Successful: 21, Failures: 0, Aborted: 0
    *
- *

- *

- * NOTE: Because there can be 1,000s of test nodes, we do not below the TestEnv nodes unless there is a failure. - * Below those nodes are where the assertions that will have failed exist. We traverse down to the first failed test node, - * and then want to show which sibling (or cousin) nodes aborted because of the failure. For example: + * + *

NOTE: Because there can be 1,000s of test nodes, we do not below the TestEnv nodes + * unless there is a failure. Below those nodes are where the assertions that will have failed + * exist. We traverse down to the first failed test node, and then want to show which sibling (or + * cousin) nodes aborted because of the failure. For example: + * * 

    * ┬─ ✘ TestPlan: smoketest-prod-aws-eu-central-1 (105817733955) on astra (node:axM) - 413 s   Successful: 436, Failures: 20, Aborted: 300
    * ├─ ✘ Workflow: vectorize-shared-workflow  (node:alW) - 41 s   Successful: 90, Failures: 18, Aborted: 270
@@ -256,11 +269,12 @@ public void allTestsFinished(DynamicTreeListener.TestReportingTracker rootTracke
    *
* * In the first example we do not go down to the Request nodes because they did not fail or abort. - *

* * @param buffer Buffer to append the message to. - * @param tracker The tracker we are writing out the summary for, and may then traverse its children. - * @param parentFailures Set true if any parent nodes have failures, this will cause us to traverse down to the children + * @param tracker The tracker we are writing out the summary for, and may then traverse its + * children. + * @param parentFailures Set true if any parent nodes have failures, this will cause us to + * traverse down to the children */ private void writeCompletedSummary( MessageBuilder buffer, @@ -275,20 +289,23 @@ private void writeCompletedSummary( // descend until we get one // OR if there are FAILURES then we descend, these are tests that ran but assertion failed. // we do not descend for aborted, these are tests that did not run because of previous failure. - for (var child : tracker.children()){ + for (var child : tracker.children()) { var hasFailures = child.stats() != null && child.stats().failures() > 0; - if (!child.runUri().leafType().descendantOf(TestUri.Segment.ENV) || hasFailures || parentFailures) { + if (!child.runUri().leafType().descendantOf(TestUri.Segment.ENV) + || hasFailures + || parentFailures) { writeCompletedSummary(buffer, child, hasFailures); } } } /** - * Writes the test node name, and it's throwable if it Failed, as in the assertions failed or it threw - * an exception. - *

- * Example: + * Writes the test node name, and it's throwable if it Failed, as in the assertions failed or it + * threw an exception. + * + *

Example: + * * 

    * ✘ isDDLSuccess [body('$') - responseIsDDLSuccess: REQUIRED:[status], OPTIONAL:[], FORBIDDEN:[data, errors]] (node:aac)
    *
@@ -310,13 +327,14 @@ private void writeCompletedSummary(
    *   "status"
    * -----
    *
- *

*/ - private void writeFailureMessages(MessageBuilder buffer, DynamicTreeListener.TestReportingTracker tracker) { + private void writeFailureMessages( + MessageBuilder buffer, DynamicTreeListener.TestReportingTracker tracker) { // if we have a throwable, write out the tree node test and the error it generated. // ignoring the ABORTED - if (tracker.throwable().isPresent() && (tracker.junitStatus() == TestExecutionResult.Status.FAILED )) { + if (tracker.throwable().isPresent() + && (tracker.junitStatus() == TestExecutionResult.Status.FAILED)) { writeTestDesc(buffer, tracker); buffer.newline(); buffer.newline(); @@ -326,18 +344,20 @@ private void writeFailureMessages(MessageBuilder buffer, DynamicTreeListener.Tes tracker.children().forEach(child -> writeFailureMessages(buffer, child)); } - private void writeTestDesc(MessageBuilder buffer, DynamicTreeListener.TestReportingTracker tracker) { + private void writeTestDesc( + MessageBuilder buffer, DynamicTreeListener.TestReportingTracker tracker) { // Icon for success for failure, - // if we have stats then this is a container we use that status, otherwise we use the JUNIT execution status - if (tracker.stats() == null){ + // if we have stats then this is a container we use that status, otherwise we use the JUNIT + // execution status + if (tracker.stats() == null) { switch (tracker.junitStatus()) { case SUCCESSFUL -> buffer.a(theme.successful()); case FAILED, ABORTED -> buffer.a(theme.failed()); case null -> {} } } else { - if (tracker.stats().failures() ==0 && tracker.stats().aborted() == 0) { + if (tracker.stats().failures() == 0 && tracker.stats().aborted() == 0) { buffer.a(theme.successful()); } else { buffer.a(theme.failed()); @@ -355,24 +375,29 @@ private void writeTestDesc(MessageBuilder buffer, DynamicTreeListener.TestReport // NOTE: does not add new line, caller should } - private void writeTestStats(MessageBuilder buffer, DynamicTreeListener.TestReportingTracker tracker) { + private void writeTestStats( + MessageBuilder buffer, DynamicTreeListener.TestReportingTracker tracker) { buffer - .a(" - %s s".formatted(tracker.stats().elapsedMillis() / 1000)) - .a(theme.blank()) - .a("Successful: ").a( tracker.stats().successful()).a(", ") - .a("Failures: ").a( tracker.stats().failures()).a(", ") - .a("Aborted: ").a( tracker.stats().aborted()); + .a(" - %s s".formatted(tracker.stats().elapsedMillis() / 1000)) + .a(theme.blank()) + .a("Successful: ") + .a(tracker.stats().successful()) + .a(", ") + .a("Failures: ") + .a(tracker.stats().failures()) + .a(", ") + .a("Aborted: ") + .a(tracker.stats().aborted()); } - private void writeTreeOutline(MessageBuilder buffer, DynamicTreeListener.TestReportingTracker tracker) { + private void writeTreeOutline( + MessageBuilder buffer, DynamicTreeListener.TestReportingTracker tracker) { // Indenting and tree building if (tracker.parent() == null) { buffer.a(theme.down()); } else { - buffer - .a(theme.blank().repeat(tracker.depth() - 1)) - .a(theme.entry()); + buffer.a(theme.blank().repeat(tracker.depth() - 1)).a(theme.entry()); } } } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/targets/AstraBackend.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/targets/AstraBackend.java index ae9b96ac32..db642a35d6 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/testbench/targets/AstraBackend.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/targets/AstraBackend.java @@ -2,9 +2,7 @@ import io.stargate.sgv2.jsonapi.testbench.testspec.Job; -/** - * DataStax / IBM Astra - */ +/** DataStax / IBM Astra */ public class AstraBackend extends Backend { public static final String NAME = "astra"; diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/targets/Backend.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/targets/Backend.java index f27e52b069..7d29c6e1d1 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/testbench/targets/Backend.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/targets/Backend.java @@ -2,24 +2,20 @@ import io.stargate.sgv2.jsonapi.testbench.lifecycle.JobLifeCycle; import io.stargate.sgv2.jsonapi.testbench.lifecycle.TestPlanLifecycle; - import java.util.regex.Pattern; /** - * A class of backend database the tests will be run against. This is not a partcular insance of a DB to run - * against, for that see {@link Target}. - *

- * There is some different behavior between C* and Astra, and potentially in any future versions of - * those products. - *

+ * A class of backend database the tests will be run against. This is not a partcular insance of a + * DB to run against, for that see {@link Target}. + * + *

There is some different behavior between C* and Astra, and potentially in any future versions + * of those products. */ public abstract class Backend implements TestPlanLifecycle, JobLifeCycle { private static final Pattern PATTERN_NOT_WORD_CHARS = Pattern.compile("\\W+"); - /** - * Sanitizes a schema name to be valid in all C* derived platforms. - */ + /** Sanitizes a schema name to be valid in all C* derived platforms. */ public static String toSafeSchemaIdentifier(String name) { var newValue = PATTERN_NOT_WORD_CHARS.matcher(name).replaceAll("_"); diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/targets/CassandraBackend.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/targets/CassandraBackend.java index 4b46b50265..826439148e 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/testbench/targets/CassandraBackend.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/targets/CassandraBackend.java @@ -7,14 +7,11 @@ import io.stargate.sgv2.jsonapi.testbench.testrun.TestUri; import io.stargate.sgv2.jsonapi.testbench.testspec.Job; import io.stargate.sgv2.jsonapi.testbench.testspec.TestCommand; +import java.util.Optional; import org.apache.commons.lang3.RandomStringUtils; import org.junit.jupiter.api.DynamicNode; -import java.util.Optional; - -/** - * A classic Cassandra backend, running locally or elsewhere. - * */ +/** A classic Cassandra backend, running locally or elsewhere. */ public class CassandraBackend extends Backend { public static final String NAME = "cassandra"; @@ -22,7 +19,8 @@ public class CassandraBackend extends Backend { @Override public void updateJobForTarget(Job job) { - // We are going to create the keyspace for every job, so making a new name here based on the name + // We are going to create the keyspace for every job, so making a new name here based on the + // name // of the job, and some random because the name will be trunc'd var keyspaceName = toSafeSchemaIdentifier( @@ -33,11 +31,10 @@ public void updateJobForTarget(Job job) { job.variables().put("KEYSPACE_NAME", keyspaceName); } - /** - * Need to create a keyspace, because C* does not have a default one. - */ + /** Need to create a keyspace, because C* does not have a default one. */ @Override - public Optional beforeJob(TestNodeFactory testNodeFactory, TestUri.Builder uriBuilder, Job job) { + public Optional beforeJob( + TestNodeFactory testNodeFactory, TestUri.Builder uriBuilder, Job job) { var command = TestCommand.fromJson( @@ -62,11 +59,10 @@ public Optional beforeJob(TestNodeFactory testNodeFactory, TestUri. return Optional.of(setupRequest.testNodes(testNodeFactory, uriBuilder)); } - /** - * Drop the keyspace we made for this job. - */ + /** Drop the keyspace we made for this job. */ @Override - public Optional afterJob(TestNodeFactory testNodeFactory, TestUri.Builder uriBuilder, Job job) { + public Optional afterJob( + TestNodeFactory testNodeFactory, TestUri.Builder uriBuilder, Job job) { var command = TestCommand.fromJson( """ @@ -82,7 +78,7 @@ public Optional afterJob(TestNodeFactory testNodeFactory, TestUri.B new TestRunRequest( env.substitutor().replace("dropKeyspace: ${KEYSPACE_NAME}"), command, - testNodeFactory.testPlan().target(), + testNodeFactory.testPlan().target(), job.withoutMatrix(testNodeFactory.testPlan()), TestAssertion.forSuccess(testNodeFactory.testPlan(), command), new TestExecutionCondition.AlwaysTrue("CassandraBackend.afterJob()")); diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/targets/ConnectionConfiguration.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/targets/ConnectionConfiguration.java index 5dfd8213bf..685caa3bdf 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/testbench/targets/ConnectionConfiguration.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/targets/ConnectionConfiguration.java @@ -2,14 +2,15 @@ /** * Configuraiton record of how to connect for a {@link TargetConfiguration}. - *

---

+ * + *

--- * * @param domain domain and protocol, e.g. http://localhost - * @param port port to connect to, typically 8181 ore 443 but may be different when running integration tests. - * @param basePath base path to the API, for Astra this is /api/json/v1 when running locally is normally - * /v1 -*/ - + * @param port port to connect to, typically 8181 ore 443 but may be + * different when running integration tests. + * @param basePath base path to the API, for Astra this is /api/json/v1 when running + * locally is normally /v1 + */ public record ConnectionConfiguration(String domain, Integer port, String basePath) { public ConnectionConfiguration { if (domain == null) { diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/targets/Target.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/targets/Target.java index 6f719047a1..f2885828cf 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/testbench/targets/Target.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/targets/Target.java @@ -10,24 +10,21 @@ import io.stargate.sgv2.jsonapi.testbench.testspec.TestCommand; import io.stargate.sgv2.jsonapi.testbench.testspec.TestSuiteSpec; import io.stargate.sgv2.jsonapi.testbench.testspec.WorkflowSpec; -import org.junit.jupiter.api.DynamicNode; - import java.util.Optional; +import org.junit.jupiter.api.DynamicNode; /** - * A particular instance of a {@link Backend} we are going to run the test against, - * so includes connection information etc. - *

- * A run of the Test Bench is run against a Target, e.g. cassandra on localhost, or - * an astra db called monkeys. - *

- *

- * This is the important entry point for the lifecycle interfaces, because the life cycle is there - * to handle different target / backends that tests run against. - *

- *

- * Because this holds the connection information, it can also make a request, see {@link #apiRequest(TestCommand, TestRunEnv)} - *

+ * A particular instance of a {@link Backend} we are going to run the test against, so includes + * connection information etc. + * + *

A run of the Test Bench is run against a Target, e.g. cassandra on localhost, or an astra db + * called monkeys. + * + *

This is the important entry point for the lifecycle interfaces, because the life cycle is + * there to handle different target / backends that tests run against. + * + *

Because this holds the connection information, it can also make a request, see {@link + * #apiRequest(TestCommand, TestRunEnv)} */ public class Target implements TestPlanLifecycle, JobLifeCycle { @@ -52,12 +49,13 @@ public TargetConfiguration configuration() { } /** - * Call this to get a new {@link APIRequest} that is configured to talk to the target - * this class represents. - * @param testCommand The command the request will send, this is needed to get the actual - * DataAPI request we want to send. + * Call this to get a new {@link APIRequest} that is configured to talk to the target this class + * represents. + * + * @param testCommand The command the request will send, this is needed to get the actual DataAPI + * request we want to send. * @param env Environment the commands will be run in, used to make the replacements in the - * command to execute for this particular test run. + * command to execute for this particular test run. * @return Configured {@link APIRequest} */ public APIRequest apiRequest(TestCommand testCommand, TestRunEnv env) { @@ -71,35 +69,43 @@ public void updateJobForTarget(Job job) { @Override public Optional beforeWorkflow( - TestNodeFactory testNodeFactory, TestUri.Builder uriBuilder, WorkflowSpec workflow) { + TestNodeFactory testNodeFactory, TestUri.Builder uriBuilder, WorkflowSpec workflow) { return backend.beforeWorkflow(testNodeFactory, uriBuilder, workflow); } @Override public Optional afterWorkflow( - TestNodeFactory testNodeFactory, TestUri.Builder uriBuilder, WorkflowSpec workflow) { + TestNodeFactory testNodeFactory, TestUri.Builder uriBuilder, WorkflowSpec workflow) { return backend.afterWorkflow(testNodeFactory, uriBuilder, workflow); } @Override - public Optional beforeJob(TestNodeFactory testNodeFactory, TestUri.Builder uriBuilder, Job job) { + public Optional beforeJob( + TestNodeFactory testNodeFactory, TestUri.Builder uriBuilder, Job job) { return backend.beforeJob(testNodeFactory, uriBuilder, job); } @Override - public Optional afterJob(TestNodeFactory testNodeFactory, TestUri.Builder uriBuilder, Job job) { + public Optional afterJob( + TestNodeFactory testNodeFactory, TestUri.Builder uriBuilder, Job job) { return backend.afterJob(testNodeFactory, uriBuilder, job); } @Override public Optional beforeTestSuite( - TestNodeFactory testNodeFactory, TestUri.Builder uriBuilder, TestSuiteSpec test, TestRunEnv env) { + TestNodeFactory testNodeFactory, + TestUri.Builder uriBuilder, + TestSuiteSpec test, + TestRunEnv env) { return backend.beforeTestSuite(testNodeFactory, uriBuilder, test, env); } @Override public Optional afterTestSuite( - TestNodeFactory testNodeFactory, TestUri.Builder uriBuilder, TestSuiteSpec test, TestRunEnv env) { + TestNodeFactory testNodeFactory, + TestUri.Builder uriBuilder, + TestSuiteSpec test, + TestRunEnv env) { return backend.afterTestSuite(testNodeFactory, uriBuilder, test, env); } } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/targets/TargetConfiguration.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/targets/TargetConfiguration.java index f88f2a835e..84eb1dfcda 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/testbench/targets/TargetConfiguration.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/targets/TargetConfiguration.java @@ -2,10 +2,12 @@ /** * Configuration record for a {@link Target}. - *

---

+ * + *

--- * * @param name Friendly name of the target * @param backend Backend name, such as astra or cassandra so we know how to run the lifecycle * @param connection Connection information for the target. */ -public record TargetConfiguration(String name, String backend, ConnectionConfiguration connection) {} +public record TargetConfiguration( + String name, String backend, ConnectionConfiguration connection) {} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/testrun/DynamicTestExecutable.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testrun/DynamicTestExecutable.java index c18c81ecd4..4a725153ca 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/testbench/testrun/DynamicTestExecutable.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testrun/DynamicTestExecutable.java @@ -14,12 +14,20 @@ public class DynamicTestExecutable implements Executable { private final boolean isTrimmed; private final TestExecutionCondition testExecutionCondition; - public DynamicTestExecutable(String description, TestUri.Builder testUri, TestExecutionCondition testExecutionCondition, Executable executable) { + public DynamicTestExecutable( + String description, + TestUri.Builder testUri, + TestExecutionCondition testExecutionCondition, + Executable executable) { this(description, testUri.build(), testExecutionCondition, executable); } @SuppressWarnings("StringEquality") - public DynamicTestExecutable(String description, TestUri testUri, TestExecutionCondition testExecutionCondition, Executable executable) { + public DynamicTestExecutable( + String description, + TestUri testUri, + TestExecutionCondition testExecutionCondition, + Executable executable) { this.description = description; this.testUri = testUri; this.testExecutionCondition = testExecutionCondition; @@ -45,12 +53,11 @@ public DynamicTest testNode(TestNodeFactory testNodeFactory) { @Override public void execute() throws Throwable { - Assumptions.assumeTrue(testExecutionCondition, testExecutionCondition.message()); + Assumptions.assumeTrue(testExecutionCondition, testExecutionCondition.message()); beforeExecute(); try { executable.execute(); - } - catch (Throwable e) { + } catch (Throwable e) { testExecutionCondition.abortFutureTests("Failed Upstream: " + trimmedDisplayName); throw e; } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/testrun/TestExecutionCondition.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testrun/TestExecutionCondition.java index 55d42ff2cd..a3ba0df3af 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/testbench/testrun/TestExecutionCondition.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testrun/TestExecutionCondition.java @@ -4,61 +4,62 @@ public interface TestExecutionCondition extends BooleanSupplier { - void abortFutureTests(String message); + void abortFutureTests(String message); - String message(); + String message(); - public class Default implements TestExecutionCondition { + public class Default implements TestExecutionCondition { - private boolean condition = true; - private String message = ""; + private boolean condition = true; + private String message = ""; - // to scope of this condition, i.e. this is for - // TestEnv: [MODEL=NV-Embed-QA, PROVIDER=nvidia] - private String scope; + // to scope of this condition, i.e. this is for + // TestEnv: [MODEL=NV-Embed-QA, PROVIDER=nvidia] + private String scope; - public Default(String scope) { - this.scope = scope; - } - @Override - public void abortFutureTests(String message) { - condition = false; - this.message = message; - } + public Default(String scope) { + this.scope = scope; + } + + @Override + public void abortFutureTests(String message) { + condition = false; + this.message = message; + } - @Override - public boolean getAsBoolean() { - return condition; - } + @Override + public boolean getAsBoolean() { + return condition; + } - @Override - public String message() { - return "TestCondition: Scope=" + scope + ", Message=" + message; - } + @Override + public String message() { + return "TestCondition: Scope=" + scope + ", Message=" + message; } + } - public class AlwaysTrue implements TestExecutionCondition { + public class AlwaysTrue implements TestExecutionCondition { - // to scope of this condition, i.e. this is for - // TestEnv: [MODEL=NV-Embed-QA, PROVIDER=nvidia] - // we never need a message, because always true, but here for debugging. - private String scope; + // to scope of this condition, i.e. this is for + // TestEnv: [MODEL=NV-Embed-QA, PROVIDER=nvidia] + // we never need a message, because always true, but here for debugging. + private String scope; - public AlwaysTrue(String scope) { - this.scope = scope; - } + public AlwaysTrue(String scope) { + this.scope = scope; + } - @Override - public void abortFutureTests(String message) {} + @Override + public void abortFutureTests(String message) {} - @Override - public boolean getAsBoolean() { - return true; - } + @Override + public boolean getAsBoolean() { + return true; + } - @Override - public String message() { - return ""; - } + @Override + public String message() { + return ""; } -} \ No newline at end of file + } +} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/testrun/TestNodeFactory.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testrun/TestNodeFactory.java index ff820e1b81..5990d2ed72 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/testbench/testrun/TestNodeFactory.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testrun/TestNodeFactory.java @@ -5,176 +5,183 @@ import io.stargate.sgv2.jsonapi.testbench.testspec.TestSpecMeta; import io.stargate.sgv2.jsonapi.testbench.testspec.TestSuiteSpec; import io.stargate.sgv2.jsonapi.testbench.testspec.WorkflowSpec; -import org.junit.jupiter.api.DynamicContainer; -import org.junit.jupiter.api.DynamicNode; -import org.junit.jupiter.api.DynamicTest; -import org.junit.jupiter.api.function.Executable; - import java.net.URI; import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Optional; import java.util.function.Supplier; +import org.junit.jupiter.api.DynamicContainer; +import org.junit.jupiter.api.DynamicNode; +import org.junit.jupiter.api.DynamicTest; +import org.junit.jupiter.api.function.Executable; /** - * A container we can pass around when building the TestNodes that has the test plan - * we are working with, and has a counter for the number of nodes we created. + * A container we can pass around when building the TestNodes that has the test plan we are working + * with, and has a counter for the number of nodes we created. * - *

- * So we can output them for the test reporter to report progress. - *

+ *

So we can output them for the test reporter to report progress. */ public class TestNodeFactory { - private final NodeCode nodeCode = new NodeCode(); - private final TestBenchPlan testPlan; - private int totalNodeCount = 0; - - public TestNodeFactory(TestBenchPlan testPlan) { - this.testPlan = testPlan; - } - - public TestBenchPlan testPlan() { - return testPlan; - } - - public int testNodeCount(){ - return totalNodeCount; - } - - /** - * NOTE: This is forcing the use of a List, so we greedily create all the test nodes, so we can count - * how many their are, so we can show progress. - * @param displayName - * @param testSourceUri - * @param dynamicNodes - * @return - */ - public DynamicContainer testPlanContainer(String displayName, URI testSourceUri, - List dynamicNodes){ - totalNodeCount++; - return DynamicContainer.dynamicContainer(appendNodeCode(displayName), testSourceUri, dynamicNodes.stream()); - } - - public DynamicTest testPlanTest(String description, URI uri, Executable executable){ - - totalNodeCount++; - return DynamicTest.dynamicTest(appendNodeCode(description), uri, executable); - } - - private String appendNodeCode(String displayName){ - return "%s (node:%s)".formatted(displayName, nodeCode.next()); - } - - public List addLifecycle( - TestUri.Builder uriBuilder, - WorkflowSpec workflow, - List dynamicNodes) { - - var beforeUriBuilder = - uriBuilder.clone().addSegment(TestUri.Segment.LIFECYCLE, "before-workflow"); - var afterUriBuilder = - uriBuilder.clone().addSegment(TestUri.Segment.LIFECYCLE, "after-workflow"); - - var nodes = new ArrayList(); - nodes.addAll(lifecycleNodes( - beforeUriBuilder, - "Before Workflow", - workflow.meta(), - () -> testPlan.target().beforeWorkflow(this, beforeUriBuilder, workflow))); - nodes.addAll(dynamicNodes); - nodes.addAll(lifecycleNodes( - afterUriBuilder, - "After Workflow", - workflow.meta(), - () -> testPlan.target().afterWorkflow(this, afterUriBuilder, workflow))); - return nodes; - } - - public List addLifecycle( - TestUri.Builder uriBuilder, Job job, List dynamicNodes) { - - var beforeUriBuilder = uriBuilder.clone().addSegment(TestUri.Segment.LIFECYCLE, "before-job"); - var afterUriBuilder = uriBuilder.clone().addSegment(TestUri.Segment.LIFECYCLE, "after-job"); - - var nodes = new ArrayList(); - nodes.addAll(lifecycleNodes( - beforeUriBuilder, - "Before Job", - job.meta(), - () -> testPlan.target().beforeJob(this, beforeUriBuilder, job))); - nodes.addAll(dynamicNodes); - nodes.addAll(lifecycleNodes( - afterUriBuilder, - "After Job", - job.meta(), - () -> testPlan.target().afterJob(this, afterUriBuilder, job))); - return nodes; - } - - public List addLifecycle( - TestUri.Builder uriBuilder, - TestSuiteSpec testSuite, - TestRunEnv environment, - List dynamicNodes) { - - var beforeUriBuilder = - uriBuilder.clone().addSegment(TestUri.Segment.LIFECYCLE, "before-test-suite"); - var afterUriBuilder = - uriBuilder.clone().addSegment(TestUri.Segment.LIFECYCLE, "after-test-suite"); - - var nodes = new ArrayList(); - nodes.addAll(lifecycleNodes( - beforeUriBuilder, - "Before TestSuite", - testSuite.meta(), - () -> testPlan.target().beforeTestSuite(this, beforeUriBuilder, testSuite, environment))); - nodes.addAll(dynamicNodes); - nodes.addAll(lifecycleNodes( - afterUriBuilder, - "After TestSuite", - testSuite.meta(), - () -> testPlan.target().afterTestSuite(this, afterUriBuilder, testSuite, environment))); - return nodes; + private final NodeCode nodeCode = new NodeCode(); + private final TestBenchPlan testPlan; + private int totalNodeCount = 0; + + public TestNodeFactory(TestBenchPlan testPlan) { + this.testPlan = testPlan; + } + + public TestBenchPlan testPlan() { + return testPlan; + } + + public int testNodeCount() { + return totalNodeCount; + } + + /** + * NOTE: This is forcing the use of a List, so we greedily create all the test nodes, so we can + * count how many their are, so we can show progress. + * + * @param displayName + * @param testSourceUri + * @param dynamicNodes + * @return + */ + public DynamicContainer testPlanContainer( + String displayName, URI testSourceUri, List dynamicNodes) { + totalNodeCount++; + return DynamicContainer.dynamicContainer( + appendNodeCode(displayName), testSourceUri, dynamicNodes.stream()); + } + + public DynamicTest testPlanTest(String description, URI uri, Executable executable) { + + totalNodeCount++; + return DynamicTest.dynamicTest(appendNodeCode(description), uri, executable); + } + + private String appendNodeCode(String displayName) { + return "%s (node:%s)".formatted(displayName, nodeCode.next()); + } + + public List addLifecycle( + TestUri.Builder uriBuilder, WorkflowSpec workflow, List dynamicNodes) { + + var beforeUriBuilder = + uriBuilder.clone().addSegment(TestUri.Segment.LIFECYCLE, "before-workflow"); + var afterUriBuilder = + uriBuilder.clone().addSegment(TestUri.Segment.LIFECYCLE, "after-workflow"); + + var nodes = new ArrayList(); + nodes.addAll( + lifecycleNodes( + beforeUriBuilder, + "Before Workflow", + workflow.meta(), + () -> testPlan.target().beforeWorkflow(this, beforeUriBuilder, workflow))); + nodes.addAll(dynamicNodes); + nodes.addAll( + lifecycleNodes( + afterUriBuilder, + "After Workflow", + workflow.meta(), + () -> testPlan.target().afterWorkflow(this, afterUriBuilder, workflow))); + return nodes; + } + + public List addLifecycle( + TestUri.Builder uriBuilder, Job job, List dynamicNodes) { + + var beforeUriBuilder = uriBuilder.clone().addSegment(TestUri.Segment.LIFECYCLE, "before-job"); + var afterUriBuilder = uriBuilder.clone().addSegment(TestUri.Segment.LIFECYCLE, "after-job"); + + var nodes = new ArrayList(); + nodes.addAll( + lifecycleNodes( + beforeUriBuilder, + "Before Job", + job.meta(), + () -> testPlan.target().beforeJob(this, beforeUriBuilder, job))); + nodes.addAll(dynamicNodes); + nodes.addAll( + lifecycleNodes( + afterUriBuilder, + "After Job", + job.meta(), + () -> testPlan.target().afterJob(this, afterUriBuilder, job))); + return nodes; + } + + public List addLifecycle( + TestUri.Builder uriBuilder, + TestSuiteSpec testSuite, + TestRunEnv environment, + List dynamicNodes) { + + var beforeUriBuilder = + uriBuilder.clone().addSegment(TestUri.Segment.LIFECYCLE, "before-test-suite"); + var afterUriBuilder = + uriBuilder.clone().addSegment(TestUri.Segment.LIFECYCLE, "after-test-suite"); + + var nodes = new ArrayList(); + nodes.addAll( + lifecycleNodes( + beforeUriBuilder, + "Before TestSuite", + testSuite.meta(), + () -> + testPlan.target().beforeTestSuite(this, beforeUriBuilder, testSuite, environment))); + nodes.addAll(dynamicNodes); + nodes.addAll( + lifecycleNodes( + afterUriBuilder, + "After TestSuite", + testSuite.meta(), + () -> testPlan.target().afterTestSuite(this, afterUriBuilder, testSuite, environment))); + return nodes; + } + + private List lifecycleNodes( + TestUri.Builder uriBuilder, + String namePrefix, + TestSpecMeta meta, + Supplier> nodeSupplier) { + + var targetDynamicNode = nodeSupplier.get(); + + if (targetDynamicNode.isEmpty()) { + return Collections.emptyList(); } - private List lifecycleNodes( - TestUri.Builder uriBuilder, - String namePrefix, - TestSpecMeta meta, - Supplier> nodeSupplier) { - - var targetDynamicNode = nodeSupplier.get(); - - if (targetDynamicNode.isEmpty()) { - return Collections.emptyList(); - } - - var lifecycleContainer = testPlanContainer( - namePrefix + ": " + meta.name(), uriBuilder.build().uri(), List.of(targetDynamicNode.get())); - return List.of(lifecycleContainer); - } - - public static class NodeCode { - - private static final char[] ALPHABET = - "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789".toCharArray(); - private static final int BASE = ALPHABET.length; // 62 - - // 3 characters of base 62 coding above gives 62^3 = 238,328 - private static final int LENGTH = 3; - - private int counter = 0; - - public String next() { - int n = counter++; - char[] code = new char[LENGTH]; - for (int i = LENGTH - 1; i >= 0; i--) { - code[i] = ALPHABET[n % BASE]; - n /= BASE; - } - return new String(code); - } + var lifecycleContainer = + testPlanContainer( + namePrefix + ": " + meta.name(), + uriBuilder.build().uri(), + List.of(targetDynamicNode.get())); + return List.of(lifecycleContainer); + } + + public static class NodeCode { + + private static final char[] ALPHABET = + "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789".toCharArray(); + private static final int BASE = ALPHABET.length; // 62 + + // 3 characters of base 62 coding above gives 62^3 = 238,328 + private static final int LENGTH = 3; + + private int counter = 0; + + public String next() { + int n = counter++; + char[] code = new char[LENGTH]; + for (int i = LENGTH - 1; i >= 0; i--) { + code[i] = ALPHABET[n % BASE]; + n /= BASE; + } + return new String(code); } + } } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/testrun/TestRunEnv.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testrun/TestRunEnv.java index bb3031c18b..7726a9910c 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/testbench/testrun/TestRunEnv.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testrun/TestRunEnv.java @@ -5,7 +5,6 @@ import java.util.HashMap; import java.util.Map; import java.util.Set; -import java.util.regex.Pattern; import org.apache.commons.text.StringSubstitutor; import org.apache.commons.text.lookup.StringLookupFactory; import org.junit.jupiter.api.DynamicContainer; @@ -21,10 +20,8 @@ public class TestRunEnv { // Also for tables public static final String ENV_COLLECTION_NAME = "COLLECTION_NAME"; - private static final Set SCHEMA_IDENTIFIER = Set.of( - ENV_KEYSPACE_NAME, - ENV_COLLECTION_NAME - ); + private static final Set SCHEMA_IDENTIFIER = + Set.of(ENV_KEYSPACE_NAME, ENV_COLLECTION_NAME); private final Map vars = new HashMap<>(); @@ -37,19 +34,21 @@ public TestRunEnv(Map vars) { } public DynamicContainer testNode( - TestNodeFactory testNodeFactory, TestUri.Builder uriBuilder, TestSuiteSpec testSuite) { + TestNodeFactory testNodeFactory, TestUri.Builder uriBuilder, TestSuiteSpec testSuite) { var d = description(); uriBuilder.addSegment(TestUri.Segment.ENV, d); var desc = "TestEnv: %s ".formatted(d); var testExecutionCondition = new TestExecutionCondition.Default(desc); - var envNodes = testSuite.testNodesForEnvironment(testNodeFactory, uriBuilder.clone(), this, testExecutionCondition); + var envNodes = + testSuite.testNodesForEnvironment( + testNodeFactory, uriBuilder.clone(), this, testExecutionCondition); return testNodeFactory.testPlanContainer( desc, uriBuilder.build().uri(), - testNodeFactory.addLifecycle(uriBuilder.clone(), testSuite, this, envNodes)); + testNodeFactory.addLifecycle(uriBuilder.clone(), testSuite, this, envNodes)); } private String description() { @@ -103,9 +102,10 @@ public StringSubstitutor substitutor() { .setEnableUndefinedVariableException(true); } - public String get(String name){ + public String get(String name) { return get(name, ""); } + public String get(String name, String defaultValue) { var value = vars.get(name); @@ -114,10 +114,12 @@ public String get(String name, String defaultValue) { } var substituted = substitutor().replace(value); - return SCHEMA_IDENTIFIER.contains(name) ? Backend.toSafeSchemaIdentifier(substituted) : substituted; + return SCHEMA_IDENTIFIER.contains(name) + ? Backend.toSafeSchemaIdentifier(substituted) + : substituted; } - @Override + @Override public String toString() { return vars.toString(); } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/testrun/TestRunRequest.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testrun/TestRunRequest.java index c2a8cf2647..bbf6877e2d 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/testbench/testrun/TestRunRequest.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testrun/TestRunRequest.java @@ -1,15 +1,11 @@ package io.stargate.sgv2.jsonapi.testbench.testrun; - - import io.stargate.sgv2.jsonapi.testbench.assertions.TestAssertion; import io.stargate.sgv2.jsonapi.testbench.targets.Target; import io.stargate.sgv2.jsonapi.testbench.testspec.TestCommand; - import java.util.ArrayList; import java.util.List; import java.util.concurrent.atomic.AtomicReference; - import org.junit.jupiter.api.DynamicContainer; import org.junit.jupiter.api.DynamicNode; @@ -54,13 +50,17 @@ public DynamicContainer testNodes(TestNodeFactory testNodeFactory, TestUri.Build testAssertions().stream() .map( testAssertion -> - testAssertion.testNodes(testNodeFactory, assertionsUriBuilder.clone(), atomicResponse, testExecutionCondition)) + testAssertion.testNodes( + testNodeFactory, + assertionsUriBuilder.clone(), + atomicResponse, + testExecutionCondition)) .toList(); // if we have assertion tests, put them in a container if (!assertionTests.isEmpty()) { nodes.add( - testNodeFactory.testPlanContainer( + testNodeFactory.testPlanContainer( "Assertions", assertionsUriBuilder.build().uri(), assertionTests)); } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/testrun/TestRunResponse.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testrun/TestRunResponse.java index d92a34e684..50a3f80f33 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/testbench/testrun/TestRunResponse.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testrun/TestRunResponse.java @@ -4,30 +4,4 @@ import io.stargate.sgv2.jsonapi.testbench.messaging.APIResponse; public record TestRunResponse( - TestRunRequest testRequest, APIRequest apiRequest, APIResponse apiResponse) { - - // public TestCaseResult validate(TestSuite integrationTest, TestCase testCase){ - // return validate(integrationTest, testCase, true); - // } - // - // public TestCaseResult validate(TestSuite testSuite, TestCase testCase, boolean throwOnError) { - // - // AssertionError assertionError = null; - // AssertionMatcher failedAssertion = null; - // for (var testAssertion : testRequest.testAssertions()){ - // try{ - // testAssertion.run(apiResponse); - // } - // catch(AssertionError e){ - // if (throwOnError){ - // throw e; - // } - // - // failedAssertion = testAssertion; - // assertionError = e; - // break; - // } - // } - // return new TestCaseResult(testSuite, testCase, this , assertionError, failedAssertion); - // } -} + TestRunRequest testRequest, APIRequest apiRequest, APIResponse apiResponse) {} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/testrun/TestUri.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testrun/TestUri.java index 0b211d24d3..cdcae2c4f2 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/testbench/testrun/TestUri.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testrun/TestUri.java @@ -82,7 +82,7 @@ public String pathName() { } public boolean descendantOf(Segment segment) { - if (parent == null){ + if (parent == null) { return false; } if (this.parent == segment) { diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/AssertionTemplateSpec.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/AssertionTemplateSpec.java index 786b4ca452..01aa785e03 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/AssertionTemplateSpec.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/AssertionTemplateSpec.java @@ -2,14 +2,16 @@ import com.fasterxml.jackson.databind.JsonNode; import io.stargate.sgv2.jsonapi.testbench.assertions.AssertionFactory; - import java.util.Map; import java.util.Optional; /** - * Spec defintions about re-usable assertions that are defined in JSON. See {@link AssertionFactory}. - *

- * Example, showing the defintion for "isSuccess" assertion and how it is defined on a per command basis. + * Spec defintions about re-usable assertions that are defined in JSON. See {@link + * AssertionFactory}. + * + *

Example, showing the defintion for "isSuccess" assertion and how it is defined on a per + * command basis. + * * 

  * {
  *   "meta": {
@@ -28,10 +30,9 @@
  *       },
  *
* - *

* @param meta Metadata for the spec. - * @param templates JSON Object that is a map of assertion name to a map of implementations per API command, - * see example + * @param templates JSON Object that is a map of assertion name to a map of implementations per API + * command, see example */ public record AssertionTemplateSpec(TestSpecMeta meta, Map templates) implements TestSpec { diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/Job.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/Job.java index 0c2ec128ac..b004ac6166 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/Job.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/Job.java @@ -1,7 +1,5 @@ package io.stargate.sgv2.jsonapi.testbench.testspec; - - import io.stargate.sgv2.jsonapi.testbench.TestBenchPlan; import io.stargate.sgv2.jsonapi.testbench.testrun.TestNodeFactory; import io.stargate.sgv2.jsonapi.testbench.testrun.TestRunEnv; @@ -31,7 +29,8 @@ public DynamicContainer testNode(TestNodeFactory testNodeFactory, TestUri.Builde testNodeFactory.testPlan().updateJobForTarget(this); var allEnvs = allEnvironments(testNodeFactory.testPlan()); - var testSuiteNodes = testSuites(testNodeFactory.testPlan()) + var testSuiteNodes = + testSuites(testNodeFactory.testPlan()) .map(testSuite -> testSuite.testNode(testNodeFactory, uriBuilder.clone(), allEnvs)) .toList(); diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/SpecFile.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/SpecFile.java index 04a5733d73..c515b927f1 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/SpecFile.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/SpecFile.java @@ -6,10 +6,9 @@ /** * A Spec file is any JSON file we have that is used to drive the test suite, workflows etc. - *

- * This is a container of the file, it's raw JSON, and the {@link TestSpec} which is the common + * + *

This is a container of the file, it's raw JSON, and the {@link TestSpec} which is the common * parsed object from the JSON. - *

*/ public record SpecFile(File file, TestSpec spec, JsonNode root) { diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/SpecFiles.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/SpecFiles.java index 617d1d5ca8..0f6e2a0798 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/SpecFiles.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/SpecFiles.java @@ -12,10 +12,11 @@ import java.util.function.Predicate; import java.util.stream.Stream; -/** Collection of all the {@link io.stargate.sgv2.jsonapi.testbench.testspec.SpecFile} we have loaded +/** + * Collection of all the {@link io.stargate.sgv2.jsonapi.testbench.testspec.SpecFile} we have loaded * from disk. - *

- * Call {@link #loadAll(List)} to load spec files from multiple directories, + * + *

Call {@link #loadAll(List)} to load spec files from multiple directories, */ public class SpecFiles { @@ -35,33 +36,30 @@ private SpecFiles(List specFiles) { } } - /** - * Loads all the spec file, paths is a list of resource dirs in the jar. - */ + /** Loads all the spec file, paths is a list of resource dirs in the jar. */ public static SpecFiles loadAll(List paths) { var specFiles = resourceDirs(paths).flatMap(SpecFiles::loadAll).toList(); return new SpecFiles(specFiles); } - /** - * Get all the SpecFiles by their metadata kind - */ + /** Get all the SpecFiles by their metadata kind */ public Stream byKind(TestSpecKind kind) { return match(kind, x -> true); } /** - * Get all the SpecFiles by the class of the Specification, the object that - * implements {@link TestSpec} + * Get all the SpecFiles by the class of the Specification, the object that implements {@link + * TestSpec} */ public Stream byType(Class clazz) { return match(TestSpecKind.fromType(clazz), x -> true) - .map(specFile -> specFile.spec().asSpecType(clazz)); + .map(specFile -> specFile.spec().asSpecType(clazz)); } /** - * Get all the spec files of the type matched by name, e.g. get all the test-suites called "monkey" + * Get all the spec files of the type matched by name, e.g. get all the test-suites called + * "monkey" */ public Stream byNameAsType(Class clazz, String name) { return match(TestSpecKind.fromType(clazz), specFiles -> specFiles.meta().name().equals(name)) @@ -70,13 +68,11 @@ public Stream byNameAsType(Class clazz, String name) private Stream match(TestSpecKind kind, Predicate predicate) { return specFiles.stream() - .filter(itFile -> itFile.spec().meta().kind() == kind) - .filter(specFile -> predicate.test(specFile.spec())); + .filter(itFile -> itFile.spec().meta().kind() == kind) + .filter(specFile -> predicate.test(specFile.spec())); } - /** - * Load all the spec files in the directory at the path - */ + /** Load all the spec files in the directory at the path */ private static Stream loadAll(Path path) { try (Stream pathStream = Files.walk(path)) { @@ -95,10 +91,7 @@ private static boolean isJsonFile(Path file) { return file.getFileName().toString().endsWith(".json"); } - - /** - * Load a single spec file denoted by path. - */ + /** Load a single spec file denoted by path. */ private static SpecFile loadOne(Path path) { var file = path.toFile(); try { @@ -123,11 +116,9 @@ private static SpecFile loadOne(Path path) { } } - public static Stream resourceDirs(List paths) { - return paths.stream() - .map(SpecFiles::resourceDir); + return paths.stream().map(SpecFiles::resourceDir); } public static Path resourceDir(String path) { @@ -146,8 +137,7 @@ public static Path resourceDir(String path) { // walking. return Paths.get(url.toURI()); } catch (URISyntaxException e) { - throw new IllegalArgumentException( - "Bad resource URI for: " + path + " -> " + url, e); + throw new IllegalArgumentException("Bad resource URI for: " + path + " -> " + url, e); } } } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/TargetsSpec.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/TargetsSpec.java index 72bfbf201f..9f2b64c9f4 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/TargetsSpec.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/TargetsSpec.java @@ -3,16 +3,13 @@ import com.fasterxml.jackson.databind.MapperFeature; import com.fasterxml.jackson.databind.ObjectMapper; import io.stargate.sgv2.jsonapi.testbench.targets.TargetConfiguration; - import java.io.IOException; import java.nio.file.Path; import java.util.HashSet; import java.util.List; import java.util.Set; -/** - * Spec file that contains targets - */ +/** Spec file that contains targets */ public record TargetsSpec(TestSpecMeta meta, List targets) implements TestSpec { @@ -34,7 +31,8 @@ public TargetConfiguration getTarget(String targetName) { return targets.stream() .filter(target -> target.name().equals(targetName)) .findFirst() - .orElseThrow(() -> new IllegalArgumentException("target targetName not found: " + targetName)); + .orElseThrow( + () -> new IllegalArgumentException("target targetName not found: " + targetName)); } public static TargetsSpec loadAll(String path) { @@ -46,5 +44,4 @@ public static TargetsSpec loadAll(String path) { throw new RuntimeException(e); } } - } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/TestCase.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/TestCase.java index 46591b8881..00059dfeb4 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/TestCase.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/TestCase.java @@ -7,7 +7,9 @@ import org.junit.jupiter.api.DynamicContainer; /** - * Spec for a single test case in a test suite, a test case is a command to run and assertions to run after. + * Spec for a single test case in a test suite, a test case is a command to run and assertions to + * run after. + * * @param name * @param command * @param asserts @@ -20,7 +22,10 @@ public record TestCase( @JsonProperty("$include") String include) { public DynamicContainer testNodesForEnvironment( - TestNodeFactory testNodeFactory, TestUri.Builder uriBuilder, TestRunEnv testEnvironment, TestExecutionCondition testExecutionCondition) { + TestNodeFactory testNodeFactory, + TestUri.Builder uriBuilder, + TestRunEnv testEnvironment, + TestExecutionCondition testExecutionCondition) { var testRequest = new TestRunRequest( diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/TestCommand.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/TestCommand.java index 7ac9419186..56b5e76a69 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/TestCommand.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/TestCommand.java @@ -11,12 +11,13 @@ import org.apache.commons.text.StringSubstitutor; /** - * A API command to send, that may be part of a setup, test, cleanup, or lifecycle process. This is a - * basic command to send, without any checks etc. - *

- * This class is re-used in Spec configurations where an API command is needed. For example below, the TestCommand - * is the Objects in the setup array. The structure is then a normal Data API command object, with a single top level - * member that is the name of the command. + * A API command to send, that may be part of a setup, test, cleanup, or lifecycle process. This is + * a basic command to send, without any checks etc. + * + *

This class is re-used in Spec configurations where an API command is needed. For example + * below, the TestCommand is the Objects in the setup array. The structure is then a normal Data API + * command object, with a single top level member that is the name of the command. + * * 

  *   "setup": [
  *     {
@@ -36,7 +37,6 @@
  *       "insertMany": {
  *         "documents": [....
  *
- *

*/ public class TestCommand { @@ -48,6 +48,7 @@ public class TestCommand { /** * Constructor called when this class is used in a record etc that is deserialized using Jackson. + * * @param request The JSON object to deserialize from, e.g. the objects in the array above. */ @JsonCreator(mode = JsonCreator.Mode.DELEGATING) @@ -66,17 +67,13 @@ public TestCommand(ObjectNode request) { } } - /** - * Get the complete request to send - */ + /** Get the complete request to send */ public ObjectNode request() { throwIfHasInclude(); return request; } - /** - * The name of the Data API command, extracted from the definition. - */ + /** The name of the Data API command, extracted from the definition. */ public CommandName commandName() { throwIfHasInclude(); return commandName; @@ -84,8 +81,8 @@ public CommandName commandName() { /** * Name of the test-suite to include command from, rather than use the definition in here. - *

- * For example, this says to include setup commands from 'vectorize-base' + * + *

For example, this says to include setup commands from 'vectorize-base' * * 

    *   "setup": [
@@ -94,16 +91,14 @@ public CommandName commandName() {
    *     }
    *   ],
    *
- *

+ * * @return value of the '$include' from the json */ public String includeFrom() { return includeFrom; } - /** - * Build from a string, useful for lifecycle commands that are created on the fly. - */ + /** Build from a string, useful for lifecycle commands that are created on the fly. */ public static TestCommand fromJson(String json) { try { @@ -115,9 +110,8 @@ public static TestCommand fromJson(String json) { /** * Extracts the name of the API command from the request. - *

- * Public for re-use. - *

+ * + *

Public for re-use. */ public static CommandName commandName(ObjectNode request) { var requestCommandName = commandNameString(request); @@ -153,7 +147,6 @@ private static String commandNameString(ObjectNode request) { return name; } - private static void walk(ObjectNode obj, StringSubstitutor subs) { obj.properties() .forEach( diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/TestSpec.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/TestSpec.java index ed8f81874d..8b9a07a44a 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/TestSpec.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/TestSpec.java @@ -4,10 +4,9 @@ /** * A specification for objects in the TestBench world, such as a test suite. - *

- * Implement this for any object types a {@link TestBenchPlan} needs - * to read from disk or know about. - *

+ * + *

Implement this for any object types a {@link TestBenchPlan} needs to read from disk or know + * about. */ public sealed interface TestSpec permits AssertionTemplateSpec, TargetsSpec, TestSuiteSpec, WorkflowSpec { @@ -15,7 +14,8 @@ public sealed interface TestSpec TestSpecMeta meta(); /** - * Gets the object as the implementing class + * Gets the object as the implementing class + * * @param type the implementing class of the {@link TestSpec} * @return The implementing object, as the type. * @param the implementing class of the {@link TestSpec} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/TestSpecMeta.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/TestSpecMeta.java index cfa5f3c453..2aad34d5e9 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/TestSpecMeta.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/TestSpecMeta.java @@ -4,13 +4,14 @@ /** * Common metadata for any object of a {@link TestSpec} + * * @param name Friendly name for, such as "vectorize-header-workflow" * @param kind {@link TestSpecKind} describing the kind. * @param tags Optional tags that can be used for filtering etc. */ public record TestSpecMeta(String name, TestSpecKind kind, List tags) { - public TestSpecMeta{ - tags = tags == null ? List.of() : tags; - } + public TestSpecMeta { + tags = tags == null ? List.of() : tags; + } } diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/TestSuiteSpec.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/TestSuiteSpec.java index 3c0961bd3c..b877493a09 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/TestSuiteSpec.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/TestSuiteSpec.java @@ -1,13 +1,9 @@ package io.stargate.sgv2.jsonapi.testbench.testspec; - - import io.stargate.sgv2.jsonapi.testbench.assertions.TestAssertion; - +import io.stargate.sgv2.jsonapi.testbench.testrun.*; import java.util.ArrayList; import java.util.List; - -import io.stargate.sgv2.jsonapi.testbench.testrun.*; import org.junit.jupiter.api.DynamicContainer; import org.junit.jupiter.api.DynamicNode; @@ -16,22 +12,23 @@ public record TestSuiteSpec( implements TestSpec { public DynamicContainer testNode( - TestNodeFactory testNodeFactory, TestUri.Builder uriBuilder, List allEnvs) { + TestNodeFactory testNodeFactory, TestUri.Builder uriBuilder, List allEnvs) { uriBuilder.addSegment(TestUri.Segment.SUITE, meta().name()); var desc = "TestSuite: %s ".formatted(meta.name()); - var childs = allEnvs.stream() + var childs = + allEnvs.stream() .map(testEnv -> testEnv.testNode(testNodeFactory, uriBuilder.clone(), this)) .toList(); - return testNodeFactory.testPlanContainer( - desc, - uriBuilder.build().uri(), - childs); + return testNodeFactory.testPlanContainer(desc, uriBuilder.build().uri(), childs); } public List testNodesForEnvironment( - TestNodeFactory testNodeFactory, TestUri.Builder uriBuilder, TestRunEnv testEnvironment, TestExecutionCondition testExecutionCondition) { + TestNodeFactory testNodeFactory, + TestUri.Builder uriBuilder, + TestRunEnv testEnvironment, + TestExecutionCondition testExecutionCondition) { // not increasing the count of test nodes here, because this code is not actually making any // test nodes, it is all in things we call, they do the increasing @@ -45,7 +42,7 @@ public List testNodesForEnvironment( new TestRunRequest( "SetupRequest[%s]: %s".formatted(i++, setupCommand.commandName()), setupCommand, - testNodeFactory.testPlan().target(), + testNodeFactory.testPlan().target(), testEnvironment, TestAssertion.forSuccess(testNodeFactory.testPlan(), setupCommand), testExecutionCondition); @@ -56,18 +53,22 @@ public List testNodesForEnvironment( var testUriBuilder = uriBuilder.clone().addSegment(TestUri.Segment.STAGE, "test"); for (var testCase : tests()) { nodes.add( - testCase.testNodesForEnvironment(testNodeFactory, testUriBuilder.clone(), testEnvironment, testExecutionCondition)); + testCase.testNodesForEnvironment( + testNodeFactory, testUriBuilder.clone(), testEnvironment, testExecutionCondition)); } - // NOTE: For Cleanup we use a condition that is always TRUE because we always want to try to run a cleanup task. - var alwaysTrueCondition = new TestExecutionCondition.AlwaysTrue("Cleanup Commands for parent URL: " + uriBuilder.build().uri().toString()); + // NOTE: For Cleanup we use a condition that is always TRUE because we always want to try to run + // a cleanup task. + var alwaysTrueCondition = + new TestExecutionCondition.AlwaysTrue( + "Cleanup Commands for parent URL: " + uriBuilder.build().uri().toString()); var cleanupUriBuilder = uriBuilder.clone().addSegment(TestUri.Segment.STAGE, "cleanup"); for (TestCommand cleanupCommand : cleanup()) { var cleanupRequest = new TestRunRequest( "CleanupRequest[%s]: %s".formatted(i++, cleanupCommand.commandName()), cleanupCommand, - testNodeFactory.testPlan().target(), + testNodeFactory.testPlan().target(), testEnvironment, TestAssertion.forSuccess(testNodeFactory.testPlan(), cleanupCommand), alwaysTrueCondition); diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/WorkflowSpec.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/WorkflowSpec.java index 79a24b8b5f..be35a7fba8 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/WorkflowSpec.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/WorkflowSpec.java @@ -1,17 +1,14 @@ package io.stargate.sgv2.jsonapi.testbench.testspec; - - import io.stargate.sgv2.jsonapi.testbench.testrun.TestNodeFactory; import io.stargate.sgv2.jsonapi.testbench.testrun.TestUri; import java.util.List; - import org.junit.jupiter.api.DynamicNode; public record WorkflowSpec(TestSpecMeta meta, List jobs) implements TestSpec { public DynamicNode testNode( - TestNodeFactory testNodeFactory, TestUri.Builder uriBuilder, boolean ignoreDisabled) { + TestNodeFactory testNodeFactory, TestUri.Builder uriBuilder, boolean ignoreDisabled) { uriBuilder.addSegment(TestUri.Segment.WORKFLOW, meta().name()); var desc = "Workflow: %s ".formatted(meta.name()); @@ -20,11 +17,12 @@ public DynamicNode testNode( ignoreDisabled ? jobs().stream().filter(job -> !job.meta().tags().contains("disabled")) : jobs().stream(); - var jobNodes = testNodeJobs - .map(job -> job.testNode(testNodeFactory, uriBuilder.clone())) - .toList(); + var jobNodes = + testNodeJobs.map(job -> job.testNode(testNodeFactory, uriBuilder.clone())).toList(); return testNodeFactory.testPlanContainer( - desc, uriBuilder.build().uri(), testNodeFactory.addLifecycle(uriBuilder.clone(), this, jobNodes)); + desc, + uriBuilder.build().uri(), + testNodeFactory.addLifecycle(uriBuilder.clone(), this, jobNodes)); } }