diff --git a/.github/workflows/test-bench-vectorize.yaml b/.github/workflows/test-bench-vectorize.yaml new file mode 100644 index 0000000000..176bf2ac86 --- /dev/null +++ b/.github/workflows/test-bench-vectorize.yaml @@ -0,0 +1,158 @@ +name: Test Bench - Vectorize Run + +on: + workflow_dispatch: + inputs: + environment: + description: 'Environment to test against' + required: true + type: environment + pull_request: + +# needed when a workflow wants to use OIDC (OpenID Connect) to authenticate to cloud +permissions: + id-token: write + contents: write + packages: write + pull-requests: write + +# global env vars, available in all jobs and steps +env: + MAVEN_OPTS: '-Xmx4g' + DS_ARTIFACTORY_USERNAME: ${{ secrets.DS_ARTIFACTORY_USERNAME }} + DS_ARTIFACTORY_PASSWORD: ${{ secrets.DS_ARTIFACTORY_PASSWORD }} + +jobs: + setup: + runs-on: ubuntu-latest + environment: ${{ github.event.inputs.environment || 'PROD' }} + + outputs: + matrix: ${{ steps.set-matrix.outputs.TEST_TARGETS }} + steps: + - id: set-matrix + env: + TEST_TARGETS: ${{ vars.TEST_TARGETS }} + run: | + { + echo "TEST_TARGETS<> "$GITHUB_OUTPUT" + + - id: write-setup-summary + name: Write setup summary + env: + TEST_TARGETS: ${{ vars.TEST_TARGETS }} + run: | + FORMATTED_TARGETS=$(echo "$TEST_TARGETS" | jq '.') + + cat >> $GITHUB_STEP_SUMMARY << 'EOF' + ## Test Run Setup + + **Environment:** ${{ github.event.inputs.environment || 'PROD' }} + **Triggered by:** ${{ github.actor }} + + ### Test Targets + EOF + + echo '```json' >> $GITHUB_STEP_SUMMARY + echo "$FORMATTED_TARGETS" >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + + # runs unit tests + build: + name: Test Bench- ENV-${{ github.event.inputs.environment || 'PROD' }} Target- ${{ matrix.name }} + needs: setup + runs-on: ubuntu-latest + environment: ${{ github.event.inputs.environment || 'PROD' }} + + timeout-minutes: 120 + + strategy: + # do not fail fast, we want to run on all the different target dbs + fail-fast: false + matrix: + include: ${{ fromJson(needs.setup.outputs.matrix) }} + + steps: + - uses: actions/checkout@v6 + + - name: Set up JDK 21 + uses: actions/setup-java@v5 + with: + distribution: 'temurin' + java-version: '21' + cache: maven + + - name: Setup Maven + run: | + mkdir -p ~/.m2 + cat < ~/.m2/settings.xml + + + + stargate-central + ${DS_ARTIFACTORY_USERNAME} + ${DS_ARTIFACTORY_PASSWORD} + + + stargate-snapshots + ${DS_ARTIFACTORY_USERNAME} + ${DS_ARTIFACTORY_PASSWORD} + + + artifactory + ${DS_ARTIFACTORY_USERNAME} + ${DS_ARTIFACTORY_PASSWORD} + + + artifactory-snapshots + ${DS_ARTIFACTORY_USERNAME} + ${DS_ARTIFACTORY_PASSWORD} + + + artifactory-releases + ${DS_ARTIFACTORY_USERNAME} + ${DS_ARTIFACTORY_PASSWORD} + + + + EOF + + - name: Set env + # Sanitize the name of the target, it could be things like "smoketest-prod-gcp-us-east4 (astra-serverless-prod-49)" + # TEST_BENCH_REPORT path to where the summary report will be written so we can include in the GH summary + run: | + SAFE_NAME=$(echo "${{ matrix.name }}" | sed 's/[^a-zA-Z0-9._-]/_/g') + echo "TEST_BENCH_REPORT=test-bench-report-${SAFE_NAME}.md" >> $GITHUB_ENV + + - name: Build & Test + env: + TEST_PLAN_FILE: classpath:testbench/testplans/test-plan-astra-vectorize.yaml + DATA_API_TOKEN: ${{ secrets.DATA_API_TOKEN }} + TARGET_NAME: ${{ matrix.name }} + ENDPOINT: ${{ matrix.endpoint }} + HUGGINGFACE_KEY: ${{ secrets.HUGGINGFACE_KEY }} + JINA_AI_KEY: ${{ secrets.JINA_AI_KEY }} + MISTRAL_KEY: ${{ secrets.MISTRAL_KEY }} + OPEN_AI_KEY: ${{ secrets.OPEN_AI_KEY }} + UPSTAGE_AI_KEY: ${{ secrets.UPSTAGE_AI_KEY }} + VOYAGE_AI_KEY: ${{ secrets.VOYAGE_AI_KEY }} + + # reducing extra output with the -Dfailsafe.printSummary=false -Dfailsafe.trimStackTrace=true + # -Dorg.fusesource.jansi.Ansi.disable=true - disable ansi because GH action summary does not handle + run: | + ./mvnw -B -ntp verify \ + -Dfmt.skip \ + -DskipUnitTests \ + -Dorg.fusesource.jansi.Ansi.disable=true \ + -Dfailsafe.printSummary=false \ + -Dfailsafe.trimStackTrace=true \ + -Dit.test=TestBenchByTestPlan \ + -Dtest-bench-report-path=$TEST_BENCH_REPORT + + - name: Write summary + if: always() + run: | + cat "$TEST_BENCH_REPORT" >> $GITHUB_STEP_SUMMARY \ No newline at end of file diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000000..048f05f30e --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,116 @@ +# Agent Guidelines + +## Code Comment Standards + +### Purpose of Comments + +Comments should explain **WHY** code exists or **WHY** a particular approach was chosen, NOT simply describe **WHAT** the code does. + +### What to Avoid + +- Redundant comments that restate obvious code behavior +- Comments that merely describe what a line or block of code does +- Stating the obvious (e.g., `// increment counter` above `counter++`) + +### What to Include + +- **Intent and reasoning**: Why this approach was chosen over alternatives +- **Business logic context**: Why certain rules or constraints exist +- **Edge cases**: Non-obvious scenarios the code handles +- **Non-obvious behavior**: Subtle interactions or side effects +- **Workarounds**: Why a particular workaround was necessary +- **Performance considerations**: Why certain optimizations were made +- **Security implications**: Why certain checks or patterns are used + +## Documentation Hierarchy + +### Class-Level Documentation + +Class-level Javadoc should provide comprehensive context and explain the design intent. Include: + +- **Purpose**: What problem this class/interface solves +- **Design rationale**: Why this approach was chosen +- **Usage context**: How and when this should be used +- **Constraints and rules**: What implementations should NOT do +- **Integration points**: How this fits into the larger system + +**Avoid repeating this information at the method level.** + +### Method-Level Documentation + +Documentation requirements vary based on the method's visibility and purpose: + +#### Public API, Interface, and Abstract Methods + +These require formal Javadoc with: +- **When the method is called**: Brief statement of why this should be called and what it does +- **Parameter documentation**: Standard `@param` tags for all parameters +- **Return value**: Standard `@return` tag + +**Do NOT repeat class-level context or add implementation examples at the method level.** + +#### Private Methods + +Private implementation methods typically only need a brief comment explaining **WHY** the method exists or **WHY** a particular approach was taken. Formal `@param` and `@return` tags are not required unless: +- The method is complex with non-obvious parameter usage +- The method has subtle behavior that needs explanation + +**Example (simple private method):** +```java +// Normalize user input to prevent injection attacks +private String sanitizeInput(String raw) { + return raw.replaceAll("[^a-zA-Z0-9]", ""); +} +``` + +**Example (complex private method needing params):** +```java +/** + * Merges overlapping time ranges to optimize query performance. + * Adjacent ranges within the tolerance window are combined to reduce + * the number of database queries. + * + * @param ranges List of time ranges, may contain overlaps + * @param toleranceMs Milliseconds of gap allowed between ranges to still merge them + * @return Consolidated list with overlapping ranges merged + */ +private List mergeTimeRanges(List ranges, long toleranceMs) { +``` + +### Documentation Example: Interface + +**Class-level (comprehensive):** +```java +/** + * Defines the stages in the lifecycle of a test bench run. + *

+ * Designed to be implemented by a Backend so that it can make changes + * to the data environment so tests can run in a common environment. + * For example, when we use Cassandra as a backend we need to create + * a keyspace but for Astra we use the default one. + *

+ *

+ * There should not be any test logic within the implementations, + * that should all be in the test definitions. + *

+ */ +public interface TestPlanLifecycle { +``` + +**Method-level (minimal):** +```java + /** + * Called to optionally add a node to execute before the workflow starts. + * + * @param testNodeFactory Factory to use to create test nodes + * @param uriBuilder Builder to use to create URIs + * @param workflow The workflow about to execute + * @return Optional DynamicNode to run before the workflow + */ + default Optional beforeWorkflow(...) { +``` + +**What to avoid at method level:** +- Repeating "useful for cleanup" or "allows setting up resources" (already covered at class level) +- Adding specific implementation examples (violates the "no test logic" constraint stated at class level) +- Restating the overall purpose of the interface \ No newline at end of file diff --git a/pom.xml b/pom.xml index a0ec524a08..c7fd42d4e7 100644 --- a/pom.xml +++ b/pom.xml @@ -54,11 +54,7 @@ false ${skipTests} - - 3.5.3 + 3.5.5 false @@ -102,6 +98,11 @@ + + com.jayway.jsonpath + json-path + 2.10.0 + io.quarkus quarkus-arc @@ -345,6 +346,18 @@ quarkus-mcp-server-test test + + org.apache.maven.plugins + maven-surefire-plugin + ${surefire-plugin.version} + test + + + me.fabriciorby + maven-surefire-junit5-tree-reporter + 1.5.1 + test + @@ -466,6 +479,7 @@ verify + ${argLine} diff --git a/src/main/resources/application.yaml b/src/main/resources/application.yaml index 22bf894d36..ee34c95d0b 100644 --- a/src/main/resources/application.yaml +++ b/src/main/resources/application.yaml @@ -16,7 +16,7 @@ stargate: database: limits: - max-collections: 5 + max-collections: 15 debug: enabled: false diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/ResponseAssertions.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/ResponseAssertions.java index 7176eb94c2..ae02dd0235 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/ResponseAssertions.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/ResponseAssertions.java @@ -87,7 +87,7 @@ public class ResponseAssertions { FieldMatcher.errors(hasErrors)); final String msg = - "%s: Response fields %s:%s, %s:%s, %s:%s" + "%s: %s:%s, %s:%s, %s:%s" .formatted( message, Presence.REQUIRED, diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/TestBenchByTestPlan.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/TestBenchByTestPlan.java new file mode 100644 index 0000000000..e7516743b8 --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/TestBenchByTestPlan.java @@ -0,0 +1,51 @@ +package io.stargate.sgv2.jsonapi.testbench; + +import io.stargate.sgv2.jsonapi.testbench.testspec.SpecFiles; +import java.nio.file.Path; +import java.util.stream.Stream; +import org.junit.jupiter.api.DynamicNode; +import org.junit.jupiter.api.TestFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Entry point for running a Test Bench from a Test Plan file. + * + *

Put the name of the test plan file in the TEST_PLAN_FILE env var, this can set + * the target to hit, and the workflows to run. See {@link TestPlanFile} + *

+ *

+ * This will look like a unit test, so we only run when the env var is set. + *

+ */ +public class TestBenchByTestPlan { + + private static final Logger LOGGER = LoggerFactory.getLogger(TestBenchByTestPlan.class); + + @TestFactory + public Stream runTestPlanFile() { + + var rawPath = System.getenv("TEST_PLAN_FILE"); + if (rawPath == null) { + return Stream.empty(); + } + LOGGER.info("runTestPlanFile() - getting TEST_PLAN_FILE from ENV, rawPath={}", rawPath); + + var path = + rawPath.startsWith("classpath:") + ? SpecFiles.resourceDir(rawPath.substring("classpath:".length())) + : Path.of(rawPath); + + var testPlan = TestBenchPlan.fromFile(path); + LOGGER.info("runTestPlanFile() - building test plan tree"); + var testPlanNodeTree = testPlan.testNode(); + + LOGGER.info( + "runTestPlanFile() - test plan tree build, totalNodeCount={}", + testPlanNodeTree.totalNodeCount()); + System.setProperty( + TestBenchPlan.TEST_PLAN_TEST_COUNT_PROPERTY, + String.valueOf(testPlanNodeTree.totalNodeCount())); + return Stream.of(testPlanNodeTree.root()); + } +} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/TestBenchPlan.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/TestBenchPlan.java new file mode 100644 index 0000000000..59012516f7 --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/TestBenchPlan.java @@ -0,0 +1,134 @@ +package io.stargate.sgv2.jsonapi.testbench; + +import com.fasterxml.jackson.core.StreamReadFeature; +import com.fasterxml.jackson.databind.MapperFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; +import io.stargate.sgv2.jsonapi.testbench.lifecycle.JobLifeCycle; +import io.stargate.sgv2.jsonapi.testbench.targets.Target; +import io.stargate.sgv2.jsonapi.testbench.targets.TargetConfiguration; +import io.stargate.sgv2.jsonapi.testbench.testrun.TestNodeFactory; +import io.stargate.sgv2.jsonapi.testbench.testrun.TestUri; +import io.stargate.sgv2.jsonapi.testbench.testspec.*; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.Set; +import java.util.stream.Stream; +import org.apache.commons.text.StringSubstitutor; +import org.junit.jupiter.api.DynamicNode; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * NOTE: called "TestBenchPlan" to avoid collision with "org.junit.platform.launcher.TestPlan" + * + * @param target + * @param specFiles + * @param workflows + * @param ignoreDisabled + */ +public record TestBenchPlan( + Target target, SpecFiles specFiles, Set workflows, boolean ignoreDisabled) + implements JobLifeCycle { + + private static final Logger LOGGER = LoggerFactory.getLogger(TestBenchPlan.class); + + public static final String TEST_PLAN_TEST_COUNT_PROPERTY = "testbench.test.count"; + + private static final ObjectMapper YAML_MAPPER = + new ObjectMapper( + YAMLFactory.builder().enable(StreamReadFeature.INCLUDE_SOURCE_IN_LOCATION).build()) + .configure(MapperFeature.ACCEPT_CASE_INSENSITIVE_ENUMS, true); + + public static TestBenchPlan fromFile(Path path) { + + LOGGER.info("fromFile() - Loading test plan file, path={}", path); + TestPlanFile planFile; + try { + var raw = Files.readString(path); + var substituted = new StringSubstitutor(System.getenv()).replace(raw); + planFile = YAML_MAPPER.readValue(substituted, TestPlanFile.class); + } catch (IOException e) { + throw new RuntimeException(e); + } + + if (planFile.targetName() != null && planFile.customTarget() != null) { + throw new RuntimeException( + "Both targetName and customTarget set, use only one. testPlanFile=" + path); + } + + var testPlan = + planFile.customTarget() != null + ? create(planFile.customTarget(), planFile.workflows(), planFile.ignoreDisabled()) + : create(planFile.targetName(), planFile.workflows(), planFile.ignoreDisabled()); + + return testPlan; + } + + public static TestBenchPlan create(String targetName, List workflows) { + return create(targetName, workflows, true); + } + + public static TestBenchPlan create( + String targetName, List workflows, Boolean ignoreDisabled) { + var targetConfigs = TargetsSpec.loadAll("testbench/targets/targets.json"); + return create(targetConfigs.getTarget(targetName), workflows, ignoreDisabled); + } + + public static TestBenchPlan create( + TargetConfiguration targetConfiguration, List workflows, Boolean ignoreDisabled) { + var target = new Target(targetConfiguration); + + var specFiles = + SpecFiles.loadAll( + List.of("testbench/assertions", "testbench/testsuites", "testbench/workflows")); + + return new TestBenchPlan( + target, + specFiles, + workflows == null ? Set.of() : Set.copyOf(workflows), + ignoreDisabled == null || ignoreDisabled); + } + + public Stream selectedWorkflows() { + + return specFiles + .byKind(TestSpecKind.WORKFLOW) + .filter( + specFile -> workflows.isEmpty() || workflows.contains(specFile.spec().meta().name())) + .map(testSpec -> (WorkflowSpec) testSpec.spec()); + } + + public TestPlanNodeTree testNode() { + + var desc = + "TestPlan: %s on %s" + .formatted(target.configuration().name(), target.configuration().backend()); + + var uriBuilder = + TestUri.builder(TestUri.Scheme.DATAAPI) + .addSegment(TestUri.Segment.TARGET, target.configuration().name()); + + var testNodeFactory = new TestNodeFactory(this); + + var root = + testNodeFactory.testPlanContainer( + desc, + uriBuilder.build().uri(), + selectedWorkflows() + .map( + workflow -> + workflow.testNode(testNodeFactory, uriBuilder.clone(), ignoreDisabled)) + .toList()); + return new TestPlanNodeTree(root, testNodeFactory.testNodeCount()); + } + + @Override + public void updateJobForTarget(Job job) { + target.updateJobForTarget(job); + } + + public record TestPlanNodeTree(DynamicNode root, int totalNodeCount) {} +} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/TestPlanFile.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/TestPlanFile.java new file mode 100644 index 0000000000..9ba1b37df9 --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/TestPlanFile.java @@ -0,0 +1,49 @@ +package io.stargate.sgv2.jsonapi.testbench; + +import io.stargate.sgv2.jsonapi.testbench.targets.TargetConfiguration; +import java.nio.file.Path; +import java.util.List; + +/** + * Structure of a test plane file stored in a jar resource or external file. These are used with the + * {@link TestBenchByTestPlan} runner to control the target and workflows to run at execution time + * via a config file. + * + *

For example testbench/testplans/test-plan-astra-vectorize.yaml, testing against + * and Astra DB: + * + * 

+ * name: Test Plan - Astra - Vectorize Workflows
+ * customTarget:
+ *   name: ${TARGET_NAME}
+ *   backend: astra
+ *   connection:
+ *     domain: ${ENDPOINT}
+ *     port: 443
+ *     basePath: /api/json/v1
+ * workflows:
+ *   - vectorize-header-workflow
+ *   - vectorize-shared-workflow
+ * ignoreDisabled: true
+ *
+ * + *

The {@link TestBenchPlan#fromFile(Path)} will run Apache command style substituions using + * {@link System#getenv(String)} as the source for replacements. This allows some sensitive + * information to be put into the env rather than a file, and for the runner to make multiple calls + * with different env vars. + * + * @param name Nice human name for the test plan, only used in this file + * @param targetName Name of the target, such as the db name, used in logging etc. + * @param customTarget Defines a {@link TargetConfiguration} of how to connect (e.g. astra or + * cassandra backend ) and the connection information. + * @param workflows List of the workflows to run, leave empty or null to run all workflows in the + * system. + * @param ignoreDisabled If true, work flow jobs marked as "disabled" will be executed. Default is + * false. + */ +public record TestPlanFile( + String name, + String targetName, + TargetConfiguration customTarget, + List workflows, + Boolean ignoreDisabled) {} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/AssertionFactory.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/AssertionFactory.java new file mode 100644 index 0000000000..dc1932db3a --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/AssertionFactory.java @@ -0,0 +1,199 @@ +package io.stargate.sgv2.jsonapi.testbench.assertions; + +import com.fasterxml.jackson.databind.JsonNode; +import io.stargate.sgv2.jsonapi.testbench.TestBenchPlan; +import io.stargate.sgv2.jsonapi.testbench.testspec.TestCommand; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.lang.reflect.Modifier; +import java.util.List; + +/** + * Describes functions that can be used to create instances of an assertion, and finds the factory + */ +public sealed interface AssertionFactory { + + /** + * Registry of the factories that can be called to create assertions, see {@link + * AssertionFactoryRegistry} + */ + AssertionFactoryRegistry REGISTRY = new AssertionFactoryRegistry(); + + /** + * A factory that returns a single AssertionMatcher, this is a single match of the response. + * + *

This is what we use with basic assertions like: + * + * 

+   *     {
+   *        "Status.isExactly" : {
+   *           "matchedCount": 1,
+   *           "modifiedCount": 1
+   *         }
+   *     }
+   *
+ */ + @FunctionalInterface + non-sealed interface AssertionMatcherFactory extends AssertionFactory { + /** + * Create an assertion matcher that can be used to match the response. + * + * @param testCommand The command the assertion will be run against. + * @param args The arguments defined in the test suite, e.g. the number of documents in a + * collection. + * @return AssertionMatcher that can be used to match the response. + */ + AssertionMatcher create(TestCommand testCommand, JsonNode args); + } + + /** + * A factory that returns a list of assertions, this is a list of matches of the response. Used + * with templated assertions. + * + *

This returns the {@link TestAssertion} which is higher up the stack than the {@link + * AssertionMatcher} because a templated assertion is a list of assertions and only the template + * factory knows how to describe them because it makes them. + */ + @FunctionalInterface + non-sealed interface TemplatedAssertionFactory extends AssertionFactory { + + /** + * Create a list of assertions that can be used to match the response. + * + * @param testPlan The test plan that is being created holds context of what we are doing. + * @param template JSON pulled from the {@link + * io.stargate.sgv2.jsonapi.testbench.testspec.TestSpecKind#ASSERTION_TEMPLATE} that matched + * the assertion name. + * @param testCommand The command the assertion will be run against. + * @param args The arguments defined in the test suite, e.g. the number of documents in a + * collection. + * @return + */ + List create( + TestBenchPlan testPlan, JsonNode template, TestCommand testCommand, JsonNode args); + } + + /** Returns true if the function matches *either* of the assertion factory functions. */ + static boolean isValidFactoryMethod(Method method) { + + // must be static + if (!Modifier.isStatic(method.getModifiers())) { + return false; + } + + // Checking for TemplatedAssertionFactory + // NOTE: not checked the generic type of the list, lazy and this is a contrained world + if (List.class.isAssignableFrom(method.getReturnType())) { + var p = method.getParameterTypes(); + return p.length == 4 + && p[0] == TestBenchPlan.class + && p[1] == JsonNode.class + && p[2] == TestCommand.class + && p[3] == JsonNode.class; + } + + // Checking for AssertionMatcherFactory + if (method.getReturnType() == AssertionMatcher.class) { + var p = method.getParameterTypes(); + return p.length == 2 && p[0] == TestCommand.class && p[1] == JsonNode.class; + } + return false; + } + + /** + * There are two situations we handle the factory methods with: first when registering them so we + * know what we can call, second when processing a test suite config we need to call them to get + * the assertion. + * + *

For the first part, the factory methods are defined as static functions on a class, and + * there can be two different types of factory. This class wraps the raw factory, so we have a + * common class we can use when registering all the factories we know. It also constructs a {@link + * AssertionName} for the java function that can be used to match against the name used in the + * test suite config. + * + *

For the second path the subclasses provide an adapter functionality: they implement the + * factory interface so we get strong type checking for calling, and then transform that call into + * an untyped call to the raw factory method via {@link Method#invoke(Object, Object...)}. + * + *

NOTE: Construct instances using {@link WrappedMethod#of(Class, Method)} + */ + abstract sealed class WrappedMethod + permits WrappedAssertionMatcherFactory, WrappedTemplatedAssertionFactory { + + private final Class clazz; + private final Method method; + private final AssertionName assertionName; + + protected WrappedMethod(Class clazz, Method method) { + this.clazz = clazz; + this.method = method; + this.assertionName = new AssertionName(clazz.getSimpleName(), method.getName()); + } + + static WrappedMethod of(Class clazz, Method method) { + + // a little sloppy, but we already know this should be a factory method + return (method.getReturnType() == AssertionMatcher.class) + ? new WrappedAssertionMatcherFactory(clazz, method) + : new WrappedTemplatedAssertionFactory(clazz, method); + } + + /** Gets the name of the factory function, without the class name. */ + public String properName() { + return AssertionName.properName(method); + } + + /** + * The identifier for the factory function that can be used to match agains the name used in the + * test suite config. e.g. "Documents.count" + */ + public AssertionName assertionName() { + return assertionName; + } + + @SuppressWarnings("unchecked") + protected T invoke(Object... args) { + try { + // pass null for the object reference, all factories are static + return (T) method.invoke(null, args); + } catch (IllegalAccessException | InvocationTargetException e) { + throw new RuntimeException(e); + } + } + } + + /** + * Adapter class for the {@link AssertionMatcherFactory}, translates strong typed called to the + * factory method into untypes method invocation. + */ + final class WrappedAssertionMatcherFactory extends WrappedMethod + implements AssertionMatcherFactory { + + WrappedAssertionMatcherFactory(Class clazz, Method method) { + super(clazz, method); + } + + @Override + public AssertionMatcher create(TestCommand testCommand, JsonNode args) { + return invoke(testCommand, args); + } + } + + /** + * Adapter class for the {@link TemplatedAssertionFactory}, translates strong typed called to the + * factory method into untypes method invocation. + */ + final class WrappedTemplatedAssertionFactory extends WrappedMethod + implements TemplatedAssertionFactory { + + WrappedTemplatedAssertionFactory(Class clazz, Method method) { + super(clazz, method); + } + + @Override + public List create( + TestBenchPlan testPlan, JsonNode template, TestCommand testCommand, JsonNode args) { + return invoke(testPlan, template, testCommand, args); + } + } +} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/AssertionFactoryRegistry.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/AssertionFactoryRegistry.java new file mode 100644 index 0000000000..5b8620c122 --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/AssertionFactoryRegistry.java @@ -0,0 +1,85 @@ +package io.stargate.sgv2.jsonapi.testbench.assertions; + +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +/** + * Registry of assertion factories that is built by the static constructors of the assertion + * classes registering themselves. + * + *

Every assertion class must register itself with the registry, so we know what factories exist + * and can match them at test execution time. Do that in a static constructor as below, the regsitry + * will walk the class and registry functions that match either of the two {@link AssertionFactory} + * interfaces. + * + * 

+ * public class Documents {
+ *
+ *   static {
+ *     AssertionFactory.REGISTRY.register(Documents.class);
+ *   }
+ *
+ * + *

Use the singleton instance at {@link AssertionFactory#REGISTRY} to access the registry. + */ +public class AssertionFactoryRegistry { + + // Map of the factory functions and the name they should be matched to in test suite config + private final Map factoryMethods = + new ConcurrentHashMap<>(); + + /** + * Registers all the static factory functions on the class that match the signature of either + * {@link AssertionFactory} interface. + */ + public void register(Class cls) { + + for (var method : cls.getMethods()) { + if (AssertionFactory.isValidFactoryMethod(method)) { + var wrapped = AssertionFactory.WrappedMethod.of(cls, method); + factoryMethods.put(wrapped.assertionName(), wrapped); + } + } + } + + /** + * Gets the factory function for the assertion with the given name. + * + * @param rawAssertionName the string name of the assertion from the config, e.g. + * "Documents.count" + * @return A wrapper for the factory function that can be used to create an assertion instance. + */ + public AssertionFactory.WrappedMethod getWrappedAssertionFactory(String rawAssertionName) { + + // need to take the raw name from the config into the internal representation that + // matches to what we used in the registry + var assertionName = AssertionName.from(rawAssertionName); + + var factoryMethod = factoryMethods.get(assertionName); + if (factoryMethod == null) { + // sometimes the class is not loaded yet, so let's just give nature a helping hand. Shhh + loadClassFor(assertionName); + } + + factoryMethod = factoryMethods.get(assertionName); + if (factoryMethod == null) { + throw new IllegalArgumentException( + "Unknown assertion factory. (parsed) assertionName: %s factoryMethods.keySet:%s" + .formatted(assertionName, factoryMethods.keySet())); + } + return factoryMethod; + } + + private void loadClassFor(AssertionName normalisedName) { + + try { + Class.forName(normalisedName.properClassName()); + // class static initializer should call register() + } catch (ClassNotFoundException e) { + throw new IllegalArgumentException( + "Unknown assertion factory. normalisedName=%s, properClassName()=%s" + .formatted(normalisedName, normalisedName.properClassName()), + e); + } + } +} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/AssertionMatcher.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/AssertionMatcher.java new file mode 100644 index 0000000000..09e738b478 --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/AssertionMatcher.java @@ -0,0 +1,20 @@ +package io.stargate.sgv2.jsonapi.testbench.assertions; + +import io.stargate.sgv2.jsonapi.testbench.messaging.APIResponse; + +/** + * Contract for matching the result of an API call to an assertion. + * + *

This is the raw function to do the work, without any descriptive elements around it. + */ +@FunctionalInterface +public interface AssertionMatcher { + + /** + * Match the response to the assertion. + * + * @param apiResponse response from the API + * @throws AssertionError if the match fails + */ + void match(APIResponse apiResponse); +} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/AssertionName.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/AssertionName.java new file mode 100644 index 0000000000..883db5ed08 --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/AssertionName.java @@ -0,0 +1,49 @@ +package io.stargate.sgv2.jsonapi.testbench.assertions; + +import java.lang.reflect.Method; + +/** + * A normalized way to represent an assertion name, used in both directions: used when registering + * assertion factories, so we map from the Java method name to this; and then to parse the assertion + * name from the test spec the user provided. + * + *

Once in this common middle ground, we can map between test spec and java method names. + * NOTE: the matching is case-insensitive, and we normalize to lower case. So "isSuccess" and + * "issuccess" are the same. + * + * @param typeName Class name of the method, or the first part of the assertion config e.g. + * "Documents" + * @param funcName The name of the method, or the second part of the assertion config e.g. "count" + */ +public record AssertionName(String typeName, String funcName) { + + // A bit of a hack, buit this is used when we force the static load. see below. + private static final String PACKAGE = "io.stargate.sgv2.jsonapi.testbench.assertions"; + + public AssertionName { + // we normalize to match on case-insensitive + typeName = typeName.toLowerCase(); + funcName = funcName.toLowerCase(); + } + + /** Create from the string name provided in the test spec config, e.g. "Documents.count" */ + public static AssertionName from(String fullKey) { + + int pos = fullKey.indexOf('.'); + if (pos < 0) { + throw new IllegalArgumentException("fullKey must have a dot: " + fullKey); + } + var type = fullKey.substring(0, pos); + var func = fullKey.substring(pos + 1); + return new AssertionName(type, func); + } + + /** Gets the name of the factory function, without the class name. */ + public static String properName(Method method) { + return method.getName(); + } + + public String properClassName() { + return PACKAGE + "." + Character.toUpperCase(typeName.charAt(0)) + typeName.substring(1); + } +} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/BodyAssertion.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/BodyAssertion.java new file mode 100644 index 0000000000..07e51fbcaf --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/BodyAssertion.java @@ -0,0 +1,44 @@ +package io.stargate.sgv2.jsonapi.testbench.assertions; + +import io.stargate.sgv2.jsonapi.testbench.messaging.APIResponse; +import org.hamcrest.Matcher; +import org.hamcrest.StringDescription; + +/** + * Assertions that check the body of the response using a {@link Matcher} from hamcrest. + * + *

Example: + * + * 

+ *   return new BodyAssertion("data.documents", hasSize(expectedCount));
+ *
+ * + *

So this is a very re-usable record, as most assertions we want to create run a matcher against + * the body of the response by calling {@link + * io.restassured.response.ValidatableResponse#body(String, Matcher)}. This record makes it easy to + * make a matcher from hamcrest, work out the description, and plug it into the {@link + * TestAssertion} structure so we can run it in the dynamic tests we build + * + * @param bodyPath Path to the body to check, e.g. "data.documents" + * @param matcher Matcher to use to check the body, e.g. hasSize(expectedCount) + */ +public record BodyAssertion(String bodyPath, Matcher matcher) + implements Describable, AssertionMatcher { + + @Override + public void match(APIResponse apiResponse) { + apiResponse.validatable().body(bodyPath(), matcher()); + } + + /** Describes the assertion, based on the path and the matcher. */ + @Override + public String describe() { + var describable = new StringDescription(); + // get hamcrest to describe the matcher it will run + matcher.describeTo(describable); + + // caller should truncate if it wants to limit it + // example: "body('data.documents') - a collection with size <3>" + return "body('%s') - %s".formatted(bodyPath(), describable.toString()); + } +} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/Describable.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/Describable.java new file mode 100644 index 0000000000..68f1730582 --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/Describable.java @@ -0,0 +1,11 @@ +package io.stargate.sgv2.jsonapi.testbench.assertions; + +/** + * Functional interface to call so an assertion can describe itself outside of its string + * representation. Typically based on how the assertion was described in the test configuration. + */ +@FunctionalInterface +public interface Describable { + + String describe(); +} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/DescribableAssertionMatcher.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/DescribableAssertionMatcher.java new file mode 100644 index 0000000000..38025cf47f --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/DescribableAssertionMatcher.java @@ -0,0 +1,32 @@ +package io.stargate.sgv2.jsonapi.testbench.assertions; + +import io.stargate.sgv2.jsonapi.testbench.messaging.APIResponse; +import org.jspecify.annotations.NonNull; + +/** + * Hold an assertion matcher and a description for it, used with very simple assertions that are + * just a function. + */ +public record DescribableAssertionMatcher(String description, AssertionMatcher matcher) + implements Describable, AssertionMatcher { + + public static DescribableAssertionMatcher described( + String description, AssertionMatcher matcher) { + return new DescribableAssertionMatcher(description, matcher); + } + + @Override + public void match(APIResponse apiResponse) { + matcher.match(apiResponse); + } + + @Override + public String describe() { + return toString(); + } + + @Override + public @NonNull String toString() { + return description(); + } +} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/Documents.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/Documents.java new file mode 100644 index 0000000000..6050bcf350 --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/Documents.java @@ -0,0 +1,38 @@ +package io.stargate.sgv2.jsonapi.testbench.assertions; + +import static net.javacrumbs.jsonunit.JsonMatchers.jsonEquals; +import static org.hamcrest.Matchers.hasSize; + +import com.fasterxml.jackson.databind.JsonNode; +import io.stargate.sgv2.jsonapi.testbench.testspec.TestCommand; + +/** + * Assertions that check the `document` or `documents` in the `data` field of the API result. + * + *

See {@link TestAssertion} + */ +public class Documents { + + static { + AssertionFactory.REGISTRY.register(Documents.class); + } + + /** + * Check the count of the `data.documents` array. + * + *

Assertion factory, see {@link AssertionFactory.AssertionMatcherFactory} + */ + public static AssertionMatcher count(TestCommand testCommand, JsonNode args) { + var expectedCount = args.asInt(); + return new BodyAssertion("data.documents", hasSize(expectedCount)); + } + + /** + * Check that the `data.document` matches the given JSON. + * + *

Assertion factory, see {@link AssertionFactory.AssertionMatcherFactory} + */ + public static AssertionMatcher isExactly(TestCommand testCommand, JsonNode args) { + return new BodyAssertion("data.document", jsonEquals(args)); + } +} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/Http.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/Http.java new file mode 100644 index 0000000000..2d99e7be3c --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/Http.java @@ -0,0 +1,22 @@ +package io.stargate.sgv2.jsonapi.testbench.assertions; + +import static io.stargate.sgv2.jsonapi.testbench.assertions.DescribableAssertionMatcher.described; + +import com.fasterxml.jackson.databind.JsonNode; +import io.stargate.sgv2.jsonapi.testbench.testspec.TestCommand; +import org.apache.http.HttpStatus; + +/** Assertions that check the structure of the HTTP Response code */ +public class Http { + + static { + AssertionFactory.REGISTRY.register(Http.class); + } + + /** Assertion factory, see {@link AssertionFactory.AssertionMatcherFactory} */ + public static AssertionMatcher success(TestCommand testCommand, JsonNode args) { + return described( + "http status is " + HttpStatus.SC_OK, + apiResponse -> apiResponse.validatable().statusCode(HttpStatus.SC_OK)); + } +} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/Response.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/Response.java new file mode 100644 index 0000000000..b2b4ef4db4 --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/Response.java @@ -0,0 +1,55 @@ +package io.stargate.sgv2.jsonapi.testbench.assertions; + +import static io.stargate.sgv2.jsonapi.api.v1.ResponseAssertions.*; + +import com.fasterxml.jackson.databind.JsonNode; +import io.stargate.sgv2.jsonapi.testbench.testspec.TestCommand; + +/** Assertions that check the structure of the API response, e.g. should it have a `data` field */ +public class Response { + + static { + AssertionFactory.REGISTRY.register(Response.class); + } + + /** + * Checks the HTTP status AND the shape of the response doc + * + *

Assertion factory, see {@link AssertionFactory.AssertionMatcherFactory} + */ + public static AssertionMatcher isSuccess(TestCommand testCommand, JsonNode args) { + + var commandName = testCommand.commandName(); + return switch (commandName.getCommandType()) { + case DDL -> isDDLSuccess(testCommand, args); + case DML -> + switch (commandName) { + case FIND_ONE, FIND -> isFindSuccess(testCommand, args); + case FIND_ONE_AND_DELETE, FIND_ONE_AND_REPLACE, FIND_ONE_AND_UPDATE -> + isFindAndSuccess(testCommand, args); + case INSERT_ONE, INSERT_MANY -> Response.isWriteSuccess(testCommand, args); + default -> + throw new IllegalStateException( + "No isSuccess mapping for command name: " + commandName); + }; + case ADMIN -> + throw new IllegalStateException("No isSuccess mapping for command name: " + commandName); + }; + } + + public static AssertionMatcher isFindSuccess(TestCommand testCommand, JsonNode args) { + return new BodyAssertion("$", responseIsFindSuccess()); + } + + public static AssertionMatcher isFindAndSuccess(TestCommand testCommand, JsonNode args) { + return new BodyAssertion("$", responseIsFindAndSuccess()); + } + + public static AssertionMatcher isWriteSuccess(TestCommand testCommand, JsonNode args) { + return new BodyAssertion("$", responseIsWriteSuccess()); + } + + public static AssertionMatcher isDDLSuccess(TestCommand testCommand, JsonNode args) { + return new BodyAssertion("$", responseIsDDLSuccess()); + } +} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/SingleTestAssertion.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/SingleTestAssertion.java new file mode 100644 index 0000000000..da1371ca9d --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/SingleTestAssertion.java @@ -0,0 +1,48 @@ +package io.stargate.sgv2.jsonapi.testbench.assertions; + +import com.fasterxml.jackson.databind.JsonNode; +import io.stargate.sgv2.jsonapi.testbench.testrun.*; +import java.util.concurrent.atomic.AtomicReference; +import org.junit.jupiter.api.DynamicNode; + +/** + * A single test assertion that performs a single test of the results of a request. + * + * @param name Name of the assertion, e.g. "Documents.count" + * @param args Raw arguments passed into the assertion factory, from the test case definition + * @param matcher The actual logic that will be run to assert the result of the request. + */ +public record SingleTestAssertion(String name, JsonNode args, AssertionMatcher matcher) + implements TestAssertion { + + @Override + public void run(TestRunResponse testResponse) { + // exceptions can bubble out, that's how the frameworks know the assertion result. + matcher.match(testResponse.apiResponse()); + } + + @Override + public DynamicNode testNodes( + TestNodeFactory testNodeFactory, + TestUri.Builder uriBuilder, + AtomicReference testResponse, + TestExecutionCondition testExecutionCondition) { + + var matcherDesc = (matcher instanceof Describable d) ? d.describe() : ""; + + var executable = + new DynamicTestExecutable( + "%s [%s]".formatted(name(), matcherDesc), + uriBuilder.addSegment(TestUri.Segment.ASSERTION, name()), + testExecutionCondition, + () -> { + var resp = testResponse.get(); + if (resp == null) { + throw new IllegalStateException("Response is null"); + } + run(resp); + }); + + return executable.testNode(testNodeFactory); + } +} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/Status.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/Status.java new file mode 100644 index 0000000000..8f1f3364fb --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/Status.java @@ -0,0 +1,19 @@ +package io.stargate.sgv2.jsonapi.testbench.assertions; + +import static net.javacrumbs.jsonunit.JsonMatchers.jsonEquals; + +import com.fasterxml.jackson.databind.JsonNode; +import io.stargate.sgv2.jsonapi.testbench.testspec.TestCommand; + +/** Assertions that match Data API status top level response field. */ +public class Status { + + static { + AssertionFactory.REGISTRY.register(Status.class); + } + + /** Assertion factory, see {@link AssertionFactory.AssertionMatcherFactory} */ + public static AssertionMatcher isExactly(TestCommand testCommand, JsonNode args) { + return new BodyAssertion("status", jsonEquals(args)); + } +} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/Templated.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/Templated.java new file mode 100644 index 0000000000..e80700e1b6 --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/Templated.java @@ -0,0 +1,73 @@ +package io.stargate.sgv2.jsonapi.testbench.assertions; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import io.stargate.sgv2.jsonapi.testbench.TestBenchPlan; +import io.stargate.sgv2.jsonapi.testbench.testspec.TestCommand; +import java.util.List; + +/** + * Assertions that are defined by a JSON template. + * + *

There are two parts to using a template, first the template must be defined in a {@link + * io.stargate.sgv2.jsonapi.testbench.testspec.TestSpecKind#ASSERTION_TEMPLATE} such as + * + * 

+ * {
+ *   "meta": {
+ *     "name": "assertions-templates",
+ *     "kind": "assertion_template"
+ *   },
+ *   "templates": {
+ *     "isSuccess": {
+ *       "createCollection": {
+ *         "http.success": null,
+ *         "response.isDDLSuccess": null
+ *       },
+ *       "createKeyspace": {
+ *         "http.success": null,
+ *         "response.isDDLSuccess": null
+ *       }
+ *   }
+ * }
+ *
+ * + * NOTE: The format of a template is fixed: the members under "templates" as the names of the + * template, the value is a JSON object that is passed to its factory below. Each template can then + * have its own style of definition. + * + *

Strongly encouraged to use the same structure as isSucces: whose members are the names of API + * commands, and those values are the assertions that should be run as you would define them + * normally. We use this structure because the idea of a template like "isSuccess" is that it is + * saying "whatever the API command we just ran it should be successful". + * + *

So, every template name from a config file like above must have a function here to create + */ +public class Templated { + + static { + AssertionFactory.REGISTRY.register(Templated.class); + } + + /** Assertion factory, see {@link AssertionFactory.TemplatedAssertionFactory} */ + public static List isSuccess( + TestBenchPlan testPlan, JsonNode template, TestCommand testCommand, JsonNode args) { + + var commandTemplate = template.get(testCommand.commandName().getApiName()); + if (commandTemplate == null) { + throw new IllegalArgumentException( + "isSuccess Assertion template not found for command: " + + testCommand.commandName().getApiName()); + } + return runTemplate(testPlan, (ObjectNode) commandTemplate, testCommand, args); + } + + private static List runTemplate( + TestBenchPlan testPlan, ObjectNode template, TestCommand testCommand, JsonNode args) { + + return template.properties().stream() + .map(entry -> new TestAssertion.AssertionDefinition(entry.getKey(), entry.getValue())) + .map(def -> TestAssertion.buildAssertion(testPlan, testCommand, def)) + .toList(); + } +} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/TestAssertion.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/TestAssertion.java new file mode 100644 index 0000000000..7ed0fac282 --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/TestAssertion.java @@ -0,0 +1,226 @@ +package io.stargate.sgv2.jsonapi.testbench.assertions; + +import com.fasterxml.jackson.databind.JsonNode; +import io.stargate.sgv2.jsonapi.testbench.TestBenchPlan; +import io.stargate.sgv2.jsonapi.testbench.testrun.*; +import io.stargate.sgv2.jsonapi.testbench.testspec.AssertionTemplateSpec; +import io.stargate.sgv2.jsonapi.testbench.testspec.TestCase; +import io.stargate.sgv2.jsonapi.testbench.testspec.TestCommand; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.Stream; +import org.junit.jupiter.api.DynamicNode; + +/** + * A single assertion to run on the result of a single request sent to the API. This is the + * assertion with name, description etc, and the logic of the matcher that will perform the test. + * See also {@link AssertionMatcher} + * + *

Assertions are defined in code, and then linked to the setup, test, or cleanup request in the + * test-suite. Or in the case of Target lifecycle (such as creating a keyspace) created in code + * entirely. + * + *

For example, this TestCase has two assertions. One is templated, that is made up of multiple + * other assertions, and the other is a simple matcher. + * + * 

+ *     {
+ *       "name": "basic findMany",
+ *       "command": {
+ *         "find": {
+ *           "sort": {
+ *             "$vectorize": "I love movies!"
+ *           }
+ *         }
+ *       },
+ *       "asserts": {
+ *         "Templated.isSuccess": null,
+ *         "Documents.count": 3
+ *       }
+ *     }
+ *
+ * + *

NOTE: The name of the assertion, e.g. "Documents.count" MUST to the name of a + * "Class.Method" in the "assertions" package. For example see {@link Documents#count(TestCommand, + * JsonNode)} and the {@link AssertionFactory} for the registry of assertions. The assertion object + * for a particular assertion, that is an instance with the configuration from above, is created in + * {@link AssertionDefWithFactory#build(TestBenchPlan, TestCommand)} + */ +public sealed interface TestAssertion permits SingleTestAssertion, TestAssertionContainer { + + /** Friendly name used in logs and reports. */ + String name(); + + /** + * Arguments that may be passed to the assertion from the test suite definition. For example, text + * node 3 will be passed to "Documents.count" in the above. Used for reporting / + * logging. + */ + JsonNode args(); + + /** + * Called for the assertion to run against the response of running the API request. + * + *

Three things can happen: + * + *

    + *
  1. Everything is OK, returns without exception. + *
  2. The assertion fails, throws an exception like a normal Junit test, and it will be + * recorded as a failure. + *
  3. Throw a {@link org.junit.AssumptionViolatedException} to say the assumption was not meet + * and the test is aborted. This is normally done before the assertion is called so we do + * not send requests after one has failed in the test env, see {@link + * io.stargate.sgv2.jsonapi.testbench.testrun.DynamicTestExecutable}. + *
+ * + * @param testResponse + */ + void run(TestRunResponse testResponse); + + /** + * Gets {@link DynamicNode} that represents this assertion in the test tree. + * + *

An assertion is always a single test node in the test tree, not a container. See {@link + * SingleTestAssertion} for the implementation for a single assertion. Where we have a group of + * assertions, they end up going to that class as well to get the test nodes for the individual + * assertions. + * + * @param testNodeFactory Factory to build test nodes with, for common naming etc. + * @param uriBuilder Builder for the URI tp descrbie the type of node in the test tree. + * @param testResponse Atomic reference that will be updated with the response of the test + * request, that the assertion will use to perform to rune once it is time to execute. + * @param testExecutionCondition Condition that can be checked to see if the test node should be + * skipped. + * @return DynamicNode representing this assertion in the test tree, may be a container if there + * are multiple assertions. + */ + DynamicNode testNodes( + TestNodeFactory testNodeFactory, + TestUri.Builder uriBuilder, + AtomicReference testResponse, + TestExecutionCondition testExecutionCondition); + + /** + * Returns a list of assertions that can be used to determine if a command was successful. + * + *

Uses the Templated.isSuccess templated assertion. + */ + static List forSuccess(TestBenchPlan testPlan, TestCommand testCommand) { + + var builder = + Stream.builder() + .add(new AssertionDefinition("Templated.isSuccess", null)); + + return buildAssertions(testPlan, testCommand, builder.build()); + } + + /** + * Creates assertions based on the configuration of the {@link TestCase}, that is what is under + * the "asserts" member in the example above. + */ + static List buildAssertions(TestBenchPlan testPlan, TestCase testCase) { + + var defs = testCase.asserts().properties().stream().map(AssertionDefinition::create); + return buildAssertions(testPlan, testCase.command(), defs); + } + + static List buildAssertions( + TestBenchPlan testPlan, TestCommand testCommand, Stream defs) { + + return defs.map(def -> buildAssertion(testPlan, testCommand, def)).toList(); + } + + static TestAssertion buildAssertion( + TestBenchPlan testPlan, TestCommand testCommand, AssertionDefinition def) { + return def.addFactory(AssertionFactory.REGISTRY).build(testPlan, testCommand); + } + + /** + * Definition of an assertion that can be used to create an instance of the assertion used that + * can be run later. + * + *

Example, the member and it's value in below: + * + * 

+   *     {
+   *         "Documents.count": 3
+   *     }
+   *
+ * + * @param name Name of the assertion, it *must* map to a class and method in the "assertions" + * package. + * @param args Arguments that will be passed to the assertion factory, see {@link + * AssertionFactory} + */ + record AssertionDefinition(String name, JsonNode args) { + + static AssertionDefinition create(Map.Entry def) { + return new AssertionDefinition(def.getKey(), def.getValue()); + } + + AssertionDefWithFactory addFactory(AssertionFactoryRegistry registry) { + + var factory = registry.getWrappedAssertionFactory(name()); + if (factory == null) { + throw new IllegalStateException("Unknown Assertion Factory name=" + name()); + } + return new AssertionDefWithFactory(factory, args); + } + } + + /** + * Definition of an assertion where we have the factory function that can create it with the + * arguments supplied by the test definition. + */ + record AssertionDefWithFactory(AssertionFactory.WrappedMethod method, JsonNode args) { + + /** + * Create an assertion instance by calling the appropriate factory function with the arguments + * from the test definition. + * + * @param testPlan the test plan that is being created holds context of what we are doing. + * @param testCommand The actual command that will be executed, that we will want to assert the + * result of + * @return A single assertion that can be run later. + */ + TestAssertion build(TestBenchPlan testPlan, TestCommand testCommand) { + + return switch (method) { + // basic single assertion + case AssertionFactory.WrappedAssertionMatcherFactory factory -> + new SingleTestAssertion( + method.properName(), args(), factory.create(testCommand, args())); + + // templated, we need to look up what assertions should be in it. + case AssertionFactory.TemplatedAssertionFactory factory -> { + // search all assertion templates to find any that have the name given in the test + // definition, there + // must be one and only one + var template = + testPlan + .specFiles() + .byType(AssertionTemplateSpec.class) + .flatMap( + assertTemplate -> assertTemplate.templateFor(method.properName()).stream()) + .reduce( + (a, b) -> { + throw new IllegalStateException( + "Multiple Assertion Templates found for name=" + method.properName()); + }) + .orElseThrow( + () -> + new IllegalStateException( + "Unknown Assertion Template name=" + method.properName())); + + // templated assertions have children, so we need an assertion that contains those + // children. + // the factory we call here will use the template to make AssertionDefinition 's and end + // up back in this method to make the childred + yield new TestAssertionContainer( + method.properName(), args(), factory.create(testPlan, template, testCommand, args())); + } + }; + } + } +} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/TestAssertionContainer.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/TestAssertionContainer.java new file mode 100644 index 0000000000..cb29c46ab4 --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/assertions/TestAssertionContainer.java @@ -0,0 +1,49 @@ +package io.stargate.sgv2.jsonapi.testbench.assertions; + +import com.fasterxml.jackson.databind.JsonNode; +import io.stargate.sgv2.jsonapi.testbench.testrun.TestExecutionCondition; +import io.stargate.sgv2.jsonapi.testbench.testrun.TestNodeFactory; +import io.stargate.sgv2.jsonapi.testbench.testrun.TestRunResponse; +import io.stargate.sgv2.jsonapi.testbench.testrun.TestUri; +import java.util.List; +import java.util.concurrent.atomic.AtomicReference; +import org.junit.jupiter.api.DynamicNode; + +/** + * An assertion made up of child assertions, for example built from a template that created multiple + * assertions + * + * @param name Name of the template, e.g. "Templated.isSuccess" + * @param args Raw arguments passed into the template factory, from the test case definition + * @param assertions Child assertions that were created by the template factory + */ +public record TestAssertionContainer(String name, JsonNode args, List assertions) + implements TestAssertion { + + @Override + public void run(TestRunResponse testResponse) { + + // we are just a container, let those exceptions bubble up + assertions.forEach(assertion -> assertion.run(testResponse)); + } + + @Override + public DynamicNode testNodes( + TestNodeFactory testNodeFactory, + TestUri.Builder uriBuilder, + AtomicReference testResponse, + TestExecutionCondition testExecutionCondition) { + + uriBuilder.addSegment(TestUri.Segment.ASSERTION, name()); + + var children = + assertions.stream() + .map( + assertion -> + assertion.testNodes( + testNodeFactory, uriBuilder.clone(), testResponse, testExecutionCondition)) + .toList(); + + return testNodeFactory.testPlanContainer(name, uriBuilder.build().uri(), children); + } +} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/lifecycle/JobLifeCycle.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/lifecycle/JobLifeCycle.java new file mode 100644 index 0000000000..e2bd83a8f1 --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/lifecycle/JobLifeCycle.java @@ -0,0 +1,24 @@ +package io.stargate.sgv2.jsonapi.testbench.lifecycle; + +import io.stargate.sgv2.jsonapi.testbench.testspec.Job; + +/** + * Defines the stages a {link @Job} may go through as it is executing. + * + *

It's limited now to just the one method, may never get bigger, but there are a couple of + * implementations so extracted. + */ +public interface JobLifeCycle { + + /** + * Called to update the Job with any information it needs to run against the particular target. + * + *

Typcially this means setting or updating the {@link Job#variables()} for things like a + * default keyspace name or implementing naming restrictions. + * + *

NOTE: The target or Backend must set the KEYSPACE_NAME job variable + * + * @param job The job to update + */ + void updateJobForTarget(Job job); +} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/lifecycle/TestPlanLifecycle.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/lifecycle/TestPlanLifecycle.java new file mode 100644 index 0000000000..77c67c6aac --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/lifecycle/TestPlanLifecycle.java @@ -0,0 +1,119 @@ +package io.stargate.sgv2.jsonapi.testbench.lifecycle; + +import io.stargate.sgv2.jsonapi.testbench.testrun.TestNodeFactory; +import io.stargate.sgv2.jsonapi.testbench.testrun.TestRunEnv; +import io.stargate.sgv2.jsonapi.testbench.testrun.TestUri; +import io.stargate.sgv2.jsonapi.testbench.testspec.Job; +import io.stargate.sgv2.jsonapi.testbench.testspec.TestSuiteSpec; +import io.stargate.sgv2.jsonapi.testbench.testspec.WorkflowSpec; +import java.util.Optional; +import org.junit.jupiter.api.DynamicNode; + +/** + * Defines the stages in the lifecycle of a test bench run. + * + *

Design to be implemented by a {@link io.stargate.sgv2.jsonapi.testbench.targets.Backend} so + * that it can make changes to the data environement so tests can run in a common environement. For + * example, when we use Cassandra as a backend we need to create a keyspace but for Astra we use the + * default one. + * + *

There should not be any test logic within the implementations, that should all be in the test + * defintoions. + * + *

Extracted to an interface so it can be reused, such as at the higher level {@link + * io.stargate.sgv2.jsonapi.testbench.targets.Target} which includes a backend. + * + *

All methods allow the implementation to return a JUNIT {@link DynamicNode} which will be + * inserted before or after the dynamic nodes at that level. The returned node could be a single + * {@link org.junit.jupiter.api.DynamicTest} or a {@link org.junit.jupiter.api.DynamicContainer}. + */ +public interface TestPlanLifecycle { + + /** + * Called to optionally add a node to execute before the start of the workflow. + * + * @param testNodeFactory Factory to use to create test nodes, see {@link TestNodeFactory} + * @param uriBuilder Builder to use to create the URI's added to dynamic nodes. + * @param workflow The workflow we are running. + * @return Optional {@link DynamicNode} to run before the workflow. + */ + default Optional beforeWorkflow( + TestNodeFactory testNodeFactory, TestUri.Builder uriBuilder, WorkflowSpec workflow) { + return Optional.empty(); + } + + /** + * Called to optionally add a node to execute after the workflow completes. + * + * @param testNodeFactory Factory to use to create test nodes, see {@link TestNodeFactory} + * @param uriBuilder Builder to use to create the URI's added to dynamic nodes. + * @param workflow The workflow that just completed. + * @return Optional {@link DynamicNode} to run after the workflow. + */ + default Optional afterWorkflow( + TestNodeFactory testNodeFactory, TestUri.Builder uriBuilder, WorkflowSpec workflow) { + return Optional.empty(); + } + + /** + * Called to optionally add a node to execute before a job starts. + * + * @param testNodeFactory Factory to use to create test nodes, see {@link TestNodeFactory} + * @param uriBuilder Builder to use to create the URI's added to dynamic nodes. + * @param job The job about to execute. + * @return Optional {@link DynamicNode} to run before the job. + */ + default Optional beforeJob( + TestNodeFactory testNodeFactory, TestUri.Builder uriBuilder, Job job) { + return Optional.empty(); + } + + /** + * Called to optionally add a node to execute after a job completes. + * + * @param testNodeFactory Factory to use to create test nodes, see {@link TestNodeFactory} + * @param uriBuilder Builder to use to create the URI's added to dynamic nodes. + * @param job The job that just completed. + * @return Optional {@link DynamicNode} to run after the job. + */ + default Optional afterJob( + TestNodeFactory testNodeFactory, TestUri.Builder uriBuilder, Job job) { + return Optional.empty(); + } + + /** + * Called to optionally add a node to execute before a test suite runs. + * + * @param testNodeFactory Factory to use to create test nodes, see {@link TestNodeFactory} + * @param uriBuilder Builder to use to create the URI's added to dynamic nodes. + * @param test The test suite about to execute. + * @param env The runtime environment for this test, including resolved variables and + * configuration. + * @return Optional {@link DynamicNode} to run before the test suite. + */ + default Optional beforeTestSuite( + TestNodeFactory testNodeFactory, + TestUri.Builder uriBuilder, + TestSuiteSpec test, + TestRunEnv env) { + return Optional.empty(); + } + + /** + * Called to optionally add a node to execute after a test suite completes. + * + * @param testNodeFactory Factory to use to create test nodes, see {@link TestNodeFactory} + * @param uriBuilder Builder to use to create the URI's added to dynamic nodes. + * @param test The test suite that just completed. + * @param env The runtime environment for this test, including resolved variables and + * configuration. + * @return Optional {@link DynamicNode} to run after the test suite. + */ + default Optional afterTestSuite( + TestNodeFactory testNodeFactory, + TestUri.Builder uriBuilder, + TestSuiteSpec test, + TestRunEnv env) { + return Optional.empty(); + } +} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/messaging/APIRequest.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/messaging/APIRequest.java new file mode 100644 index 0000000000..eb175c7a5e --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/messaging/APIRequest.java @@ -0,0 +1,161 @@ +package io.stargate.sgv2.jsonapi.testbench.messaging; + +import static io.restassured.RestAssured.given; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.google.common.base.Strings; +import io.restassured.http.ContentType; +import io.restassured.response.Response; +import io.restassured.response.ValidatableResponse; +import io.restassured.specification.RequestSpecification; +import io.stargate.sgv2.jsonapi.api.model.command.CommandTarget; +import io.stargate.sgv2.jsonapi.config.constants.HttpConstants; +import io.stargate.sgv2.jsonapi.testbench.targets.ConnectionConfiguration; +import io.stargate.sgv2.jsonapi.testbench.testrun.TestRunEnv; +import io.stargate.sgv2.jsonapi.testbench.testspec.TestCommand; +import java.util.HashMap; +import java.util.Map; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * The lowest level to represent a request sent to the API, that is only concerned with the + * mechanics of sending the request. + * + *

Handles retries based on detecting substrings in the response body, these occur before {@link + * #execute()} returns. + */ +public class APIRequest { + private static final Logger LOGGER = LoggerFactory.getLogger(APIRequest.class); + + public static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + + private static int RETRY_MAX_ATTEMPTS = 5; + + // API paths based on the target of the command. + private static String COLLECTION_TABLE_PATH = "/{keyspace}/{collection}"; + private static String KEYSPACE_PATH = "/{keyspace}"; + private static String DB_PATH = "/"; + + private final ConnectionConfiguration connection; + private final TestRunEnv testRunEnv; + private final ObjectNode request; + private final APIRetryPolicy retryPolicy; + + /** + * Initializes a new instance of the class. + * + * @param connection Connection info for the API instance to use. + * @param testRunEnv Environment for this test run, used to find schema names to use in the URL + * path. + * @param request the complete API request to send, NOTE: any substitutions into the body + * of the request must also be done. + */ + public APIRequest(ConnectionConfiguration connection, TestRunEnv testRunEnv, ObjectNode request) { + + this.connection = connection; + this.testRunEnv = testRunEnv; + this.request = request; + this.retryPolicy = APIRetryPolicy.createRetryPolicy(testRunEnv); + } + + /** + * Executes the request, including any retries. + * + *

No validation of the response is performed, that is left for assertions to handle later. + * + * @return {@link APIResponse} holding the response of the request. + */ + public APIResponse execute() { + + ValidatableResponse lastValidatableResponse = null; + + var retryDecision = retryPolicy.firstAttempt(); + while (retryDecision.retry()) { + + // Create a new request spec, there is some state that is left in it when a request is run + // for path params + var requestSpec = requestSpec(); + var rawResponse = executeRequest(requestSpec); + lastValidatableResponse = rawResponse.then(); + + // log even if we retry, the request will be logged and it makes sense to see the respose that + // caused the retry + lastValidatableResponse.log().status().and().log().body(); + retryDecision = retryPolicy.decide(retryDecision, rawResponse); + } + return new APIResponse(this, lastValidatableResponse); + } + + /** + * Get the request ready to send our request. + * + * @return + */ + private RequestSpecification requestSpec() { + + String requestString; + try { + requestString = OBJECT_MAPPER.writeValueAsString(request); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + return requestForTaget().headers(getHeaders()).body(requestString).when(); + } + + /** Create a new RequestSpecification for JSON to send to the target */ + private RequestSpecification requestForTaget() { + + return given() + .log() + .uri() + .log() + .body() + .baseUri(connection.domain()) + .port(connection.port()) + .basePath(connection.basePath()) + .contentType(ContentType.JSON); + } + + /** Get the wellknown headers we need to send with the request. */ + protected Map getHeaders() { + + var headers = new HashMap(); + headers.put( + HttpConstants.AUTHENTICATION_TOKEN_HEADER_NAME, + testRunEnv.requiredValue(HttpConstants.AUTHENTICATION_TOKEN_HEADER_NAME)); + + var embeddingApiKey = testRunEnv.get(HttpConstants.EMBEDDING_AUTHENTICATION_TOKEN_HEADER_NAME); + if (!Strings.isNullOrEmpty(embeddingApiKey)) { + headers.put(HttpConstants.EMBEDDING_AUTHENTICATION_TOKEN_HEADER_NAME, embeddingApiKey); + } + return headers; + } + + private Response executeRequest(RequestSpecification requestSpec) { + + var commandName = TestCommand.commandName(request); + Response rawRresponse; + + // URL to send to depends on the target of the command. + if (commandName.getTargets().contains(CommandTarget.COLLECTION) + || commandName.getTargets().contains(CommandTarget.TABLE)) { + rawRresponse = + requestSpec.post( + COLLECTION_TABLE_PATH, + testRunEnv.requiredValue(TestRunEnv.ENV_KEYSPACE_NAME), + testRunEnv.requiredValue(TestRunEnv.ENV_COLLECTION_NAME)); + } else if (commandName.getTargets().contains(CommandTarget.KEYSPACE)) { + rawRresponse = + requestSpec.post(KEYSPACE_PATH, testRunEnv.requiredValue(TestRunEnv.ENV_KEYSPACE_NAME)); + } else if (commandName.getTargets().contains(CommandTarget.DATABASE)) { + rawRresponse = requestSpec.post(DB_PATH); + } else { + throw new IllegalArgumentException("Do not know how to execute command: " + commandName); + } + + return rawRresponse; + } +} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/messaging/APIResponse.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/messaging/APIResponse.java new file mode 100644 index 0000000000..3b1a22225f --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/messaging/APIResponse.java @@ -0,0 +1,6 @@ +package io.stargate.sgv2.jsonapi.testbench.messaging; + +import io.restassured.response.ValidatableResponse; + +/** Basic holder for the response, so we can tie it back to the request that created it */ +public record APIResponse(APIRequest apiRequest, ValidatableResponse validatable) {} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/messaging/APIRetryPolicy.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/messaging/APIRetryPolicy.java new file mode 100644 index 0000000000..233d93c905 --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/messaging/APIRetryPolicy.java @@ -0,0 +1,166 @@ +package io.stargate.sgv2.jsonapi.testbench.messaging; + +import io.restassured.response.Response; +import io.stargate.sgv2.jsonapi.testbench.testrun.TestRunEnv; +import java.util.List; +import java.util.concurrent.ThreadLocalRandom; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** --- */ +interface APIRetryPolicy { + + Logger LOGGER = LoggerFactory.getLogger(APIRetryPolicy.class); + + String RETRY_MATCH_STRING_DELIM = "\t"; + String ENV_RETRY_MATCH_STRING = "RETRY_MATCH_STRING"; + + String DEFAULT_MAX_ATTEMPTS = "5"; + String ENV_RETRY_MAX_ATTEMPTS = "END_RETRY_MAX_ATTEMPTS"; + + String DEFAULT_BASE_SLEEP_MS = "5000"; + String ENV_RETRY_BASE_SLEEP_MS = "RETRY_BASE_SLEEP_MS"; + + String DEFAULT_JITTER_MS = "2000"; + String ENV_RETRY_JITTER_MS = "RETRY_JITTER_MS"; + + /** + * Retries are driven by the presence of the following keys in the {@link TestRunEnv}. When a + * retry is performed the response for that request is lost, the response of the call to + * execute will be the response from the last attempt. + * + *

    + *
  • {@link #ENV_RETRY_MATCH_STRING} Enables retry if present and non blank string, treated as + * a tab {@link #RETRY_MATCH_STRING_DELIM} delimtered string. If any of the tokens is + * present in the response body the request is retried. Example: + * "EMBEDDING_PROVIDER_RATE_LIMITED\tEMBEDDING_PROVIDER_TIMEOUT" + *
  • {@link #ENV_RETRY_MAX_ATTEMPTS} total number of attempts to make before returning, must + * be above 1, default is {@link #ENV_RETRY_MAX_ATTEMPTS} + *
  • {@link #ENV_RETRY_BASE_SLEEP_MS} base number of milliseconds between attempts, this is + * multiplied by 2 ^ attempt, so base of 5000 means sleep of 5, 10, 20 seconds + *
  • {@link #ENV_RETRY_JITTER_MS} Upper bound on the random number of milliseconds to add to + * each attempt. + *
+ */ + static APIRetryPolicy createRetryPolicy(TestRunEnv testRunEnv) { + + var retryMatch = testRunEnv.get(ENV_RETRY_MATCH_STRING); + if (retryMatch == null || retryMatch.isBlank()) { + return new NoAPIRetryPolicy(); + } + + var maxAttempts = + Integer.parseInt(testRunEnv.get(ENV_RETRY_MAX_ATTEMPTS, DEFAULT_MAX_ATTEMPTS)); + var baseSleepMs = + Long.parseLong(testRunEnv.get(ENV_RETRY_BASE_SLEEP_MS, DEFAULT_BASE_SLEEP_MS)); + var jitterMs = Long.parseLong(testRunEnv.get(ENV_RETRY_JITTER_MS, DEFAULT_JITTER_MS)); + + if (LOGGER.isDebugEnabled()) { + LOGGER.debug( + "createRetryPolicy() - retryMatch={}, maxAttempts={}, baseSleepMs={}, jitterMs={}", + retryMatch, + maxAttempts, + baseSleepMs, + jitterMs); + } + + return new ConfiguredAPIRetryPolicy( + List.of(retryMatch.split(RETRY_MATCH_STRING_DELIM)), maxAttempts, baseSleepMs, jitterMs); + } + + /** Call to get the first decision, this is used to count the number of attempts. */ + default RetryDecision firstAttempt() { + return RetryDecision.FIRST_ATTEMPT; + } + + /** + * Call to decide if we should retry or not, implementations will sleep if needed. + * + * @param lastDecision The last decision made, used to count the number of attempts + * @param response The response from the last attempt + * @return A decision to retry or not, along with the attempt count, use for the next call to this + * method. + */ + RetryDecision decide(RetryDecision lastDecision, Response response); + + /** + * A decision to retry or not, along with the attempt count. This is where we count the number of + * attempts. + */ + record RetryDecision(boolean retry, int attempt) { + + static final RetryDecision FIRST_ATTEMPT = new RetryDecision(true, 1); + + RetryDecision stopAttempts() { + // do not increase the attempt count, we won't make another. + return new RetryDecision(false, attempt); + } + } + + /** A retry policy that never retries, makes a single attempt. */ + record NoAPIRetryPolicy() implements APIRetryPolicy { + private static final RetryDecision NO_RETRY = new RetryDecision(false, 1); + + @Override + public RetryDecision decide(RetryDecision lastDecision, Response response) { + return NO_RETRY; + } + } + + /** + * Configurable retry policy with customizable retry conditions, maximum attempts, and backoff + * strategy. + */ + record ConfiguredAPIRetryPolicy( + List retryMatch, int maxAttempts, long baseSleepMs, long jitterMs) + implements APIRetryPolicy { + + public ConfiguredAPIRetryPolicy { + + if (retryMatch == null || retryMatch.isEmpty()) { + throw new IllegalArgumentException("retryMatch is null or empty"); + } + if (maxAttempts < 2) { + throw new IllegalArgumentException( + "maxAttempts must be greater than 1, got: " + maxAttempts); + } + } + + @Override + public RetryDecision decide(RetryDecision lastDecision, Response response) { + + if (lastDecision.attempt == maxAttempts) { + return lastDecision.stopAttempts(); + } + + var body = response.body().asString(); + + for (var match : retryMatch) { + if (body.contains(match)) { + // Service has a concurrency limit and retrying runners can collide regardless of jitter. + // Base backoff is long enough to wait out an in-flight request (5s, 10s, 20s...), + // plus a small random offset to avoid re-synchronising after the wait. + long baseMs = (long) (5000 * Math.pow(2, lastDecision.attempt)); + long jitterMs = ThreadLocalRandom.current().nextLong(2000); + long sleepMs = baseMs + jitterMs; + + LOGGER.info( + "executeRequest() - Retrying, found retry string in response. match={}, sleepMs={} ms, lastDecision.attempt={}", + match, + sleepMs, + lastDecision.attempt); + try { + Thread.sleep(sleepMs); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException("Interrupted during retry sleep", e); + } + + // try again, increase the attempt counter. + return new RetryDecision(true, lastDecision.attempt + 1); + } + } + return lastDecision.stopAttempts(); + } + } +} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/reporting/DynamicTreeListener.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/reporting/DynamicTreeListener.java new file mode 100644 index 0000000000..7e41260395 --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/reporting/DynamicTreeListener.java @@ -0,0 +1,322 @@ +package io.stargate.sgv2.jsonapi.testbench.reporting; + +import io.stargate.sgv2.jsonapi.testbench.TestBenchPlan; +import io.stargate.sgv2.jsonapi.testbench.testrun.TestUri; +import java.util.*; +import java.util.concurrent.ConcurrentHashMap; +import org.junit.platform.engine.TestExecutionResult; +import org.junit.platform.engine.UniqueId; +import org.junit.platform.engine.support.descriptor.UriSource; +import org.junit.platform.launcher.TestExecutionListener; +import org.junit.platform.launcher.TestIdentifier; +import org.junit.platform.launcher.TestPlan; + +/** + * Listens to the execution of the dynammic tests created by the {@link TestPlan} and logs the + * results using the {@link TestBenchConsoleWriter}. + * + *

The class mus tbe registeded by a text file at + * META-INF/services/org.junit.platform.launcher.TestExecutionListener that contains the + * fully qualitified path to the class. + * + *

Type types of output are generated: + * + *

    + *
  • As the tests are running the name of every test node is outputted together with progress, + * so we can see how long there is to go. See {@link TestBenchConsoleWriter#testStarted(int, + * int, TestReportingTracker)}. At this point we do not know how long child nodes will take to + * process and what their result will be. + *
  • Once complet a summary is outputted that does not include every node to brevity, see {@link + * TestBenchConsoleWriter#allTestsFinished(TestReportingTracker)}. At this point we know how + * long child nodes took to process and their result. + *
+ */ +public class DynamicTreeListener implements TestExecutionListener { + + private Integer totalTestCount = null; + private int startedTestCount = 0; + + private TestReportingTracker rootTracker; + // Keyed on TestIdentifier.uniqueID() see {@link TestIdentifier#uniqueId()} + private final Map testTrackers = new ConcurrentHashMap<>(); + + private final TestBenchConsoleWriter writer = new TestBenchConsoleWriter(); + + @Override + public void testPlanExecutionFinished(TestPlan testPlan) { + // All done, write the summary. + writer.allTestsFinished(rootTracker); + } + + @Override + public void executionStarted(TestIdentifier id) { + if (!isTestBenchNode(id)) { + return; + } + + var tracker = getCreateTestTracker(id); + if (tracker == null) { + return; + } + + // Test count will not be in the system properties until we see the first dymamic test node we + // create, e.g. + // "TestPlan: smoketest-aws-us-east-1 on astra workflows vectorize-header-workflow" + // because the nodes will not have been created until then. + if (totalTestCount == null) { + totalTestCount = + Integer.parseInt(System.getProperty(TestBenchPlan.TEST_PLAN_TEST_COUNT_PROPERTY, "0")); + } + + writer.testStarted(totalTestCount, ++startedTestCount, tracker); + } + + @Override + public void executionFinished(TestIdentifier id, TestExecutionResult result) { + var tracker = getCreateTestTracker(id); + if (tracker == null) { + return; + } + + tracker.executionFinished(result); + } + + @Override + public void executionSkipped(TestIdentifier id, String reason) { + // Looks like we never get skipped, included for completeness. + var tracker = getCreateTestTracker(id); + if (tracker == null) { + return; + } + + tracker.executionSkipped(); + } + + /** Determine if tests are running that we should be tracking. */ + private static boolean isTestBenchNode(TestIdentifier testIdentifier) { + + // This is the uniqueID created by jupiter as it is traversing the code, once we get to the + // nodes that are created by the test plan that have different formatting + return testIdentifier + .getUniqueId() + .startsWith("[engine:junit-jupiter]/[class:io.stargate.sgv2.jsonapi.testbench."); + } + + /** + * We use a Tracker for every node in the test plan, to track the execution time and result of it + * and all of its children. + */ + private TestReportingTracker getCreateTestTracker(TestIdentifier testIdentifier) { + + var existingTracker = testTrackers.get(testIdentifier.getUniqueIdObject()); + if (existingTracker != null) { + return existingTracker; + } + + // The getSource() is a URI, jupiter / junit use it to identify the test file, but we dont have + // those. + // We use the {@link TestUri} instead. We need one to know what sort of test node this is + var testUri = + testIdentifier + .getSource() + .map( + testSource -> { + if (testSource instanceof UriSource uriSource) { + return TestUri.parse(uriSource.getUri()).orElse(null); + } + return null; + }); + if (testUri.isEmpty()) { + return null; + } + + // The TARGET is the top level item, everything else should have a parent + var parentTracker = + testUri.get().leafType() != TestUri.Segment.TARGET + ? Objects.requireNonNull( + testTrackers.get(testIdentifier.getParentIdObject().get()), + "parentID not found for testIdentifier: " + testIdentifier) + : null; + + var tracker = new TestReportingTracker(testIdentifier, testUri.get(), parentTracker); + + if (rootTracker == null) { + rootTracker = tracker; + } + testTrackers.put(tracker.identifier.getUniqueIdObject(), tracker); + return tracker; + } + + /** + * Container for tracking the execution of a test, and all of its children. + * + *

--- + */ + public class TestReportingTracker { + + private final TestIdentifier identifier; + private final TestUri runUri; + private final TestReportingTracker parent; + private final int depth; + private final TestContainerStats stats; + + private final List children = new ArrayList<>(); + + // Set when we know the test completed, value on the test result is an optional + private Optional throwable = Optional.empty(); + // Set when we know the test completed + private TestExecutionResult.Status junitStatus; + + public TestReportingTracker( + TestIdentifier identifier, TestUri runUri, TestReportingTracker parent) { + + this.identifier = identifier; + this.runUri = runUri; + this.parent = parent; + this.depth = parent == null ? 0 : parent.depth + 1; + + this.stats = identifier.isContainer() ? new TestContainerStats() : null; + if (parent != null) { + parent.children.add(this); + } + } + + public Optional throwable() { + return throwable; + } + + public TestExecutionResult.Status junitStatus() { + return junitStatus; + } + + public TestIdentifier identifier() { + return identifier; + } + + public TestUri runUri() { + return runUri; + } + + public TestReportingTracker parent() { + return parent; + } + + public List children() { + return Collections.unmodifiableList(children); + } + + public int depth() { + return depth; + } + + public TestContainerStats stats() { + return stats; + } + + /** + * Call when the execution of the test is finished, updates tracking for the node and for any + * ancestors. + * + * @param result + */ + public void executionFinished(TestExecutionResult result) { + junitStatus = result.getStatus(); + throwable = result.getThrowable(); + + if (stats != null) { + stats.testCompleted(this, result); + } + if (parent != null) { + parent.descendantExecutionFinished(this, result); + } + } + + private void descendantExecutionFinished( + TestReportingTracker originalTracker, TestExecutionResult result) { + if (stats != null) { + stats.testCompleted(originalTracker, result); + } + if (parent != null) { + parent.descendantExecutionFinished(originalTracker, result); + } + } + + public void executionSkipped() { + if (stats != null) { + stats.testSkipped(); + } + if (parent != null) { + parent.executionSkipped(); + } + } + } + + /** + * Count the tests that failed etc. + * + *

Modeled on org.apache.maven.plugin.surefire.report.TestSetStats + */ + public class TestContainerStats { + + private final long startedAtMillis; + private long selfOrDescendantFinishedAtMillis; + + private int successful; + + // Aborted happens when the test node decided not to run, normally by calling + // Assumptions.assumeTrue() which throws a TestAbortedException which is tracked + // differently by junit. + private int aborted; + + // An actual failure of an assertion or unexpected error thrown + private int failures; + + // dont think used, kept for completeness + private int skipped; + + public TestContainerStats() { + this.startedAtMillis = System.currentTimeMillis(); + } + + public long elapsedMillis() { + return selfOrDescendantFinishedAtMillis == 0 + ? System.currentTimeMillis() - startedAtMillis + : selfOrDescendantFinishedAtMillis - startedAtMillis; + } + + public int successful() { + return successful; + } + + public int aborted() { + return aborted; + } + + public int failures() { + return failures; + } + + public int skipped() { + return skipped; + } + + public void testCompleted(TestReportingTracker tracker, TestExecutionResult result) { + selfOrDescendantFinishedAtMillis = System.currentTimeMillis(); + + // we only update the stats IF the test we are tracking is a TEST, we do not update for + // containers. + if (tracker.identifier().isTest()) { + switch (result.getStatus()) { + case FAILED -> failures++; + case ABORTED -> aborted++; + case SUCCESSFUL -> successful++; + } + } + } + + public void testSkipped() { + selfOrDescendantFinishedAtMillis = System.currentTimeMillis(); + skipped++; + } + } +} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/reporting/TestBenchConsoleWriter.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/reporting/TestBenchConsoleWriter.java new file mode 100644 index 0000000000..eefdd5866e --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/reporting/TestBenchConsoleWriter.java @@ -0,0 +1,403 @@ +package io.stargate.sgv2.jsonapi.testbench.reporting; + +import static org.apache.maven.surefire.shared.utils.logging.MessageUtils.buffer; + +import io.stargate.sgv2.jsonapi.testbench.testrun.TestUri; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Objects; +import org.apache.maven.plugin.surefire.report.Theme; +import org.apache.maven.surefire.shared.utils.logging.MessageBuilder; +import org.junit.platform.engine.TestExecutionResult; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Writes messages for the output of Test Bench using the logging system. + * + *

NOTE: to get the best output via the logging system, we need to configure to remove the normal + * formatting that comes with logging. Below should be in the `applicaiton.yaml` in addition to the + * regular config + * + * 

+ * quarkus:
+ *   log:
+ *     console:
+ *       format: "%-5p [%t] %d{yyyy-MM-dd HH:mm:ss,SSS} %F:%L - %m%n"
+ *     handler:
+ *       console:
+ *         PLAIN_CONSOLE:
+ *           format: "%m%n"
+ *     category:
+ *       'io.stargate.sgv2.jsonapi.testbench.reporting.TestBenchConsoleWriter':
+ *         level: INFO
+ *         handlers:
+ *           - PLAIN_CONSOLE
+ *         use-parent-handlers: false
+ *
+ * + *

Works in two modes because we need to wait for all the tests to complete so we know what was + * successful. For info on how the output will look see {@link Theme} and {@link + * org.apache.maven.surefire.shared.utils.logging.AnsiMessageBuilder} + * + *

Call {@link #testStarted(int, int, DynamicTreeListener.TestReportingTracker)} when a test + * starts executing, will print the progress of the tests + * + *

Call {@link #allTestsFinished(DynamicTreeListener.TestReportingTracker)} with the root tracker + * for the rest run, this should be called once all the tests have completed so we know what failed + * and how long things took. This will print the full test tree, but only go down to the request + + * assertion level for test scenarios that have failed. + */ +public class TestBenchConsoleWriter { + + private static final Logger LOGGER = LoggerFactory.getLogger(TestBenchConsoleWriter.class); + + public static final String ENV_TEST_BENCH_REPORT_PATH = "test-bench-report-path"; + + private boolean firstLine = true; + + private final Theme theme; + + public TestBenchConsoleWriter() { + this(Theme.UNICODE); + } + + public TestBenchConsoleWriter(Theme theme) { + this.theme = Objects.requireNonNull(theme, "theme must not be null"); + } + + /** + * Writes that a test has started, without knowing how it will end, used when the test suite is + * running. + * + *

Example output: + * + * 

+   * ── ── ── ── ── ── ── ── ── ── ── ── ── ── ── ── ── ── ── ──
+   * Running Test Bench, results shown at completion...
+   * ── ── ── ── ── ── ── ── ── ── ── ── ── ── ── ── ── ── ── ──
+   *  ┬─ 1 of 1465: TestPlan: smoketest-prod-aws-ap-south-1 (701031671627) on astra (node:axM)
+   * ├─ 2 of 1465: Workflow: vectorize-shared-workflow  (node:alW)
+   *    ├─ 3 of 1465: Job: open-ai-vectorize  (node:ab7)
+   *       ├─ 4 of 1465: TestSuite: vectorize-shared-auth  (node:ab6)
+   *          ├─ 5 of 1465: TestEnv: [CREDENTIAL=open-ai-key, MODEL=text-embedding-3-small, PROVIDER=openai]  (node:aaN)
+   *             ├─ 6 of 1465: Request: SetupRequest[1]: CREATE_COLLECTION (node:aaf)
+   *                ├─ 7 of 1465: Command: createCollection (node:aaa)
+   *
+ * + * @param totalTestCount the total number of tests that will be run + * @param startedTestCount the number of tests that have started running, including this one + * @param tracker the tracker for the test that started + */ + public void testStarted( + int totalTestCount, int startedTestCount, DynamicTreeListener.TestReportingTracker tracker) { + + var buffer = buffer(); + if (firstLine) { + firstLine = false; + buffer + .newline() + .a(theme.dash().repeat(20)) + .newline() + .a("Running Test Bench, results shown at completion...") + .newline() + .a(theme.dash().repeat(20)) + .newline(); + } + + writeTreeOutline(buffer, tracker); + + // the progress "x of Y" and the displayName from the node set when it was created by the + // TestPlan. + // NOTE: the "(node:aaN)" is part of the displayName, it is so that when we print failed nodes + // we can + // find them in the full log easily. + buffer + .a(startedTestCount) + .a(" of ") + .a(totalTestCount) + .a(": ") + .strong(tracker.identifier().getDisplayName()); + + LOGGER.info(buffer.toString()); + } + + /** + * Call when all the tests have finished running, so we can print a summary for test suites that + * pass and details for those that fail. + * + *

Writes two types of output: + * + *

    + *
  • If {@link LOGGER#isInfoEnabled()} logs a detailed report of the test results, skips + * details of assertions that have completed successfully. See {@link + * #writeCompletedSummary(MessageBuilder, DynamicTreeListener.TestReportingTracker, + * boolean)} + *
  • If {@link #ENV_TEST_BENCH_REPORT_PATH} is set in the path, writes a markdown file in a + * format to be included in the summary for GitHub actions via the + * $GITHUB_STEP_SUMMARY. + *
+ * + * @param rootTracker Tracker for the root of the test tree, this is the one that has the full + * test tree under it. + */ + public void allTestsFinished(DynamicTreeListener.TestReportingTracker rootTracker) { + + var reportBuffer = buffer(); + writeCompletedSummary(reportBuffer, rootTracker, false); + var testReport = reportBuffer.toString(); + + if (LOGGER.isInfoEnabled()) { + + var logLineBuffer = buffer(); + logLineBuffer + .newline() + .a(theme.dash().repeat(20)) + .newline() + .a("Test Bench Results") + .newline() + .a(theme.dash().repeat(20)) + .newline(); + logLineBuffer.a(testReport); + LOGGER.info(logLineBuffer.toString()); + } + + var reportFilePath = System.getProperty(ENV_TEST_BENCH_REPORT_PATH); + if (reportFilePath != null) { + LOGGER.info("Writing report file to: {}", reportFilePath); + + // this is the info for the top node, so peeps know if they want to go down into the report. + var testPlanNodeDesc = buffer(); + writeTestDesc(testPlanNodeDesc, rootTracker); + testPlanNodeDesc.newline(); + + // the failed nodes and their throwable, if any + String failureReport; + if (rootTracker.stats().failures() == 0) { + failureReport = "No failures"; + } else { + var failureReportBuffer = buffer(); + writeFailureMessages(failureReportBuffer, rootTracker); + failureReport = failureReportBuffer.toString(); + } + + // report is a markdown file + // Using GitHub collapsable sections + // https://docs.github.com/en/get-started/writing-on-github/working-with-advanced-formatting/organizing-information-with-collapsed-sections + var markdownReport = + """ + ## %s + + %s + +
+ + Test Bench Summary + + ``` + %s + ``` + +
+ +
+ + Test Bench Failures + + ``` + %s + ``` + +
+ """ + .formatted( + rootTracker.identifier().getDisplayName(), + testPlanNodeDesc.toString(), + testReport, + failureReport); + + try { + Files.writeString(Path.of(reportFilePath), markdownReport); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + } + + /** + * Walks the test tree, outputs the results of running each node now that we know the completed + * results. + * + *

Example (the header is put in th by the caller) : + * + * 

+   * ── ── ── ── ── ── ── ── ── ── ── ── ── ── ── ── ── ── ── ──
+   * Test Bench Results
+   * ── ── ── ── ── ── ── ── ── ── ── ── ── ── ── ── ── ── ── ──
+   * ┬─ ✔ TestPlan: smoketest-prod-aws-ap-south-1 (701031671627) on astra (node:axM) - 835 s   Successful: 756, Failures: 0, Aborted: 0
+   * ├─ ✔ Workflow: vectorize-shared-workflow  (node:alW) - 417 s   Successful: 378, Failures: 0, Aborted: 0
+   *    ├─ ✔ Job: open-ai-vectorize  (node:ab7) - 76 s   Successful: 63, Failures: 0, Aborted: 0
+   *       ├─ ✔ TestSuite: vectorize-shared-auth  (node:ab6) - 76 s   Successful: 63, Failures: 0, Aborted: 0
+   *          ├─ ✔ TestEnv: [CREDENTIAL=open-ai-key, MODEL=text-embedding-3-small, PROVIDER=openai]  (node:aaN) - 30 s   Successful: 21, Failures: 0, Aborted: 0
+   *          ├─ ✔ TestEnv: [CREDENTIAL=open-ai-key, MODEL=text-embedding-3-large, PROVIDER=openai]  (node:abr) - 23 s   Successful: 21, Failures: 0, Aborted: 0
+   *          ├─ ✔ TestEnv: [CREDENTIAL=open-ai-key, MODEL=text-embedding-ada-002, PROVIDER=openai]  (node:ab5) - 22 s   Successful: 21, Failures: 0, Aborted: 0
+   *
+ * + *

NOTE: Because there can be 1,000s of test nodes, we do not below the TestEnv nodes + * unless there is a failure. Below those nodes are where the assertions that will have failed + * exist. We traverse down to the first failed test node, and then want to show which sibling (or + * cousin) nodes aborted because of the failure. For example: + * + * 

+   * ┬─ ✘ TestPlan: smoketest-prod-aws-eu-central-1 (105817733955) on astra (node:axM) - 413 s   Successful: 436, Failures: 20, Aborted: 300
+   * ├─ ✘ Workflow: vectorize-shared-workflow  (node:alW) - 41 s   Successful: 90, Failures: 18, Aborted: 270
+   *    ├─ ✘ Job: open-ai-vectorize  (node:ab7) - 10 s   Successful: 15, Failures: 3, Aborted: 45
+   *       ├─ ✘ TestSuite: vectorize-shared-auth  (node:ab6) - 10 s   Successful: 15, Failures: 3, Aborted: 45
+   *          ├─ ✘ TestEnv: [CREDENTIAL=open-ai-key, MODEL=text-embedding-3-small, PROVIDER=openai]  (node:aaN) - 5 s   Successful: 5, Failures: 1, Aborted: 15
+   *             ├─ ✘ Request: SetupRequest[1]: CREATE_COLLECTION (node:aaf) - 4 s   Successful: 2, Failures: 1, Aborted: 0
+   *                ├─ ✔ Command: createCollection (node:aaa)
+   *                ├─ ✘ Assertions (node:aae) - 0 s   Successful: 1, Failures: 1, Aborted: 0
+   *                   ├─ ✘ isSuccess (node:aad) - 0 s   Successful: 1, Failures: 1, Aborted: 0
+   *                      ├─ ✔ success [http status is 200] (node:aab)
+   *                      ├─ ✘ isDDLSuccess [body('$') - responseIsDDLSuccess: REQUIRED:[status], OPTIONAL:[], FORBIDDEN:[data, errors]] (node:aac)
+   *             ├─ ✘ Request: SetupRequest[2]: INSERT_ONE (node:aal) - 0 s   Successful: 0, Failures: 0, Aborted: 3
+   *             ├─ ✘ Request: SetupRequest[3]: INSERT_MANY (node:aar) - 0 s   Successful: 0, Failures: 0, Aborted: 3
+   *             ├─ ✘ Request: TestCase: name=basic findMany (node:aay) - 0 s   Successful: 0, Failures: 0, Aborted: 4
+   *             ├─ ✘ Request: TestCase: name=findOneAndUpdate (node:aaG) - 0 s   Successful: 0, Failures: 0, Aborted: 5
+   *             ├─ ✔ Request: CleanupRequest[4]: DELETE_COLLECTION (node:aaM) - 0 s   Successful: 3, Failures: 0, Aborted: 0
+   *
+ * + * In the first example we do not go down to the Request nodes because they did not fail or abort. + * + * @param buffer Buffer to append the message to. + * @param tracker The tracker we are writing out the summary for, and may then traverse its + * children. + * @param parentFailures Set true if any parent nodes have failures, this will cause us to + * traverse down to the children + */ + private void writeCompletedSummary( + MessageBuilder buffer, + DynamicTreeListener.TestReportingTracker tracker, + boolean parentFailures) { + + writeTreeOutline(buffer, tracker); + writeTestDesc(buffer, tracker); + buffer.newline(); + + // If we have a TestEnv then we want to write out the summary of results for it, otherwise + // descend until we get one + // OR if there are FAILURES then we descend, these are tests that ran but assertion failed. + // we do not descend for aborted, these are tests that did not run because of previous failure. + for (var child : tracker.children()) { + var hasFailures = child.stats() != null && child.stats().failures() > 0; + + if (!child.runUri().leafType().descendantOf(TestUri.Segment.ENV) + || hasFailures + || parentFailures) { + writeCompletedSummary(buffer, child, hasFailures); + } + } + } + + /** + * Writes the test node name, and it's throwable if it Failed, as in the assertions failed or it + * threw an exception. + * + *

Example: + * + * 

+   * ✘ isDDLSuccess [body('$') - responseIsDDLSuccess: REQUIRED:[status], OPTIONAL:[], FORBIDDEN:[data, errors]] (node:aac)
+   *
+   * [responseIsDDLSuccess: REQUIRED:[status], OPTIONAL:[], FORBIDDEN:[data, errors]]
+   * Expecting actual:
+   *   {"errors"=[{"errorCode"="VECTORIZE_CREDENTIAL_INVALID", "family"="REQUEST", "id"="95b651e3-815b-4be3-8cad-3074fcd86cd9", "message"="Invalid credential name for vectorize, with error: Embedding Gateway unable to resolve authentication type.
+   * Underlying problem: Sync service has internal server error. Error Code: 500; response description: Internal Server Error.      .", "scope"="SCHEMA", "title"="Invalid credential name for vectorize"}]}
+   * to contain key:
+   *   "status"
+   * -----
+   *
+   * ✘ isDDLSuccess [body('$') - responseIsDDLSuccess: REQUIRED:[status], OPTIONAL:[], FORBIDDEN:[data, errors]] (node:aaQ)
+   *
+   * [responseIsDDLSuccess: REQUIRED:[status], OPTIONAL:[], FORBIDDEN:[data, errors]]
+   * Expecting actual:
+   *   {"errors"=[{"errorCode"="VECTORIZE_CREDENTIAL_INVALID", "family"="REQUEST", "id"="9d512a57-ca08-491b-8440-a5abc14b99a4", "message"="Invalid credential name for vectorize, with error: Embedding Gateway unable to resolve authentication type.
+   * Underlying problem: Sync service has internal server error. Error Code: 500; response description: Internal Server Error.      .", "scope"="SCHEMA", "title"="Invalid credential name for vectorize"}]}
+   * to contain key:
+   *   "status"
+   * -----
+   *
+ */ + private void writeFailureMessages( + MessageBuilder buffer, DynamicTreeListener.TestReportingTracker tracker) { + + // if we have a throwable, write out the tree node test and the error it generated. + // ignoring the ABORTED + if (tracker.throwable().isPresent() + && (tracker.junitStatus() == TestExecutionResult.Status.FAILED)) { + writeTestDesc(buffer, tracker); + buffer.newline(); + buffer.newline(); + buffer.a(tracker.throwable().get().getMessage()); + buffer.newline().a("-----").newline().newline(); + } + tracker.children().forEach(child -> writeFailureMessages(buffer, child)); + } + + private void writeTestDesc( + MessageBuilder buffer, DynamicTreeListener.TestReportingTracker tracker) { + + // Icon for success for failure, + // if we have stats then this is a container we use that status, otherwise we use the JUNIT + // execution status + if (tracker.stats() == null) { + switch (tracker.junitStatus()) { + case SUCCESSFUL -> buffer.a(theme.successful()); + case FAILED, ABORTED -> buffer.a(theme.failed()); + case null -> {} + } + } else { + if (tracker.stats().failures() == 0 && tracker.stats().aborted() == 0) { + buffer.a(theme.successful()); + } else { + buffer.a(theme.failed()); + } + } + + // The name of the container or test + buffer.strong(tracker.identifier().getDisplayName()); + + // if we have stats write a stats line, these are aggregate for all things below. + if (tracker.stats() != null) { + writeTestStats(buffer, tracker); + } + + // NOTE: does not add new line, caller should + } + + private void writeTestStats( + MessageBuilder buffer, DynamicTreeListener.TestReportingTracker tracker) { + buffer + .a(" - %s s".formatted(tracker.stats().elapsedMillis() / 1000)) + .a(theme.blank()) + .a("Successful: ") + .a(tracker.stats().successful()) + .a(", ") + .a("Failures: ") + .a(tracker.stats().failures()) + .a(", ") + .a("Aborted: ") + .a(tracker.stats().aborted()); + } + + private void writeTreeOutline( + MessageBuilder buffer, DynamicTreeListener.TestReportingTracker tracker) { + + // Indenting and tree building + if (tracker.parent() == null) { + buffer.a(theme.down()); + } else { + buffer.a(theme.blank().repeat(tracker.depth() - 1)).a(theme.entry()); + } + } +} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/targets/AstraBackend.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/targets/AstraBackend.java new file mode 100644 index 0000000000..db642a35d6 --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/targets/AstraBackend.java @@ -0,0 +1,16 @@ +package io.stargate.sgv2.jsonapi.testbench.targets; + +import io.stargate.sgv2.jsonapi.testbench.testspec.Job; + +/** DataStax / IBM Astra */ +public class AstraBackend extends Backend { + + public static final String NAME = "astra"; + + @Override + public void updateJobForTarget(Job job) { + + // always using the `default_keyspace` for astra, because they cannot be made using the API + job.variables().put("KEYSPACE_NAME", "default_keyspace"); + } +} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/targets/Backend.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/targets/Backend.java new file mode 100644 index 0000000000..7d29c6e1d1 --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/targets/Backend.java @@ -0,0 +1,27 @@ +package io.stargate.sgv2.jsonapi.testbench.targets; + +import io.stargate.sgv2.jsonapi.testbench.lifecycle.JobLifeCycle; +import io.stargate.sgv2.jsonapi.testbench.lifecycle.TestPlanLifecycle; +import java.util.regex.Pattern; + +/** + * A class of backend database the tests will be run against. This is not a partcular insance of a + * DB to run against, for that see {@link Target}. + * + *

There is some different behavior between C* and Astra, and potentially in any future versions + * of those products. + */ +public abstract class Backend implements TestPlanLifecycle, JobLifeCycle { + + private static final Pattern PATTERN_NOT_WORD_CHARS = Pattern.compile("\\W+"); + + /** Sanitizes a schema name to be valid in all C* derived platforms. */ + public static String toSafeSchemaIdentifier(String name) { + + var newValue = PATTERN_NOT_WORD_CHARS.matcher(name).replaceAll("_"); + if (newValue.length() > 48) { + return newValue.substring(0, 47); + } + return newValue; + } +} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/targets/CassandraBackend.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/targets/CassandraBackend.java new file mode 100644 index 0000000000..826439148e --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/targets/CassandraBackend.java @@ -0,0 +1,88 @@ +package io.stargate.sgv2.jsonapi.testbench.targets; + +import io.stargate.sgv2.jsonapi.testbench.assertions.TestAssertion; +import io.stargate.sgv2.jsonapi.testbench.testrun.TestExecutionCondition; +import io.stargate.sgv2.jsonapi.testbench.testrun.TestNodeFactory; +import io.stargate.sgv2.jsonapi.testbench.testrun.TestRunRequest; +import io.stargate.sgv2.jsonapi.testbench.testrun.TestUri; +import io.stargate.sgv2.jsonapi.testbench.testspec.Job; +import io.stargate.sgv2.jsonapi.testbench.testspec.TestCommand; +import java.util.Optional; +import org.apache.commons.lang3.RandomStringUtils; +import org.junit.jupiter.api.DynamicNode; + +/** A classic Cassandra backend, running locally or elsewhere. */ +public class CassandraBackend extends Backend { + + public static final String NAME = "cassandra"; + + @Override + public void updateJobForTarget(Job job) { + + // We are going to create the keyspace for every job, so making a new name here based on the + // name + // of the job, and some random because the name will be trunc'd + var keyspaceName = + toSafeSchemaIdentifier( + "job_" + + job.meta().name().substring(0, Math.min(job.meta().name().length(), 27)) + + "_" + + RandomStringUtils.insecure().nextAlphanumeric(16)); + job.variables().put("KEYSPACE_NAME", keyspaceName); + } + + /** Need to create a keyspace, because C* does not have a default one. */ + @Override + public Optional beforeJob( + TestNodeFactory testNodeFactory, TestUri.Builder uriBuilder, Job job) { + + var command = + TestCommand.fromJson( + """ + { + "createKeyspace": { + "name": "${KEYSPACE_NAME}" + } + } + """); + + var env = job.withoutMatrix(testNodeFactory.testPlan()); + var setupRequest = + new TestRunRequest( + env.substitutor().replace("createKeyspace: ${KEYSPACE_NAME}"), + command, + testNodeFactory.testPlan().target(), + env, + TestAssertion.forSuccess(testNodeFactory.testPlan(), command), + new TestExecutionCondition.AlwaysTrue("CassandraBackend.beforeJob()")); + + return Optional.of(setupRequest.testNodes(testNodeFactory, uriBuilder)); + } + + /** Drop the keyspace we made for this job. */ + @Override + public Optional afterJob( + TestNodeFactory testNodeFactory, TestUri.Builder uriBuilder, Job job) { + var command = + TestCommand.fromJson( + """ + { + "dropKeyspace": { + "name": "${KEYSPACE_NAME}" + } + } + """); + + var env = job.withoutMatrix(testNodeFactory.testPlan()); + var setupRequest = + new TestRunRequest( + env.substitutor().replace("dropKeyspace: ${KEYSPACE_NAME}"), + command, + testNodeFactory.testPlan().target(), + job.withoutMatrix(testNodeFactory.testPlan()), + TestAssertion.forSuccess(testNodeFactory.testPlan(), command), + new TestExecutionCondition.AlwaysTrue("CassandraBackend.afterJob()")); + + return Optional.of(setupRequest.testNodes(testNodeFactory, uriBuilder)); + } +} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/targets/ConnectionConfiguration.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/targets/ConnectionConfiguration.java new file mode 100644 index 0000000000..685caa3bdf --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/targets/ConnectionConfiguration.java @@ -0,0 +1,26 @@ +package io.stargate.sgv2.jsonapi.testbench.targets; + +/** + * Configuraiton record of how to connect for a {@link TargetConfiguration}. + * + *

--- + * + * @param domain domain and protocol, e.g. http://localhost + * @param port port to connect to, typically 8181 ore 443 but may be + * different when running integration tests. + * @param basePath base path to the API, for Astra this is /api/json/v1 when running + * locally is normally /v1 + */ +public record ConnectionConfiguration(String domain, Integer port, String basePath) { + public ConnectionConfiguration { + if (domain == null) { + domain = "localhost"; + } + if (port == null) { + port = 8181; + } + if (basePath == null) { + basePath = "/v1"; + } + } +} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/targets/Target.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/targets/Target.java new file mode 100644 index 0000000000..f2885828cf --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/targets/Target.java @@ -0,0 +1,111 @@ +package io.stargate.sgv2.jsonapi.testbench.targets; + +import io.stargate.sgv2.jsonapi.testbench.lifecycle.JobLifeCycle; +import io.stargate.sgv2.jsonapi.testbench.lifecycle.TestPlanLifecycle; +import io.stargate.sgv2.jsonapi.testbench.messaging.APIRequest; +import io.stargate.sgv2.jsonapi.testbench.testrun.TestNodeFactory; +import io.stargate.sgv2.jsonapi.testbench.testrun.TestRunEnv; +import io.stargate.sgv2.jsonapi.testbench.testrun.TestUri; +import io.stargate.sgv2.jsonapi.testbench.testspec.Job; +import io.stargate.sgv2.jsonapi.testbench.testspec.TestCommand; +import io.stargate.sgv2.jsonapi.testbench.testspec.TestSuiteSpec; +import io.stargate.sgv2.jsonapi.testbench.testspec.WorkflowSpec; +import java.util.Optional; +import org.junit.jupiter.api.DynamicNode; + +/** + * A particular instance of a {@link Backend} we are going to run the test against, so includes + * connection information etc. + * + *

A run of the Test Bench is run against a Target, e.g. cassandra on localhost, or an astra db + * called monkeys. + * + *

This is the important entry point for the lifecycle interfaces, because the life cycle is + * there to handle different target / backends that tests run against. + * + *

Because this holds the connection information, it can also make a request, see {@link + * #apiRequest(TestCommand, TestRunEnv)} + */ +public class Target implements TestPlanLifecycle, JobLifeCycle { + + private final TargetConfiguration targetConfiguration; + private final Backend backend; + + public Target(TargetConfiguration targetConfiguration) { + this.targetConfiguration = targetConfiguration; + + this.backend = + switch (targetConfiguration.backend()) { + case CassandraBackend.NAME -> new CassandraBackend(); + case AstraBackend.NAME -> new AstraBackend(); + default -> + throw new IllegalArgumentException( + "Unknown backend: " + targetConfiguration.backend()); + }; + } + + public TargetConfiguration configuration() { + return targetConfiguration; + } + + /** + * Call this to get a new {@link APIRequest} that is configured to talk to the target this class + * represents. + * + * @param testCommand The command the request will send, this is needed to get the actual DataAPI + * request we want to send. + * @param env Environment the commands will be run in, used to make the replacements in the + * command to execute for this particular test run. + * @return Configured {@link APIRequest} + */ + public APIRequest apiRequest(TestCommand testCommand, TestRunEnv env) { + return new APIRequest(targetConfiguration.connection(), env, testCommand.withEnvironment(env)); + } + + @Override + public void updateJobForTarget(Job job) { + backend.updateJobForTarget(job); + } + + @Override + public Optional beforeWorkflow( + TestNodeFactory testNodeFactory, TestUri.Builder uriBuilder, WorkflowSpec workflow) { + return backend.beforeWorkflow(testNodeFactory, uriBuilder, workflow); + } + + @Override + public Optional afterWorkflow( + TestNodeFactory testNodeFactory, TestUri.Builder uriBuilder, WorkflowSpec workflow) { + return backend.afterWorkflow(testNodeFactory, uriBuilder, workflow); + } + + @Override + public Optional beforeJob( + TestNodeFactory testNodeFactory, TestUri.Builder uriBuilder, Job job) { + return backend.beforeJob(testNodeFactory, uriBuilder, job); + } + + @Override + public Optional afterJob( + TestNodeFactory testNodeFactory, TestUri.Builder uriBuilder, Job job) { + return backend.afterJob(testNodeFactory, uriBuilder, job); + } + + @Override + public Optional beforeTestSuite( + TestNodeFactory testNodeFactory, + TestUri.Builder uriBuilder, + TestSuiteSpec test, + TestRunEnv env) { + return backend.beforeTestSuite(testNodeFactory, uriBuilder, test, env); + } + + @Override + public Optional afterTestSuite( + TestNodeFactory testNodeFactory, + TestUri.Builder uriBuilder, + TestSuiteSpec test, + TestRunEnv env) { + return backend.afterTestSuite(testNodeFactory, uriBuilder, test, env); + } +} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/targets/TargetConfiguration.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/targets/TargetConfiguration.java new file mode 100644 index 0000000000..84eb1dfcda --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/targets/TargetConfiguration.java @@ -0,0 +1,13 @@ +package io.stargate.sgv2.jsonapi.testbench.targets; + +/** + * Configuration record for a {@link Target}. + * + *

--- + * + * @param name Friendly name of the target + * @param backend Backend name, such as astra or cassandra so we know how to run the lifecycle + * @param connection Connection information for the target. + */ +public record TargetConfiguration( + String name, String backend, ConnectionConfiguration connection) {} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/testrun/DynamicTestExecutable.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testrun/DynamicTestExecutable.java new file mode 100644 index 0000000000..4a725153ca --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testrun/DynamicTestExecutable.java @@ -0,0 +1,76 @@ +package io.stargate.sgv2.jsonapi.testbench.testrun; + +import org.junit.jupiter.api.Assumptions; +import org.junit.jupiter.api.DynamicTest; +import org.junit.jupiter.api.function.Executable; + +public class DynamicTestExecutable implements Executable { + + private final String description; + private final TestUri testUri; + private final Executable executable; + + private final String trimmedDisplayName; + private final boolean isTrimmed; + private final TestExecutionCondition testExecutionCondition; + + public DynamicTestExecutable( + String description, + TestUri.Builder testUri, + TestExecutionCondition testExecutionCondition, + Executable executable) { + this(description, testUri.build(), testExecutionCondition, executable); + } + + @SuppressWarnings("StringEquality") + public DynamicTestExecutable( + String description, + TestUri testUri, + TestExecutionCondition testExecutionCondition, + Executable executable) { + this.description = description; + this.testUri = testUri; + this.testExecutionCondition = testExecutionCondition; + this.executable = executable; + + var truncated = + (description != null && description.length() > 120) + ? description.substring(0, 117) + "..." + : description; + + this.trimmedDisplayName = truncated; + // using reference quality to see it is a diff object. + this.isTrimmed = truncated != description; + } + + public String trimmedDisplayName() { + return trimmedDisplayName; + } + + public DynamicTest testNode(TestNodeFactory testNodeFactory) { + return testNodeFactory.testPlanTest(trimmedDisplayName, testUri.uri(), this); + } + + @Override + public void execute() throws Throwable { + Assumptions.assumeTrue(testExecutionCondition, testExecutionCondition.message()); + beforeExecute(); + try { + executable.execute(); + } catch (Throwable e) { + testExecutionCondition.abortFutureTests("Failed Upstream: " + trimmedDisplayName); + throw e; + } + afterExecute(); + } + + private void beforeExecute() { + // if (isTrimmed) { + // System.out.printf(description + "\n"); + // } + } + + private void afterExecute() { + // System.out.printf("Executed - " + testUri.uri().toString() + "\n"); + } +} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/testrun/TestExecutionCondition.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testrun/TestExecutionCondition.java new file mode 100644 index 0000000000..a3ba0df3af --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testrun/TestExecutionCondition.java @@ -0,0 +1,65 @@ +package io.stargate.sgv2.jsonapi.testbench.testrun; + +import java.util.function.BooleanSupplier; + +public interface TestExecutionCondition extends BooleanSupplier { + + void abortFutureTests(String message); + + String message(); + + public class Default implements TestExecutionCondition { + + private boolean condition = true; + private String message = ""; + + // to scope of this condition, i.e. this is for + // TestEnv: [MODEL=NV-Embed-QA, PROVIDER=nvidia] + private String scope; + + public Default(String scope) { + this.scope = scope; + } + + @Override + public void abortFutureTests(String message) { + condition = false; + this.message = message; + } + + @Override + public boolean getAsBoolean() { + return condition; + } + + @Override + public String message() { + return "TestCondition: Scope=" + scope + ", Message=" + message; + } + } + + public class AlwaysTrue implements TestExecutionCondition { + + // to scope of this condition, i.e. this is for + // TestEnv: [MODEL=NV-Embed-QA, PROVIDER=nvidia] + // we never need a message, because always true, but here for debugging. + private String scope; + + public AlwaysTrue(String scope) { + this.scope = scope; + } + + @Override + public void abortFutureTests(String message) {} + + @Override + public boolean getAsBoolean() { + return true; + } + + @Override + public String message() { + return ""; + } + } +} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/testrun/TestNodeFactory.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testrun/TestNodeFactory.java new file mode 100644 index 0000000000..5990d2ed72 --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testrun/TestNodeFactory.java @@ -0,0 +1,187 @@ +package io.stargate.sgv2.jsonapi.testbench.testrun; + +import io.stargate.sgv2.jsonapi.testbench.TestBenchPlan; +import io.stargate.sgv2.jsonapi.testbench.testspec.Job; +import io.stargate.sgv2.jsonapi.testbench.testspec.TestSpecMeta; +import io.stargate.sgv2.jsonapi.testbench.testspec.TestSuiteSpec; +import io.stargate.sgv2.jsonapi.testbench.testspec.WorkflowSpec; +import java.net.URI; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Optional; +import java.util.function.Supplier; +import org.junit.jupiter.api.DynamicContainer; +import org.junit.jupiter.api.DynamicNode; +import org.junit.jupiter.api.DynamicTest; +import org.junit.jupiter.api.function.Executable; + +/** + * A container we can pass around when building the TestNodes that has the test plan we are working + * with, and has a counter for the number of nodes we created. + * + *

So we can output them for the test reporter to report progress. + */ +public class TestNodeFactory { + + private final NodeCode nodeCode = new NodeCode(); + private final TestBenchPlan testPlan; + private int totalNodeCount = 0; + + public TestNodeFactory(TestBenchPlan testPlan) { + this.testPlan = testPlan; + } + + public TestBenchPlan testPlan() { + return testPlan; + } + + public int testNodeCount() { + return totalNodeCount; + } + + /** + * NOTE: This is forcing the use of a List, so we greedily create all the test nodes, so we can + * count how many their are, so we can show progress. + * + * @param displayName + * @param testSourceUri + * @param dynamicNodes + * @return + */ + public DynamicContainer testPlanContainer( + String displayName, URI testSourceUri, List dynamicNodes) { + totalNodeCount++; + return DynamicContainer.dynamicContainer( + appendNodeCode(displayName), testSourceUri, dynamicNodes.stream()); + } + + public DynamicTest testPlanTest(String description, URI uri, Executable executable) { + + totalNodeCount++; + return DynamicTest.dynamicTest(appendNodeCode(description), uri, executable); + } + + private String appendNodeCode(String displayName) { + return "%s (node:%s)".formatted(displayName, nodeCode.next()); + } + + public List addLifecycle( + TestUri.Builder uriBuilder, WorkflowSpec workflow, List dynamicNodes) { + + var beforeUriBuilder = + uriBuilder.clone().addSegment(TestUri.Segment.LIFECYCLE, "before-workflow"); + var afterUriBuilder = + uriBuilder.clone().addSegment(TestUri.Segment.LIFECYCLE, "after-workflow"); + + var nodes = new ArrayList(); + nodes.addAll( + lifecycleNodes( + beforeUriBuilder, + "Before Workflow", + workflow.meta(), + () -> testPlan.target().beforeWorkflow(this, beforeUriBuilder, workflow))); + nodes.addAll(dynamicNodes); + nodes.addAll( + lifecycleNodes( + afterUriBuilder, + "After Workflow", + workflow.meta(), + () -> testPlan.target().afterWorkflow(this, afterUriBuilder, workflow))); + return nodes; + } + + public List addLifecycle( + TestUri.Builder uriBuilder, Job job, List dynamicNodes) { + + var beforeUriBuilder = uriBuilder.clone().addSegment(TestUri.Segment.LIFECYCLE, "before-job"); + var afterUriBuilder = uriBuilder.clone().addSegment(TestUri.Segment.LIFECYCLE, "after-job"); + + var nodes = new ArrayList(); + nodes.addAll( + lifecycleNodes( + beforeUriBuilder, + "Before Job", + job.meta(), + () -> testPlan.target().beforeJob(this, beforeUriBuilder, job))); + nodes.addAll(dynamicNodes); + nodes.addAll( + lifecycleNodes( + afterUriBuilder, + "After Job", + job.meta(), + () -> testPlan.target().afterJob(this, afterUriBuilder, job))); + return nodes; + } + + public List addLifecycle( + TestUri.Builder uriBuilder, + TestSuiteSpec testSuite, + TestRunEnv environment, + List dynamicNodes) { + + var beforeUriBuilder = + uriBuilder.clone().addSegment(TestUri.Segment.LIFECYCLE, "before-test-suite"); + var afterUriBuilder = + uriBuilder.clone().addSegment(TestUri.Segment.LIFECYCLE, "after-test-suite"); + + var nodes = new ArrayList(); + nodes.addAll( + lifecycleNodes( + beforeUriBuilder, + "Before TestSuite", + testSuite.meta(), + () -> + testPlan.target().beforeTestSuite(this, beforeUriBuilder, testSuite, environment))); + nodes.addAll(dynamicNodes); + nodes.addAll( + lifecycleNodes( + afterUriBuilder, + "After TestSuite", + testSuite.meta(), + () -> testPlan.target().afterTestSuite(this, afterUriBuilder, testSuite, environment))); + return nodes; + } + + private List lifecycleNodes( + TestUri.Builder uriBuilder, + String namePrefix, + TestSpecMeta meta, + Supplier> nodeSupplier) { + + var targetDynamicNode = nodeSupplier.get(); + + if (targetDynamicNode.isEmpty()) { + return Collections.emptyList(); + } + + var lifecycleContainer = + testPlanContainer( + namePrefix + ": " + meta.name(), + uriBuilder.build().uri(), + List.of(targetDynamicNode.get())); + return List.of(lifecycleContainer); + } + + public static class NodeCode { + + private static final char[] ALPHABET = + "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789".toCharArray(); + private static final int BASE = ALPHABET.length; // 62 + + // 3 characters of base 62 coding above gives 62^3 = 238,328 + private static final int LENGTH = 3; + + private int counter = 0; + + public String next() { + int n = counter++; + char[] code = new char[LENGTH]; + for (int i = LENGTH - 1; i >= 0; i--) { + code[i] = ALPHABET[n % BASE]; + n /= BASE; + } + return new String(code); + } + } +} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/testrun/TestRunEnv.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testrun/TestRunEnv.java new file mode 100644 index 0000000000..7726a9910c --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testrun/TestRunEnv.java @@ -0,0 +1,126 @@ +package io.stargate.sgv2.jsonapi.testbench.testrun; + +import io.stargate.sgv2.jsonapi.testbench.targets.Backend; +import io.stargate.sgv2.jsonapi.testbench.testspec.TestSuiteSpec; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; +import org.apache.commons.text.StringSubstitutor; +import org.apache.commons.text.lookup.StringLookupFactory; +import org.junit.jupiter.api.DynamicContainer; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class TestRunEnv { + + private static final Logger LOGGER = LoggerFactory.getLogger(TestRunEnv.class); + + // Wellknown environment variables use this because we know they are schema identifiers + public static final String ENV_KEYSPACE_NAME = "KEYSPACE_NAME"; + // Also for tables + public static final String ENV_COLLECTION_NAME = "COLLECTION_NAME"; + + private static final Set SCHEMA_IDENTIFIER = + Set.of(ENV_KEYSPACE_NAME, ENV_COLLECTION_NAME); + + private final Map vars = new HashMap<>(); + + public TestRunEnv() { + this(new HashMap<>()); + } + + public TestRunEnv(Map vars) { + this.vars.putAll(vars); + } + + public DynamicContainer testNode( + TestNodeFactory testNodeFactory, TestUri.Builder uriBuilder, TestSuiteSpec testSuite) { + + var d = description(); + uriBuilder.addSegment(TestUri.Segment.ENV, d); + var desc = "TestEnv: %s ".formatted(d); + + var testExecutionCondition = new TestExecutionCondition.Default(desc); + var envNodes = + testSuite.testNodesForEnvironment( + testNodeFactory, uriBuilder.clone(), this, testExecutionCondition); + + return testNodeFactory.testPlanContainer( + desc, + uriBuilder.build().uri(), + testNodeFactory.addLifecycle(uriBuilder.clone(), testSuite, this, envNodes)); + } + + private String description() { + return vars.entrySet().stream() + .filter( + entry -> { + var name = entry.getKey().toUpperCase(); + if (SCHEMA_IDENTIFIER.contains(name)) { + return false; // schema names not usually interesting + } + if (name.contains("KEY") || name.contains("API") || name.contains("TOKEN")) { + return false; // assume security + } + return true; + }) + .sorted(Map.Entry.comparingByKey()) + .toList() + .toString(); + } + + private TestRunEnv(TestRunEnv other) { + this.vars.putAll(other.vars); + } + + public TestRunEnv clone() { + return new TestRunEnv(this); + } + + public TestRunEnv put(TestRunEnv other) { + this.vars.putAll(other.vars); + return this; + } + + public void put(String key, String value) { + this.vars.put(key, value); + } + + public String requiredValue(String name) { + if (vars.containsKey(name)) { + return get(name); + } + throw new RuntimeException( + String.format( + "Required env var not found name:%s, defined: %s", + name, String.join(", ", vars.keySet()))); + } + + public StringSubstitutor substitutor() { + + return new StringSubstitutor(StringLookupFactory.INSTANCE.functionStringLookup(this::get)) + .setEnableUndefinedVariableException(true); + } + + public String get(String name) { + return get(name, ""); + } + + public String get(String name, String defaultValue) { + + var value = vars.get(name); + if (value == null) { + value = defaultValue; + } + + var substituted = substitutor().replace(value); + return SCHEMA_IDENTIFIER.contains(name) + ? Backend.toSafeSchemaIdentifier(substituted) + : substituted; + } + + @Override + public String toString() { + return vars.toString(); + } +} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/testrun/TestRunRequest.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testrun/TestRunRequest.java new file mode 100644 index 0000000000..bbf6877e2d --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testrun/TestRunRequest.java @@ -0,0 +1,69 @@ +package io.stargate.sgv2.jsonapi.testbench.testrun; + +import io.stargate.sgv2.jsonapi.testbench.assertions.TestAssertion; +import io.stargate.sgv2.jsonapi.testbench.targets.Target; +import io.stargate.sgv2.jsonapi.testbench.testspec.TestCommand; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.atomic.AtomicReference; +import org.junit.jupiter.api.DynamicContainer; +import org.junit.jupiter.api.DynamicNode; + +public record TestRunRequest( + String name, + TestCommand testCommand, + Target target, + TestRunEnv testEnvironment, + List testAssertions, + TestExecutionCondition testExecutionCondition) { + + public TestRunResponse execute() { + + var apiRequest = target.apiRequest(testCommand, testEnvironment); + return new TestRunResponse(this, apiRequest, apiRequest.execute()); + } + + public DynamicContainer testNodes(TestNodeFactory testNodeFactory, TestUri.Builder uriBuilder) { + + uriBuilder.addSegment(TestUri.Segment.REQUEST, name()); + + var nodes = new ArrayList(); + + AtomicReference atomicResponse = new AtomicReference<>(); + + // Execute the request, and set so the assertions can pull the response after. + var commandExecutable = + new DynamicTestExecutable( + "Command: " + testCommand.commandName().getApiName(), + uriBuilder + .clone() + .addSegment(TestUri.Segment.COMMAND, testCommand.commandName().getApiName()), + testExecutionCondition, + () -> atomicResponse.set(execute())); + + nodes.add(commandExecutable.testNode(testNodeFactory)); + + // tests for each assertion + var assertionsUriBuilder = + uriBuilder.clone().addSegment(TestUri.Segment.ASSERTION_CONTAINER, "assertions"); + var assertionTests = + testAssertions().stream() + .map( + testAssertion -> + testAssertion.testNodes( + testNodeFactory, + assertionsUriBuilder.clone(), + atomicResponse, + testExecutionCondition)) + .toList(); + + // if we have assertion tests, put them in a container + if (!assertionTests.isEmpty()) { + nodes.add( + testNodeFactory.testPlanContainer( + "Assertions", assertionsUriBuilder.build().uri(), assertionTests)); + } + + return testNodeFactory.testPlanContainer("Request: " + name, uriBuilder.build().uri(), nodes); + } +} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/testrun/TestRunResponse.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testrun/TestRunResponse.java new file mode 100644 index 0000000000..50a3f80f33 --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testrun/TestRunResponse.java @@ -0,0 +1,7 @@ +package io.stargate.sgv2.jsonapi.testbench.testrun; + +import io.stargate.sgv2.jsonapi.testbench.messaging.APIRequest; +import io.stargate.sgv2.jsonapi.testbench.messaging.APIResponse; + +public record TestRunResponse( + TestRunRequest testRequest, APIRequest apiRequest, APIResponse apiResponse) {} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/testrun/TestUri.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testrun/TestUri.java new file mode 100644 index 0000000000..cdcae2c4f2 --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testrun/TestUri.java @@ -0,0 +1,170 @@ +package io.stargate.sgv2.jsonapi.testbench.testrun; + +import static java.util.stream.Collectors.joining; + +import java.net.URI; +import java.util.*; +import java.util.regex.Pattern; +import java.util.stream.Stream; + +public record TestUri(Scheme scheme, List segments) { + + public static TestUri.Builder builder(Scheme scheme) { + return new TestUri.Builder(scheme); + } + + public Segment leafType() { + return segments.getLast().segment; + } + + public URI uri() { + var path = segments.stream().map(SegmentValue::toString).collect(joining("/")); + + return URI.create(scheme.name() + "://" + AUTHORITY + "/" + path); + } + + public static Optional parse(URI uri) { + + Scheme scheme; + try { + scheme = Scheme.valueOf(uri.getScheme().toUpperCase()); + } catch (IllegalArgumentException e) { + return Optional.empty(); + } + + var builder = builder(scheme); + SegmentValue.parse(uri).forEach(builder::addSegment); + return Optional.of(builder.build()); + } + + public enum Scheme { + DATAAPI; + + public String pathName() { + return name().toLowerCase(); + } + } + + // aka the domain + public static final String AUTHORITY = "TESTRUN"; + + public enum Segment { + TARGET(null), + LIFECYCLE(null), // used in multiple places + WORKFLOW(TARGET), + JOB(WORKFLOW), + SUITE(JOB), + ENV(SUITE), + STAGE(ENV), + REQUEST(STAGE), + COMMAND(REQUEST), + ASSERTION_CONTAINER(REQUEST), + ASSERTION(ASSERTION_CONTAINER); + + private final Segment parent; + + Segment(Segment parent) { + this.parent = parent; + } + + public Segment parent() { + return parent; + } + + public boolean isParentValid(Segment segment) { + // XXX TODO: needs work + return true; + // return (parent == null) || (parent == segment) || (segment == LIFECYCLE) ; + } + + public String pathName() { + return name().toLowerCase(); + } + + public boolean descendantOf(Segment segment) { + if (parent == null) { + return false; + } + if (this.parent == segment) { + return true; + } + return this.parent.descendantOf(segment); + } + } + + public record SegmentValue(Segment segment, String value) { + + private static final Pattern INVALID_CHARS = Pattern.compile("[^a-zA-Z0-9\\-_.~]"); + + public SegmentValue { + Objects.requireNonNull(segment, "segment must not be null"); + Objects.requireNonNull(value, "value must not be null"); + } + + public static Stream parse(URI uri) { + var path = uri.getPath().startsWith("/") ? uri.getPath().substring(1) : uri.getPath(); + + return Arrays.stream(path.split("/")).map(SegmentValue::parse); + } + + public static SegmentValue parse(String segmentKeyValue) { + var parts = segmentKeyValue.split("=", 2); + if (parts.length != 2) { + throw new IllegalArgumentException( + "Invalid segment, expected key=value format: " + segmentKeyValue); + } + try { + return new SegmentValue(Segment.valueOf(parts[0].toUpperCase()), parts[1]); + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException("Unknown segment key: " + parts[0]); + } + } + + @Override + public String toString() { + return segment.name() + "=" + INVALID_CHARS.matcher(value).replaceAll("_"); + } + } + + public static class Builder { + + private final Scheme scheme; + private final List segmentValues; + + protected Builder(Scheme scheme) { + this(scheme, new ArrayList<>()); + } + + private Builder(Scheme scheme, List segmentValues) { + this.scheme = Objects.requireNonNull(scheme, "scheme must not be null"); + this.segmentValues = Objects.requireNonNull(segmentValues, "segmentValues must not be null"); + } + + public Builder addSegment(SegmentValue segmentValue) { + segmentValues.add(segmentValue); + return this; + } + + public Builder addSegment(Segment segment, String value) { + segmentValues.add(new SegmentValue(segment, value)); + return this; + } + + public Builder clone() { + return new Builder(scheme, new ArrayList<>(segmentValues)); + } + + public TestUri build() { + for (int i = 0; i < segmentValues.size(); i++) { + var current = segmentValues.get(i); + var previous = i == 0 ? null : segmentValues.get(i - 1).segment(); + if (!current.segment().isParentValid(previous)) { + throw new IllegalArgumentException( + "Invalid segment order. segment=%s expected parent=%s but previous=%s" + .formatted(current.segment(), current.segment().parent(), previous)); + } + } + return new TestUri(scheme, List.copyOf(segmentValues)); + } + } +} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/AssertionTemplateSpec.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/AssertionTemplateSpec.java new file mode 100644 index 0000000000..01aa785e03 --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/AssertionTemplateSpec.java @@ -0,0 +1,43 @@ +package io.stargate.sgv2.jsonapi.testbench.testspec; + +import com.fasterxml.jackson.databind.JsonNode; +import io.stargate.sgv2.jsonapi.testbench.assertions.AssertionFactory; +import java.util.Map; +import java.util.Optional; + +/** + * Spec defintions about re-usable assertions that are defined in JSON. See {@link + * AssertionFactory}. + * + *

Example, showing the defintion for "isSuccess" assertion and how it is defined on a per + * command basis. + * + * 

+ * {
+ *   "meta": {
+ *     "name": "assertions-templates",
+ *     "kind": "assertion_template"
+ *   },
+ *   "templates": {
+ *     "isSuccess": {
+ *       "createCollection": {
+ *         "http.success": null,
+ *         "response.isDDLSuccess": null
+ *       },
+ *       "find": {
+ *         "http.success": null,
+ *         "response.isFindSuccess": null
+ *       },
+ *
+ * + * @param meta Metadata for the spec. + * @param templates JSON Object that is a map of assertion name to a map of implementations per API + * command, see example + */ +public record AssertionTemplateSpec(TestSpecMeta meta, Map templates) + implements TestSpec { + + public Optional templateFor(String assertionName) { + return Optional.ofNullable(templates.get(assertionName)); + } +} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/Job.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/Job.java new file mode 100644 index 0000000000..b004ac6166 --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/Job.java @@ -0,0 +1,98 @@ +package io.stargate.sgv2.jsonapi.testbench.testspec; + +import io.stargate.sgv2.jsonapi.testbench.TestBenchPlan; +import io.stargate.sgv2.jsonapi.testbench.testrun.TestNodeFactory; +import io.stargate.sgv2.jsonapi.testbench.testrun.TestRunEnv; +import io.stargate.sgv2.jsonapi.testbench.testrun.TestUri; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.stream.Stream; +import org.junit.jupiter.api.DynamicContainer; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public record Job( + TestSpecMeta meta, + Map fromEnvironment, + Map variables, + Map> matrix, + List tests) { + + private static final Logger LOGGER = LoggerFactory.getLogger(Job.class); + + public DynamicContainer testNode(TestNodeFactory testNodeFactory, TestUri.Builder uriBuilder) { + + uriBuilder.addSegment(TestUri.Segment.JOB, meta.name()); + + var desc = "Job: %s ".formatted(meta.name()); + + testNodeFactory.testPlan().updateJobForTarget(this); + var allEnvs = allEnvironments(testNodeFactory.testPlan()); + var testSuiteNodes = + testSuites(testNodeFactory.testPlan()) + .map(testSuite -> testSuite.testNode(testNodeFactory, uriBuilder.clone(), allEnvs)) + .toList(); + + return testNodeFactory.testPlanContainer( + desc, + uriBuilder.build().uri(), + testNodeFactory.addLifecycle(uriBuilder.clone(), this, testSuiteNodes)); + } + + public Stream testSuites(TestBenchPlan testPlan) { + Stream.Builder allTests = Stream.builder(); + tests() + .forEach( + testName -> { + testPlan.specFiles().byNameAsType(TestSuiteSpec.class, testName).forEach(allTests); + }); + + return allTests.build(); + } + + public TestRunEnv withoutMatrix(TestBenchPlan testPlan) { + + var fromEnv = new TestRunEnv(); + + for (Map.Entry entry : fromEnvironment.entrySet()) { + fromEnv.put(entry.getKey(), TestEnvAccess.getEnvVar(entry.getKey())); + } + + var fromVariables = new TestRunEnv(variables); + + return fromEnv.clone().put(fromVariables); + } + + public List allEnvironments(TestBenchPlan testPlan) { + + var fromEnv = new TestRunEnv(); + + for (Map.Entry entry : fromEnvironment.entrySet()) { + fromEnv.put(entry.getKey(), TestEnvAccess.getEnvVar(entry.getValue())); + } + + var fromVariables = new TestRunEnv(variables); + + // TODO: handle more matrix + List fromMatrix = new ArrayList<>(); + matrix + .get("MODEL") + .forEach( + model -> { + var env = new TestRunEnv(); + env.put("MODEL", model); + fromMatrix.add(env); + }); + + List allEnvs = new ArrayList<>(); + for (var matrixEnv : fromMatrix) { + + var completeEnv = fromEnv.clone().put(fromVariables).put(matrixEnv); + + allEnvs.add(completeEnv); + } + + return allEnvs; + } +} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/SpecFile.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/SpecFile.java new file mode 100644 index 0000000000..c515b927f1 --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/SpecFile.java @@ -0,0 +1,24 @@ +package io.stargate.sgv2.jsonapi.testbench.testspec; + +import com.fasterxml.jackson.databind.JsonNode; +import java.io.File; +import org.jspecify.annotations.NonNull; + +/** + * A Spec file is any JSON file we have that is used to drive the test suite, workflows etc. + * + *

This is a container of the file, it's raw JSON, and the {@link TestSpec} which is the common + * parsed object from the JSON. + */ +public record SpecFile(File file, TestSpec spec, JsonNode root) { + + @Override + public @NonNull String toString() { + return new StringBuilder("SpecFile{") + .append("file=") + .append(file) + .append("spec.meta=") + .append(spec.meta()) + .toString(); + } +} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/SpecFiles.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/SpecFiles.java new file mode 100644 index 0000000000..0f6e2a0798 --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/SpecFiles.java @@ -0,0 +1,143 @@ +package io.stargate.sgv2.jsonapi.testbench.testspec; + +import com.fasterxml.jackson.databind.MapperFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.net.URISyntaxException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.List; +import java.util.function.Predicate; +import java.util.stream.Stream; + +/** + * Collection of all the {@link io.stargate.sgv2.jsonapi.testbench.testspec.SpecFile} we have loaded + * from disk. + * + *

Call {@link #loadAll(List)} to load spec files from multiple directories, + */ +public class SpecFiles { + + private static final ObjectMapper MAPPER = + new ObjectMapper().configure(MapperFeature.ACCEPT_CASE_INSENSITIVE_ENUMS, true); + + private final List specFiles; + + private SpecFiles(List specFiles) { + this.specFiles = specFiles; + + for (SpecFile file : specFiles) { + if (file.spec() instanceof TestSuiteSpec it) { + // expand the includes + it.expand(this); + } + } + } + + /** Loads all the spec file, paths is a list of resource dirs in the jar. */ + public static SpecFiles loadAll(List paths) { + + var specFiles = resourceDirs(paths).flatMap(SpecFiles::loadAll).toList(); + return new SpecFiles(specFiles); + } + + /** Get all the SpecFiles by their metadata kind */ + public Stream byKind(TestSpecKind kind) { + return match(kind, x -> true); + } + + /** + * Get all the SpecFiles by the class of the Specification, the object that implements {@link + * TestSpec} + */ + public Stream byType(Class clazz) { + return match(TestSpecKind.fromType(clazz), x -> true) + .map(specFile -> specFile.spec().asSpecType(clazz)); + } + + /** + * Get all the spec files of the type matched by name, e.g. get all the test-suites called + * "monkey" + */ + public Stream byNameAsType(Class clazz, String name) { + return match(TestSpecKind.fromType(clazz), specFiles -> specFiles.meta().name().equals(name)) + .map(specFile -> specFile.spec().asSpecType(clazz)); + } + + private Stream match(TestSpecKind kind, Predicate predicate) { + return specFiles.stream() + .filter(itFile -> itFile.spec().meta().kind() == kind) + .filter(specFile -> predicate.test(specFile.spec())); + } + + /** Load all the spec files in the directory at the path */ + private static Stream loadAll(Path path) { + + try (Stream pathStream = Files.walk(path)) { + return pathStream + .filter(Files::isRegularFile) + .filter(SpecFiles::isJsonFile) + .map(SpecFiles::loadOne) + .toList() // force so the files are read before closing + .stream(); + } catch (IOException e) { + throw new UncheckedIOException("Failed reading test resources under: " + path, e); + } + } + + private static boolean isJsonFile(Path file) { + return file.getFileName().toString().endsWith(".json"); + } + + /** Load a single spec file denoted by path. */ + private static SpecFile loadOne(Path path) { + var file = path.toFile(); + try { + // It's always JSON + var root = MAPPER.readTree(file); + + var kindNode = root.path("meta").path("kind"); + if (!kindNode.isTextual()) { + throw new IllegalArgumentException("Missing/invalid meta.kind in " + file); + } + + var element = + switch (TestSpecKind.valueOf(kindNode.asText().toUpperCase())) { + case ASSERTION_TEMPLATE -> MAPPER.treeToValue(root, AssertionTemplateSpec.class); + case TARGETS -> MAPPER.treeToValue(root, TargetsSpec.class); + case TEST_SUITE -> MAPPER.treeToValue(root, TestSuiteSpec.class); + case WORKFLOW -> MAPPER.treeToValue(root, WorkflowSpec.class); + }; + return new SpecFile(file, element, root); + } catch (IOException e) { + throw new UncheckedIOException("Failed parsing JSON file: " + file, e); + } + } + + public static Stream resourceDirs(List paths) { + + return paths.stream().map(SpecFiles::resourceDir); + } + + public static Path resourceDir(String path) { + + var cl = Thread.currentThread().getContextClassLoader(); + String normalized = path.startsWith("/") ? path.substring(1) : path; + + var url = cl.getResource(normalized); + if (url == null) { + throw new IllegalArgumentException("Test resource folder not found: " + path); + } + + try { + // Works for file: URLs; if you run tests from a jar, switch to + // getResourceAsStream-based + // walking. + return Paths.get(url.toURI()); + } catch (URISyntaxException e) { + throw new IllegalArgumentException("Bad resource URI for: " + path + " -> " + url, e); + } + } +} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/TargetsSpec.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/TargetsSpec.java new file mode 100644 index 0000000000..9f2b64c9f4 --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/TargetsSpec.java @@ -0,0 +1,47 @@ +package io.stargate.sgv2.jsonapi.testbench.testspec; + +import com.fasterxml.jackson.databind.MapperFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import io.stargate.sgv2.jsonapi.testbench.targets.TargetConfiguration; +import java.io.IOException; +import java.nio.file.Path; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +/** Spec file that contains targets */ +public record TargetsSpec(TestSpecMeta meta, List targets) + implements TestSpec { + + private static final ObjectMapper MAPPER = + new ObjectMapper().configure(MapperFeature.ACCEPT_CASE_INSENSITIVE_ENUMS, true); + + public TargetsSpec { + Set seen = new HashSet(); + + for (TargetConfiguration target : targets) { + if (seen.contains(target.name())) { + throw new IllegalArgumentException("target name already exists: " + target.name()); + } + seen.add(target.name()); + } + } + + public TargetConfiguration getTarget(String targetName) { + return targets.stream() + .filter(target -> target.name().equals(targetName)) + .findFirst() + .orElseThrow( + () -> new IllegalArgumentException("target targetName not found: " + targetName)); + } + + public static TargetsSpec loadAll(String path) { + final Path dir = SpecFiles.resourceDir(path); + + try { + return MAPPER.readValue(dir.toFile(), TargetsSpec.class); + } catch (IOException e) { + throw new RuntimeException(e); + } + } +} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/TestCase.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/TestCase.java new file mode 100644 index 0000000000..00059dfeb4 --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/TestCase.java @@ -0,0 +1,41 @@ +package io.stargate.sgv2.jsonapi.testbench.testspec; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.node.ObjectNode; +import io.stargate.sgv2.jsonapi.testbench.assertions.TestAssertion; +import io.stargate.sgv2.jsonapi.testbench.testrun.*; +import org.junit.jupiter.api.DynamicContainer; + +/** + * Spec for a single test case in a test suite, a test case is a command to run and assertions to + * run after. + * + * @param name + * @param command + * @param asserts + * @param include + */ +public record TestCase( + String name, + TestCommand command, + ObjectNode asserts, + @JsonProperty("$include") String include) { + + public DynamicContainer testNodesForEnvironment( + TestNodeFactory testNodeFactory, + TestUri.Builder uriBuilder, + TestRunEnv testEnvironment, + TestExecutionCondition testExecutionCondition) { + + var testRequest = + new TestRunRequest( + "TestCase: name=%s".formatted(name, command.commandName()), + command(), + testNodeFactory.testPlan().target(), + testEnvironment, + TestAssertion.buildAssertions(testNodeFactory.testPlan(), this), + testExecutionCondition); + + return testRequest.testNodes(testNodeFactory, uriBuilder); + } +} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/TestCommand.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/TestCommand.java new file mode 100644 index 0000000000..56b5e76a69 --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/TestCommand.java @@ -0,0 +1,188 @@ +package io.stargate.sgv2.jsonapi.testbench.testspec; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.fasterxml.jackson.databind.node.TextNode; +import io.stargate.sgv2.jsonapi.api.model.command.CommandName; +import io.stargate.sgv2.jsonapi.testbench.testrun.TestRunEnv; +import org.apache.commons.text.StringSubstitutor; + +/** + * A API command to send, that may be part of a setup, test, cleanup, or lifecycle process. This is + * a basic command to send, without any checks etc. + * + *

This class is re-used in Spec configurations where an API command is needed. For example + * below, the TestCommand is the Objects in the setup array. The structure is then a normal Data API + * command object, with a single top level member that is the name of the command. + * + * 

+ *   "setup": [
+ *     {
+ *       "insertOne": {
+ *         "document": {
+ *           "_id": "Inception",
+ *           "name": "Inception",
+ *           "genre": "Science Fiction",
+ *           "artist": [
+ *             "Leonardo DiCaprio"
+ *           ],
+ *           "$vectorize": "Inception is a science fiction action film about a professional thief who steals information by infiltrating the subconscious, entering people's dreams. He is offered a chance to have his criminal history erased as payment for implanting another person's idea into a target's subconscious."
+ *         }
+ *       }
+ *     },
+ *     {
+ *       "insertMany": {
+ *         "documents": [....
+ *
+ */ +public class TestCommand { + + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + + private final ObjectNode request; + private final CommandName commandName; + private final String includeFrom; + + /** + * Constructor called when this class is used in a record etc that is deserialized using Jackson. + * + * @param request The JSON object to deserialize from, e.g. the objects in the array above. + */ + @JsonCreator(mode = JsonCreator.Mode.DELEGATING) + public TestCommand(ObjectNode request) { + + // if non-null, that this is a pointer to find commands in the named test-suite. + var includeField = request.get("$include"); + this.includeFrom = includeField == null ? null : includeField.asText(); + + if (includeField != null) { + this.request = null; + this.commandName = null; + } else { + this.request = request; + this.commandName = commandName(request); + } + } + + /** Get the complete request to send */ + public ObjectNode request() { + throwIfHasInclude(); + return request; + } + + /** The name of the Data API command, extracted from the definition. */ + public CommandName commandName() { + throwIfHasInclude(); + return commandName; + } + + /** + * Name of the test-suite to include command from, rather than use the definition in here. + * + *

For example, this says to include setup commands from 'vectorize-base' + * + * 

+   *   "setup": [
+   *     {
+   *       "$include": "vectorize-base"
+   *     }
+   *   ],
+   *
+ * + * @return value of the '$include' from the json + */ + public String includeFrom() { + return includeFrom; + } + + /** Build from a string, useful for lifecycle commands that are created on the fly. */ + public static TestCommand fromJson(String json) { + + try { + return new TestCommand((ObjectNode) OBJECT_MAPPER.readTree(json)); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + } + + /** + * Extracts the name of the API command from the request. + * + *

Public for re-use. + */ + public static CommandName commandName(ObjectNode request) { + var requestCommandName = commandNameString(request); + for (CommandName name : CommandName.values()) { + if (name.getApiName().equals(requestCommandName)) { + return name; + } + } + throw new IllegalArgumentException("Unknown command name: " + requestCommandName); + } + + public ObjectNode withEnvironment(TestRunEnv env) { + ObjectNode updated = request.deepCopy(); + walk(updated, env.substitutor()); + return updated; + } + + private void throwIfHasInclude() { + if (includeFrom != null) { + throw new IllegalStateException("TestCommand is defined to $include from: " + includeFrom); + } + } + + private static String commandNameString(ObjectNode request) { + var it = request.fieldNames(); + if (!it.hasNext()) { + throw new IllegalStateException("Expected exactly one field, found none"); + } + String name = it.next(); + if (it.hasNext()) { + throw new IllegalStateException("Expected exactly one field, found multiple"); + } + return name; + } + + private static void walk(ObjectNode obj, StringSubstitutor subs) { + obj.properties() + .forEach( + (entry) -> { + switch (entry.getValue()) { + case TextNode text -> { + obj.put(entry.getKey(), subs.replace(text.textValue())); + } + case ObjectNode nested -> { + walk(nested, subs); + } + case ArrayNode arr -> { + walk(arr, subs); + } + default -> {} + } + }); + } + + private static void walk(ArrayNode arr, StringSubstitutor subs) { + for (int i = 0; i < arr.size(); i++) { + var child = arr.get(i); + + switch (child) { + case TextNode text -> { + String value = subs.replace(text.textValue()); + arr.set(i, TextNode.valueOf(value)); + } + case ObjectNode nested -> { + walk(nested, subs); + } + case ArrayNode nestedArr -> { + walk(nestedArr, subs); + } + default -> {} + } + } + } +} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/TestEnvAccess.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/TestEnvAccess.java new file mode 100644 index 0000000000..889d552f67 --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/TestEnvAccess.java @@ -0,0 +1,22 @@ +package io.stargate.sgv2.jsonapi.testbench.testspec; + +public abstract class TestEnvAccess { + + public static void putEnvVar(String key, String value) { + System.getProperties().put(key, value); + } + + public static String getEnvVar(String varName) { + + var value = System.getProperty(varName); + if (value != null) { + return value; + } + value = System.getenv(varName); + if (value == null) { + throw new RuntimeException( + "Environment variable not found in System Properties or Environment. varName=" + varName); + } + return value; + } +} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/TestSpec.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/TestSpec.java new file mode 100644 index 0000000000..8b9a07a44a --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/TestSpec.java @@ -0,0 +1,31 @@ +package io.stargate.sgv2.jsonapi.testbench.testspec; + +import io.stargate.sgv2.jsonapi.testbench.TestBenchPlan; + +/** + * A specification for objects in the TestBench world, such as a test suite. + * + *

Implement this for any object types a {@link TestBenchPlan} needs to read from disk or know + * about. + */ +public sealed interface TestSpec + permits AssertionTemplateSpec, TargetsSpec, TestSuiteSpec, WorkflowSpec { + + TestSpecMeta meta(); + + /** + * Gets the object as the implementing class + * + * @param type the implementing class of the {@link TestSpec} + * @return The implementing object, as the type. + * @param the implementing class of the {@link TestSpec} + */ + default T asSpecType(Class type) { + + if (!type.isInstance(this)) { + throw new IllegalArgumentException( + "TestSpec is not of required type. expected=%s, spec.meta=%s".formatted(type, meta())); + } + return type.cast(this); + } +} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/TestSpecKind.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/TestSpecKind.java new file mode 100644 index 0000000000..55e248f62e --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/TestSpecKind.java @@ -0,0 +1,24 @@ +package io.stargate.sgv2.jsonapi.testbench.testspec; + +public enum TestSpecKind { + ASSERTION_TEMPLATE, + TARGETS, + TEST_SUITE, + WORKFLOW; + + public static TestSpecKind fromType(Class clazz) { + if (clazz == AssertionTemplateSpec.class) { + return ASSERTION_TEMPLATE; + } + if (clazz == TargetsSpec.class) { + return TARGETS; + } + if (clazz == TestSuiteSpec.class) { + return TEST_SUITE; + } + if (clazz == WorkflowSpec.class) { + return WORKFLOW; + } + throw new IllegalArgumentException("Unknown TestSpec type: " + clazz); + } +} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/TestSpecMeta.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/TestSpecMeta.java new file mode 100644 index 0000000000..2aad34d5e9 --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/TestSpecMeta.java @@ -0,0 +1,17 @@ +package io.stargate.sgv2.jsonapi.testbench.testspec; + +import java.util.List; + +/** + * Common metadata for any object of a {@link TestSpec} + * + * @param name Friendly name for, such as "vectorize-header-workflow" + * @param kind {@link TestSpecKind} describing the kind. + * @param tags Optional tags that can be used for filtering etc. + */ +public record TestSpecMeta(String name, TestSpecKind kind, List tags) { + + public TestSpecMeta { + tags = tags == null ? List.of() : tags; + } +} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/TestSuiteSpec.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/TestSuiteSpec.java new file mode 100644 index 0000000000..b877493a09 --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/TestSuiteSpec.java @@ -0,0 +1,126 @@ +package io.stargate.sgv2.jsonapi.testbench.testspec; + +import io.stargate.sgv2.jsonapi.testbench.assertions.TestAssertion; +import io.stargate.sgv2.jsonapi.testbench.testrun.*; +import java.util.ArrayList; +import java.util.List; +import org.junit.jupiter.api.DynamicContainer; +import org.junit.jupiter.api.DynamicNode; + +public record TestSuiteSpec( + TestSpecMeta meta, List setup, List tests, List cleanup) + implements TestSpec { + + public DynamicContainer testNode( + TestNodeFactory testNodeFactory, TestUri.Builder uriBuilder, List allEnvs) { + + uriBuilder.addSegment(TestUri.Segment.SUITE, meta().name()); + + var desc = "TestSuite: %s ".formatted(meta.name()); + var childs = + allEnvs.stream() + .map(testEnv -> testEnv.testNode(testNodeFactory, uriBuilder.clone(), this)) + .toList(); + return testNodeFactory.testPlanContainer(desc, uriBuilder.build().uri(), childs); + } + + public List testNodesForEnvironment( + TestNodeFactory testNodeFactory, + TestUri.Builder uriBuilder, + TestRunEnv testEnvironment, + TestExecutionCondition testExecutionCondition) { + + // not increasing the count of test nodes here, because this code is not actually making any + // test nodes, it is all in things we call, they do the increasing + List nodes = new ArrayList<>(); + + int i = 1; + var setupUriBuilder = uriBuilder.clone().addSegment(TestUri.Segment.STAGE, "setup"); + + for (TestCommand setupCommand : setup()) { + var setupRequest = + new TestRunRequest( + "SetupRequest[%s]: %s".formatted(i++, setupCommand.commandName()), + setupCommand, + testNodeFactory.testPlan().target(), + testEnvironment, + TestAssertion.forSuccess(testNodeFactory.testPlan(), setupCommand), + testExecutionCondition); + + nodes.add(setupRequest.testNodes(testNodeFactory, setupUriBuilder.clone())); + } + + var testUriBuilder = uriBuilder.clone().addSegment(TestUri.Segment.STAGE, "test"); + for (var testCase : tests()) { + nodes.add( + testCase.testNodesForEnvironment( + testNodeFactory, testUriBuilder.clone(), testEnvironment, testExecutionCondition)); + } + + // NOTE: For Cleanup we use a condition that is always TRUE because we always want to try to run + // a cleanup task. + var alwaysTrueCondition = + new TestExecutionCondition.AlwaysTrue( + "Cleanup Commands for parent URL: " + uriBuilder.build().uri().toString()); + var cleanupUriBuilder = uriBuilder.clone().addSegment(TestUri.Segment.STAGE, "cleanup"); + for (TestCommand cleanupCommand : cleanup()) { + var cleanupRequest = + new TestRunRequest( + "CleanupRequest[%s]: %s".formatted(i++, cleanupCommand.commandName()), + cleanupCommand, + testNodeFactory.testPlan().target(), + testEnvironment, + TestAssertion.forSuccess(testNodeFactory.testPlan(), cleanupCommand), + alwaysTrueCondition); + + nodes.add(cleanupRequest.testNodes(testNodeFactory, cleanupUriBuilder.clone())); + } + + return nodes; + } + + public void expand(SpecFiles specFiles) { + + List expandedSetup = new ArrayList<>(); + for (TestCommand command : setup) { + if (command.includeFrom() != null) { + var includedTest = + specFiles + .byNameAsType(TestSuiteSpec.class, command.includeFrom()) + .findFirst() + .orElseThrow( + () -> + new IllegalStateException( + "Included TestSuite Setup not found. parent=%s, included=%s" + .formatted(meta().name(), command.includeFrom()))); + + expandedSetup.addAll(includedTest.setup()); + } else { + expandedSetup.add(command); + } + } + setup.clear(); + setup.addAll(expandedSetup); + + List expandedTests = new ArrayList<>(); + for (TestCase testCase : tests) { + if (testCase.include() != null) { + var includedTest = + specFiles + .byNameAsType(TestSuiteSpec.class, testCase.include()) + .findFirst() + .orElseThrow( + () -> + new IllegalStateException( + "Included TestSuite TestCase not found. parent=%s, included=%s" + .formatted(meta().name(), testCase.include()))); + + expandedTests.addAll(includedTest.tests()); + } else { + expandedTests.add(testCase); + } + } + tests.clear(); + tests.addAll(expandedTests); + } +} diff --git a/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/WorkflowSpec.java b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/WorkflowSpec.java new file mode 100644 index 0000000000..be35a7fba8 --- /dev/null +++ b/src/test/java/io/stargate/sgv2/jsonapi/testbench/testspec/WorkflowSpec.java @@ -0,0 +1,28 @@ +package io.stargate.sgv2.jsonapi.testbench.testspec; + +import io.stargate.sgv2.jsonapi.testbench.testrun.TestNodeFactory; +import io.stargate.sgv2.jsonapi.testbench.testrun.TestUri; +import java.util.List; +import org.junit.jupiter.api.DynamicNode; + +public record WorkflowSpec(TestSpecMeta meta, List jobs) implements TestSpec { + + public DynamicNode testNode( + TestNodeFactory testNodeFactory, TestUri.Builder uriBuilder, boolean ignoreDisabled) { + + uriBuilder.addSegment(TestUri.Segment.WORKFLOW, meta().name()); + var desc = "Workflow: %s ".formatted(meta.name()); + + var testNodeJobs = + ignoreDisabled + ? jobs().stream().filter(job -> !job.meta().tags().contains("disabled")) + : jobs().stream(); + var jobNodes = + testNodeJobs.map(job -> job.testNode(testNodeFactory, uriBuilder.clone())).toList(); + + return testNodeFactory.testPlanContainer( + desc, + uriBuilder.build().uri(), + testNodeFactory.addLifecycle(uriBuilder.clone(), this, jobNodes)); + } +} diff --git a/src/test/resources/META-INF/services/org.junit.platform.launcher.TestExecutionListener b/src/test/resources/META-INF/services/org.junit.platform.launcher.TestExecutionListener new file mode 100644 index 0000000000..031e53e9c4 --- /dev/null +++ b/src/test/resources/META-INF/services/org.junit.platform.launcher.TestExecutionListener @@ -0,0 +1 @@ +io.stargate.sgv2.jsonapi.testbench.reporting.DynamicTreeListener \ No newline at end of file diff --git a/src/test/resources/application.yaml b/src/test/resources/application.yaml index 3e963be935..5fd20660f0 100644 --- a/src/test/resources/application.yaml +++ b/src/test/resources/application.yaml @@ -12,6 +12,10 @@ quarkus: log: console: format: "%-5p [%t] %d{yyyy-MM-dd HH:mm:ss,SSS} %F:%L - %m%n" + handler: + console: + PLAIN_CONSOLE: + format: "%m%n" category: # production log level for this category is DEBUG, way too noisy for tests 'io.stargate': @@ -22,3 +26,8 @@ quarkus: # Fine for JVM mode; if building native images, use index-dependency instead 'io.quarkus.deployment.steps.ReflectiveHierarchyStep': level: ERROR + 'io.stargate.sgv2.jsonapi.testbench.reporting.TestBenchConsoleWriter': + level: INFO + handlers: + - PLAIN_CONSOLE + use-parent-handlers: false diff --git a/src/test/resources/testbench/assertions/assertion-templates.json b/src/test/resources/testbench/assertions/assertion-templates.json new file mode 100644 index 0000000000..f2961c8a87 --- /dev/null +++ b/src/test/resources/testbench/assertions/assertion-templates.json @@ -0,0 +1,58 @@ +{ + "meta": { + "name": "assertions-templates", + "kind": "assertion_template" + }, + "templates": { + "isSuccess": { + "createCollection": { + "http.success": null, + "response.isDDLSuccess": null + }, + "createKeyspace": { + "http.success": null, + "response.isDDLSuccess": null + }, + "deleteCollection": { + "http.success": null, + "response.isDDLSuccess": null + }, + "dropKeyspace": { + "http.success": null, + "response.isDDLSuccess": null + }, + "find": { + "http.success": null, + "response.isFindSuccess": null + }, + "findOne": { + "http.success": null, + "response.isFindSuccess": null + }, + "insertOne": { + "http.success": null, + "response.isWriteSuccess": null + }, + "insertMany": { + "http.success": null, + "response.isWriteSuccess": null + }, + "findEmbeddingProviders": { + "http.success": null, + "response.isDDLSuccess": null + }, + "findOneAndDelete": { + "http.success": null, + "response.isFindAndSuccess": null + }, + "findOneAndReplace": { + "http.success": null, + "response.isFindAndSuccess": null + }, + "findOneAndUpdate": { + "http.success": null, + "response.isFindAndSuccess": null + } + } + } +} \ No newline at end of file diff --git a/src/test/resources/testbench/targets/targets.json b/src/test/resources/testbench/targets/targets.json new file mode 100644 index 0000000000..67c7d54535 --- /dev/null +++ b/src/test/resources/testbench/targets/targets.json @@ -0,0 +1,35 @@ +{ + "meta": { + "name": "targets", + "kind": "targets" + }, + "targets": [ + { + "name": "local", + "backend": "cassandra", + "connection": { + "domain": "http://localhost", + "port": 8181, + "basePath": "/v1" + } + }, + { + "name": "integration", + "backend": "cassandra", + "connection": { + "domain": "http://localhost", + "port": 9080, + "basePath": "/v1" + } + }, + { + "name": "astra-dev", + "backend": "astra", + "connection": { + "domain": "https://2637a6ae-d489-498a-a740-69362af254ef-us-west-2.apps.astra-dev.datastax.com", + "port": 443, + "basePath": "/api/json/v1" + } + } + ] +} \ No newline at end of file diff --git a/src/test/resources/testbench/testplans/test-plan-astra-vectorize.yaml b/src/test/resources/testbench/testplans/test-plan-astra-vectorize.yaml new file mode 100644 index 0000000000..937dd2f67c --- /dev/null +++ b/src/test/resources/testbench/testplans/test-plan-astra-vectorize.yaml @@ -0,0 +1,12 @@ +name: Test Plan - Astra - Vectorize Workflows +customTarget: + name: ${TARGET_NAME} + backend: astra + connection: + domain: ${ENDPOINT} + port: 443 + basePath: /api/json/v1 +workflows: + - vectorize-header-workflow + - vectorize-shared-workflow +ignoreDisabled: true diff --git a/src/test/resources/testbench/testsuites/findEmbeddingProviders.json b/src/test/resources/testbench/testsuites/findEmbeddingProviders.json new file mode 100644 index 0000000000..b8b2cdf0f0 --- /dev/null +++ b/src/test/resources/testbench/testsuites/findEmbeddingProviders.json @@ -0,0 +1,19 @@ +{ + "meta": { + "name": "findEmbeddingProviders", + "kind" : "test_suite" + }, + "setup": [ + ], + "tests": [ + { + "name": "findEmbeddingProviders", + "command": { + "findEmbeddingProviders": {} + }, + "asserts": { + "Templated.isSuccess": null + } + } + ] +} \ No newline at end of file diff --git a/src/test/resources/testbench/testsuites/vectorize-base.json b/src/test/resources/testbench/testsuites/vectorize-base.json new file mode 100644 index 0000000000..43f07afb92 --- /dev/null +++ b/src/test/resources/testbench/testsuites/vectorize-base.json @@ -0,0 +1,97 @@ +{ + "meta": { + "name": "vectorize-base", + "kind" : "test_suite" + }, + "setup": [ + { + "insertOne": { + "document": { + "_id": "Inception", + "name": "Inception", + "genre": "Science Fiction", + "artist": [ + "Leonardo DiCaprio" + ], + "$vectorize": "Inception is a science fiction action film about a professional thief who steals information by infiltrating the subconscious, entering people's dreams. He is offered a chance to have his criminal history erased as payment for implanting another person's idea into a target's subconscious." + } + } + }, + { + "insertMany": { + "documents": [ + { + "_id": "The Shawshank Redemption", + "name": "The Shawshank Redemption", + "genre": "Drama", + "artist": [ + "Tim Robbins", + "Morgan Freeman" + ], + "$vectorize": "The Shawshank Redemption is a drama film about a banker who is sentenced to life in Shawshank State Penitentiary for the murder of his wife and her lover. He forms a bond with a fellow prisoner and manages to maintain his dignity and sense of hope in the face of unjust treatment." + }, + { + "_id": "The Godfather", + "name": "The Godfather", + "genre": "Crime", + "artist": [ + "Marlon Brando", + "Al Pacino" + ], + "$vectorize": "The Godfather is a crime film about the aging patriarch of an organized crime dynasty who transfers control of his clandestine empire to his reluctant son. The film follows the family under the patriarch's youngest son, Michael, as he becomes increasingly involved in the family business." + } + ] + } + } + ], + "tests": [ + { + "name": "basic findMany", + "command": { + "find": { + "sort": { + "$vectorize": "I love movies!" + } + } + }, + "asserts": { + "Templated.isSuccess": null, + "Documents.count": 3 + } + }, + { + "name": "findOneAndUpdate", + "command": { + "findOneAndUpdate": { + "sort": { + "$vectorize": "Inception is a science fiction action film" + }, + "update": { + "$set": { + "status": "active" + } + }, + "options": { + "returnDocument": "after" + } + } + }, + "asserts": { + "Templated.isSuccess": null, + "Documents.isExactly": { + "_id": "Inception", + "name": "Inception", + "genre": "Science Fiction", + "artist": [ + "Leonardo DiCaprio" + ], + "status": "active" + }, + "Status.isExactly" : { + "matchedCount": 1, + "modifiedCount": 1 + } + } + } + ] +} \ No newline at end of file diff --git a/src/test/resources/testbench/testsuites/vectorize-header-auth.json b/src/test/resources/testbench/testsuites/vectorize-header-auth.json new file mode 100644 index 0000000000..2aa9cd3053 --- /dev/null +++ b/src/test/resources/testbench/testsuites/vectorize-header-auth.json @@ -0,0 +1,37 @@ +{ + "meta": { + "name": "vectorize-header-auth", + "kind": "test_suite" + }, + "setup": [ + { + "createCollection": { + "name": "${COLLECTION_NAME}", + "options": { + "vector": { + "metric": "cosine", + "service": { + "provider": "${PROVIDER}", + "modelName": "${MODEL}" + } + } + } + } + }, + { + "$include": "vectorize-base" + } + ], + "tests": [ + { + "$include": "vectorize-base" + } + ], + "cleanup": [ + { + "deleteCollection": { + "name": "${COLLECTION_NAME}" + } + } + ] +} \ No newline at end of file diff --git a/src/test/resources/testbench/testsuites/vectorize-shared-auth.json b/src/test/resources/testbench/testsuites/vectorize-shared-auth.json new file mode 100644 index 0000000000..eda94d81c9 --- /dev/null +++ b/src/test/resources/testbench/testsuites/vectorize-shared-auth.json @@ -0,0 +1,40 @@ +{ + "meta": { + "name": "vectorize-shared-auth", + "kind": "test_suite" + }, + "setup": [ + { + "createCollection": { + "name": "${COLLECTION_NAME}", + "options": { + "vector": { + "metric": "cosine", + "service": { + "provider": "${PROVIDER}", + "modelName": "${MODEL}", + "authentication":{ + "providerKey": "${CREDENTIAL}.providerKey" + } + } + } + } + } + }, + { + "$include": "vectorize-base" + } + ], + "tests": [ + { + "$include": "vectorize-base" + } + ], + "cleanup": [ + { + "deleteCollection": { + "name": "${COLLECTION_NAME}" + } + } + ] +} \ No newline at end of file diff --git a/src/test/resources/testbench/workflows/findEmbeddingProviders-workflow.json b/src/test/resources/testbench/workflows/findEmbeddingProviders-workflow.json new file mode 100644 index 0000000000..e71581803b --- /dev/null +++ b/src/test/resources/testbench/workflows/findEmbeddingProviders-workflow.json @@ -0,0 +1,30 @@ +{ + "meta": { + "name": "findEmbeddingProviders-workflow", + "kind": "workflow" + }, + "jobs": [ + { + "meta": { + "name": "findEmbeddingProviders", + "tags": [ + ] + }, + "fromEnvironment": { + "Token": "Token" + }, + "variables": { + }, + "matrix": { + "filterModelStatus": [ + "SUPPORTED", + "DEPRECATED", + "END_OF_LIFE" + ] + }, + "tests": [ + "findEmbeddingProviders" + ] + } + ] +} \ No newline at end of file diff --git a/src/test/resources/testbench/workflows/vectorize-header-workflow.json b/src/test/resources/testbench/workflows/vectorize-header-workflow.json new file mode 100644 index 0000000000..91d55da229 --- /dev/null +++ b/src/test/resources/testbench/workflows/vectorize-header-workflow.json @@ -0,0 +1,189 @@ +{ + "meta": { + "name": "vectorize-header-workflow", + "kind": "workflow" + }, + "jobs": [ + { + "meta": { + "name": "open-ai-vectorize", + "tags": [ + ] + }, + "fromEnvironment": { + "x-embedding-api-key": "OPEN_AI_KEY", + "Token": "DATA_API_TOKEN" + }, + "variables": { + "PROVIDER": "openai", + "COLLECTION_NAME": "${PROVIDER}-${MODEL}" + }, + "matrix": { + "MODEL": [ + "text-embedding-3-small", + "text-embedding-3-large", + "text-embedding-ada-002" + ] + }, + "tests": [ + "vectorize-header-auth" + ] + }, + { + "meta": { + "name": "voyageAI-vectorize", + "tags": [ + ] + }, + "fromEnvironment": { + "x-embedding-api-key": "VOYAGE_AI_KEY", + "Token": "DATA_API_TOKEN" + }, + "variables": { + "PROVIDER": "voyageAI", + "COLLECTION_NAME": "${PROVIDER}-${MODEL}" + }, + "matrix": { + "MODEL": [ + "voyage-large-2-instruct", + "voyage-law-2", + "voyage-code-2", + "voyage-large-2", + "voyage-2" + ] + }, + "tests": [ + "vectorize-header-auth" + ] + }, + { + "meta": { + "name": "jinaAI-vectorize", + "tags": [ + "disabled" + ] + }, + "fromEnvironment": { + "x-embedding-api-key": "JINA_AI_KEY", + "Token": "DATA_API_TOKEN" + }, + "variables": { + "PROVIDER": "jinaAI", + "COLLECTION_NAME": "${PROVIDER}-${MODEL}" + }, + "matrix": { + "MODEL": [ + "jina-embeddings-v2-base-en", + "jina-embeddings-v2-base-de", + "jina-embeddings-v2-base-es", + "jina-embeddings-v2-base-code", + "jina-embeddings-v2-base-zh" + ] + }, + "tests": [ + "vectorize-header-auth" + ] + }, + { + "meta": { + "name": "huggingface-non-dedicated-vectorize", + "tags": [ + + ] + }, + "fromEnvironment": { + "x-embedding-api-key": "HUGGINGFACE_KEY", + "Token": "DATA_API_TOKEN" + }, + "variables": { + "PROVIDER": "huggingface", + "COLLECTION_NAME": "${PROVIDER}-${MODEL}" + }, + "matrix": { + "MODEL": [ + "sentence-transformers/all-MiniLM-L6-v2", + "intfloat/multilingual-e5-large", + "intfloat/multilingual-e5-large-instruct", + "BAAI/bge-small-en-v1.5", + "BAAI/bge-base-en-v1.5", + "BAAI/bge-large-en-v1.5" + ] + }, + "tests": [ + "vectorize-header-auth" + ] + }, + { + "meta": { + "name": "mistral-vectorize", + "tags": [ + + ] + }, + "fromEnvironment": { + "x-embedding-api-key": "MISTRAL_KEY", + "Token": "DATA_API_TOKEN" + }, + "variables": { + "PROVIDER": "mistral", + "COLLECTION_NAME": "${PROVIDER}-${MODEL}" + }, + "matrix": { + "MODEL": [ + "mistral-embed" + ] + }, + "tests": [ + "vectorize-header-auth" + ] + }, + { + "meta": { + "name": "upstageAI-vectorize", + "tags": [ + + ] + }, + "fromEnvironment": { + "x-embedding-api-key": "UPSTAGE_AI_KEY", + "Token": "DATA_API_TOKEN" + }, + "variables": { + "PROVIDER": "upstageAI", + "COLLECTION_NAME": "${PROVIDER}-${MODEL}" + }, + "matrix": { + "MODEL": [ + "solar-embedding-1-large" + ] + }, + "tests": [ + "vectorize-header-auth" + ] + }, + { + "meta": { + "name": "nvidia-vectorize", + "tags": [ + ] + }, + "fromEnvironment": { + "x-embedding-api-key": "DATA_API_TOKEN", + "Token": "DATA_API_TOKEN" + }, + "variables": { + "PROVIDER": "nvidia", + "COLLECTION_NAME": "${PROVIDER}-${MODEL}" + }, + "matrix": { + "MODEL": [ + "NV-Embed-QA", + "nvidia/nv-embedqa-e5-v5" + ] + }, + "tests": [ + "vectorize-header-auth" + ] + } + ] +} \ No newline at end of file diff --git a/src/test/resources/testbench/workflows/vectorize-shared-workflow.json b/src/test/resources/testbench/workflows/vectorize-shared-workflow.json new file mode 100644 index 0000000000..0dd3cecb80 --- /dev/null +++ b/src/test/resources/testbench/workflows/vectorize-shared-workflow.json @@ -0,0 +1,167 @@ +{ + "meta": { + "name": "vectorize-shared-workflow", + "kind": "workflow" + }, + "jobs": [ + { + "meta": { + "name": "open-ai-vectorize", + "tags": [ + + ] + }, + "fromEnvironment": { + "Token": "DATA_API_TOKEN" + }, + "variables": { + "PROVIDER": "openai", + "COLLECTION_NAME": "${PROVIDER}-${MODEL}", + "CREDENTIAL" : "open-ai-key" + }, + "matrix": { + "MODEL": [ + "text-embedding-3-small", + "text-embedding-3-large", + "text-embedding-ada-002" + ] + }, + "tests": [ + "vectorize-shared-auth" + ] + }, + { + "meta": { + "name": "voyageAI-vectorize", + "tags": [ + ] + }, + "fromEnvironment": { + "Token": "DATA_API_TOKEN" + }, + "variables": { + "PROVIDER": "voyageAI", + "COLLECTION_NAME": "${PROVIDER}-${MODEL}", + "CREDENTIAL" : "kent-voyageai-key" + + }, + "matrix": { + "MODEL": [ + "voyage-large-2-instruct", + "voyage-law-2", + "voyage-large-2", + "voyage-2", + "voyage-code-2", + "voyage-finance-2", + "voyage-multilingual-2" + ] + }, + "tests": [ + "vectorize-shared-auth" + ] + }, + { + "meta": { + "name": "jinaAI-vectorize", + "tags": [ + "disabled" + ] + }, + "fromEnvironment": { + "Token": "DATA_API_TOKEN" + }, + "variables": { + "PROVIDER": "jinaAI", + "COLLECTION_NAME": "${PROVIDER}-${MODEL}", + "CREDENTIAL" : "aaron-jinaai-key" + }, + "matrix": { + "MODEL": [ + "jina-embeddings-v2-base-en", + "jina-embeddings-v2-base-de", + "jina-embeddings-v2-base-es", + "jina-embeddings-v2-base-code", + "jina-embeddings-v2-base-zh", + "jina-embeddings-v3" + ] + }, + "tests": [ + "vectorize-shared-auth" + ] + }, + { + "meta": { + "name": "huggingface-non-dedicated-vectorize", + "tags": [ + ] + }, + "fromEnvironment": { + "Token": "DATA_API_TOKEN" + }, + "variables": { + "PROVIDER": "huggingface", + "COLLECTION_NAME": "${PROVIDER}-${MODEL}", + "CREDENTIAL" : "yuqi-huggingface-serverless-key" + }, + "matrix": { + "MODEL": [ + "sentence-transformers/all-MiniLM-L6-v2", + "intfloat/multilingual-e5-large", + "intfloat/multilingual-e5-large-instruct", + "BAAI/bge-small-en-v1.5", + "BAAI/bge-base-en-v1.5", + "BAAI/bge-large-en-v1.5" + ] + }, + "tests": [ + "vectorize-shared-auth" + ] + }, + { + "meta": { + "name": "mistral-vectorize", + "tags": [ + ] + }, + "fromEnvironment": { + "Token": "DATA_API_TOKEN" + }, + "variables": { + "PROVIDER": "mistral", + "COLLECTION_NAME": "${PROVIDER}-${MODEL}", + "CREDENTIAL" : "kent-mistralai-key" + }, + "matrix": { + "MODEL": [ + "mistral-embed" + ] + }, + "tests": [ + "vectorize-shared-auth" + ] + }, + { + "meta": { + "name": "upstageAI-vectorize", + "tags": [ + ] + }, + "fromEnvironment": { + "Token": "DATA_API_TOKEN" + }, + "variables": { + "PROVIDER": "upstageAI", + "COLLECTION_NAME": "${PROVIDER}-${MODEL}", + "CREDENTIAL" : "kent-upstage-key" + }, + "matrix": { + "MODEL": [ + "solar-embedding-1-large" + ] + }, + "tests": [ + "vectorize-shared-auth" + ] + } + ] +} \ No newline at end of file