diff --git a/.github/actions/checkout-and-build/action.yml b/.github/actions/checkout-and-build/action.yml index a15d31b3..0a0c1919 100644 --- a/.github/actions/checkout-and-build/action.yml +++ b/.github/actions/checkout-and-build/action.yml @@ -33,4 +33,4 @@ runs: - name: Build with Maven shell: bash run: | - mvn install -Dmaven.test.skip=${{ inputs.skip_tests_on_build }} \ No newline at end of file + mvn install -Dmaven.test.skip=${{ inputs.skip_tests_on_build }} -Dgpg.skip \ No newline at end of file diff --git a/.github/workflows/_astra-integration-tests.yml b/.github/workflows/_astra-integration-tests.yml new file mode 100644 index 00000000..7f615c10 --- /dev/null +++ b/.github/workflows/_astra-integration-tests.yml @@ -0,0 +1,31 @@ +name: Run integration tests against Astra + +on: + workflow_call: + workflow_dispatch: + pull_request: # temporary until it's merged + branches: [ KG-ci-additions ] + push: + branches: [ KG-ci-additions ] + +permissions: + id-token: write + +jobs: + test-integration-astra: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v6 + + - name: Setup Java 17 + uses: actions/setup-java@v5 + with: + java-version: '17' + distribution: 'temurin' + cache: 'maven' + + - name: Run integration tests + env: + ASTRA_DB_APPLICATION_TOKEN: ${{ secrets.ASTRA_DB_APPLICATION_TOKEN_DEV }} + run: mvn clean verify -pl astra-db-java -Pastra-dev diff --git a/.github/workflows/_docs-compile-tests.yml b/.github/workflows/_docs-compile-tests.yml new file mode 100644 index 00000000..8ce700e8 --- /dev/null +++ b/.github/workflows/_docs-compile-tests.yml @@ -0,0 +1,38 @@ +name: Compile against documentation examples + +on: + workflow_call: + workflow_dispatch: +# pull_request: # temporary until it's merged +# branches: [ KG-ci-additions ] +# push: +# branches: [ KG-ci-additions ] + +jobs: + test-compilation: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v6 + + - name: Set up JDK 17 + uses: actions/setup-java@v5 + with: + java-version: 17 + distribution: 'temurin' + cache: 'maven' + + - name: Build with Maven + shell: bash + id: build + run: | + mvn install -pl astra-db-java -am -Dmaven.test.skip=true -Dgpg.skip + VERSION=$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout) + echo "version=$VERSION" >> $GITHUB_OUTPUT + + - name: Run compilation tests + uses: datastax/astra-client-docs-tests/.github/actions/test-compilation@master + with: + clients: 'java' + args: -A java='\"com.datastax.astra:astra-db-java:${{ steps.build.outputs.version }}\"' + docs-repo-token: ${{ secrets.DOC_GITHUB_PAT_CROSS_ORG }} diff --git a/.github/workflows/_hcd-integration-tests.yml b/.github/workflows/_hcd-integration-tests.yml new file mode 100644 index 00000000..77e368ea --- /dev/null +++ b/.github/workflows/_hcd-integration-tests.yml @@ -0,0 +1,42 @@ +name: Run integration tests against HCD + +on: + workflow_call: + workflow_dispatch: +# pull_request: # temporary until it's merged +# branches: [ KG-ci-additions ] +# push: +# branches: [ KG-ci-additions ] + +permissions: + id-token: write + +jobs: + test-integration-hcd: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v6 + + - name: Setup Java 17 + uses: actions/setup-java@v5 + with: + java-version: '17' + distribution: 'temurin' + cache: 'maven' + + - name: Configure AWS credentials from OIDC + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: arn:aws:iam::559669398656:role/data-api-clients-ecr-ro-user + aws-region: us-west-2 + + - name: Login to Amazon ECR + uses: aws-actions/amazon-ecr-login@v2 + + - name: Start HCD and Data API + run: docker compose -f docker-compose-hcd.yml up -d --wait + working-directory: ${{ github.workspace }}/astra-db-java/src/test/resources + + - name: Run integration tests + run: mvn clean verify -pl astra-db-java -Plocal diff --git a/.github/workflows/_unit-tests.yml b/.github/workflows/_unit-tests.yml new file mode 100644 index 00000000..3636b710 --- /dev/null +++ b/.github/workflows/_unit-tests.yml @@ -0,0 +1,35 @@ +name: Run unit tests + +on: + workflow_call: + inputs: + java_versions: + type: string + description: 'Comma-separated list of Java versions e.g. "17","21"' + required: true + workflow_dispatch: + inputs: + java_versions: + description: 'Comma-separated list of Java versions e.g. "17","21"' + default: '"17"' + required: false + +jobs: + unit-tests: + runs-on: ubuntu-latest + strategy: + matrix: + java-version: ${{ fromJSON(format('[{0}]', inputs.java_versions || '"17"')) }} + steps: + - name: Checkout code + uses: actions/checkout@v6 + + - name: Set up JDK ${{ matrix.java-version }} + uses: actions/setup-java@v5 + with: + java-version: ${{ matrix.java-version }} + distribution: 'temurin' + cache: 'maven' + + - name: Run unit tests + run: mvn clean test -Dtest='!**/*IT' -Dtest.skipped=false -pl astra-db-java -B diff --git a/.github/workflows/ci-astra-col-db-dbadmin.yaml b/.github/workflows/legacy/ci-astra-col-db-dbadmin.yaml similarity index 100% rename from .github/workflows/ci-astra-col-db-dbadmin.yaml rename to .github/workflows/legacy/ci-astra-col-db-dbadmin.yaml diff --git a/.github/workflows/ci-astra-dev-collection.yml b/.github/workflows/legacy/ci-astra-dev-collection.yml similarity index 100% rename from .github/workflows/ci-astra-dev-collection.yml rename to .github/workflows/legacy/ci-astra-dev-collection.yml diff --git a/.github/workflows/ci-astra-dev-database.yml b/.github/workflows/legacy/ci-astra-dev-database.yml similarity index 100% rename from .github/workflows/ci-astra-dev-database.yml rename to .github/workflows/legacy/ci-astra-dev-database.yml diff --git a/.github/workflows/ci-astra-dev-databaseadmin.yml b/.github/workflows/legacy/ci-astra-dev-databaseadmin.yml similarity index 100% rename from .github/workflows/ci-astra-dev-databaseadmin.yml rename to .github/workflows/legacy/ci-astra-dev-databaseadmin.yml diff --git a/.github/workflows/ci-astra-dev-devops.yml b/.github/workflows/legacy/ci-astra-dev-devops.yml similarity index 100% rename from .github/workflows/ci-astra-dev-devops.yml rename to .github/workflows/legacy/ci-astra-dev-devops.yml diff --git a/.github/workflows/ci-astra-dev-new.yml b/.github/workflows/legacy/ci-astra-dev-new.yml similarity index 100% rename from .github/workflows/ci-astra-dev-new.yml rename to .github/workflows/legacy/ci-astra-dev-new.yml diff --git a/.github/workflows/ci-astra-dev-vectorize-aws-bedrock.yml b/.github/workflows/legacy/ci-astra-dev-vectorize-aws-bedrock.yml similarity index 100% rename from .github/workflows/ci-astra-dev-vectorize-aws-bedrock.yml rename to .github/workflows/legacy/ci-astra-dev-vectorize-aws-bedrock.yml diff --git a/.github/workflows/ci-astra-dev-vectorize-hf-dedicated.yml b/.github/workflows/legacy/ci-astra-dev-vectorize-hf-dedicated.yml similarity index 100% rename from .github/workflows/ci-astra-dev-vectorize-hf-dedicated.yml rename to .github/workflows/legacy/ci-astra-dev-vectorize-hf-dedicated.yml diff --git a/.github/workflows/ci-astra-dev.yml b/.github/workflows/legacy/ci-astra-dev.yml similarity index 100% rename from .github/workflows/ci-astra-dev.yml rename to .github/workflows/legacy/ci-astra-dev.yml diff --git a/.github/workflows/ci-astra-devops.yaml b/.github/workflows/legacy/ci-astra-devops.yaml similarity index 100% rename from .github/workflows/ci-astra-devops.yaml rename to .github/workflows/legacy/ci-astra-devops.yaml diff --git a/.github/workflows/ci-astra-prod-devops.yml b/.github/workflows/legacy/ci-astra-prod-devops.yml similarity index 100% rename from .github/workflows/ci-astra-prod-devops.yml rename to .github/workflows/legacy/ci-astra-prod-devops.yml diff --git a/.github/workflows/ci-astra-vectorize-bedrock-hf.yaml b/.github/workflows/legacy/ci-astra-vectorize-bedrock-hf.yaml similarity index 100% rename from .github/workflows/ci-astra-vectorize-bedrock-hf.yaml rename to .github/workflows/legacy/ci-astra-vectorize-bedrock-hf.yaml diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 00000000..5bd1a4a3 --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,48 @@ +name: Tests + +on: + merge_group: + pull_request: + +# alias act='act --container-architecture linux/amd64 -P ubuntu-latest=quay.io/jamezp/act-maven' + +jobs: + run-unit-tests: + name: Run unit tests + uses: ./.github/workflows/_unit-tests.yml +# with: +# java_versions: ${{ github.event_name == 'merge_group' && '"17","21"' || '"17"' }} + + run-docs-compilation-tests: + name: Run docs compilation tests + needs: [run-unit-tests] + if: github.event_name == 'merge_group' + uses: ./.github/workflows/_docs-compile-tests.yml + secrets: inherit + + run-hcd-integration-tests: + name: Run HCD integration tests + needs: [run-unit-tests] + if: github.event_name == 'merge_group' + uses: ./.github/workflows/_hcd-integration-tests.yml + secrets: inherit + + can-enqueue: + name: Can enqueue + needs: [run-unit-tests] + if: always() && github.event_name != 'merge_group' + runs-on: ubuntu-latest + steps: + - name: Check outcomes + run: | + echo '${{ toJSON(needs) }}' | jq -e 'to_entries | map(.value.result == "success" or .value.result == "skipped") | all' + + can-merge: + name: Can merge + needs: [run-unit-tests, run-docs-compilation-tests, run-hcd-integration-tests] + if: always() && github.event_name == 'merge_group' + runs-on: ubuntu-latest + steps: + - name: Check outcomes + run: | + echo '${{ toJSON(needs) }}' | jq -e 'to_entries | map(.value.result == "success" or .value.result == "skipped") | all' diff --git a/.gitignore b/.gitignore index f6250b89..c4f6c06c 100644 --- a/.gitignore +++ b/.gitignore @@ -32,4 +32,5 @@ test-config-astra.properties test-config-embedding-providers.properties release.properties -pom.xml.releaseBackup \ No newline at end of file +pom.xml.releaseBackup +.bob diff --git a/astra-db-java/src/test/resources/docker-compose-hcd.yml b/astra-db-java/src/test/resources/docker-compose-hcd.yml new file mode 100644 index 00000000..48b38a3c --- /dev/null +++ b/astra-db-java/src/test/resources/docker-compose-hcd.yml @@ -0,0 +1,56 @@ +version: '2' + +services: + hcd: + image: 559669398656.dkr.ecr.us-west-2.amazonaws.com/engops-shared/hcd/prod/hcd:1.2.3 + networks: + - stargate + mem_limit: 2G + environment: + - MAX_HEAP_SIZE=1536M + - CLUSTER_NAME=hcd-1.2.3-cluster + - DS_LICENSE=accept + - HCD_AUTO_CONF_OFF=xcassandra.yaml + - JVM_EXTRA_OPTS=-Dcassandra.sai.latest.version=ec + volumes: + - ./cassandra-hcd.yaml:/opt/hcd/resources/cassandra/conf/xcassandra.yaml:rw + ports: + - "9042:9042" + healthcheck: + test: [ "CMD-SHELL", "cqlsh -u cassandra -p cassandra -e 'describe keyspaces'" ] + interval: 15s + timeout: 10s + retries: 20 + + data-api: + image: stargateio/data-api:v1 + depends_on: + hcd: + condition: service_healthy + networks: + - stargate + ports: + - "8181:8181" + mem_limit: 2G + environment: + - JAVA_MAX_MEM_RATIO=75 + - JAVA_INITIAL_MEM_RATIO=50 + - STARGATE_DATA_STORE_IGNORE_BRIDGE=true + - GC_CONTAINER_OPTIONS=-XX:+UseG1GC + - STARGATE_JSONAPI_OPERATIONS_DATABASE_CONFIG_CASSANDRA_END_POINTS=hcd + - STARGATE_JSONAPI_OPERATIONS_DATABASE_CONFIG_LOCAL_DATACENTER=dc1 + - STARGATE_DATABASE_LIMITS_MAX_COLLECTIONS=50 + - QUARKUS_HTTP_ACCESS_LOG_ENABLED=FALSE + - QUARKUS_LOG_LEVEL=INFO + - STARGATE_JSONAPI_OPERATIONS_VECTORIZE_ENABLED=true + - STARGATE_FEATURE_FLAGS_TABLES=true + - STARGATE_FEATURE_FLAGS_RERANKING=true + - JAVA_OPTS_APPEND=-Dquarkus.http.host=0.0.0.0 -Djava.util.logging.manager=org.jboss.logmanager.LogManager + healthcheck: + test: curl -f http://localhost:8181/stargate/health || exit 1 + interval: 5s + timeout: 10s + retries: 10 + +networks: + stargate: