diff --git a/.github/workflows/docs-pr.yml b/.github/workflows/docs-pr.yml index 8e761c058bd..e5cac1aa9f8 100644 --- a/.github/workflows/docs-pr.yml +++ b/.github/workflows/docs-pr.yml @@ -41,4 +41,4 @@ jobs: run: make -C docs setupenv - name: Build docs - run: make -C docs test \ No newline at end of file + run: make -C docs test diff --git a/.github/workflows/tests@v1.yml b/.github/workflows/tests@v1.yml index 0153b38c4d7..22116628fb2 100644 --- a/.github/workflows/tests@v1.yml +++ b/.github/workflows/tests@v1.yml @@ -27,77 +27,126 @@ on: - ".gitignore" workflow_dispatch: +permissions: + contents: read + checks: write + +env: + CI: true + jobs: build: name: Build runs-on: ubuntu-latest timeout-minutes: 10 - strategy: - matrix: - java-version: [8] - fail-fast: false + env: + JAVA_VERSION: '8' steps: - name: Checkout source uses: actions/checkout@v5 - - name: Set up JDK ${{ matrix.java-version }} + - name: Set up JDK ${{ env.JAVA_VERSION }} uses: actions/setup-java@v5 with: - java-version: ${{ matrix.java-version }} + java-version: ${{ env.JAVA_VERSION }} distribution: 'temurin' - - name: Get POM hash - id: get-pom-hash - run: echo "value=${{ hashFiles('**/pom.xml') }}" >> "$GITHUB_OUTPUT" - - name: Restore maven repository cache uses: actions/cache/restore@v4 id: java-cache with: path: ~/.m2/repository - key: ${{ runner.os }}-${{ matrix.java-version }}-maven-${{ steps.get-pom-hash.outputs.value }} + key: v3-${{ runner.os }}-${{ env.JAVA_VERSION }}-maven-${{ hashFiles('**/pom.xml') }} + restore-keys: | + v3-${{ runner.os }}-${{ env.JAVA_VERSION }}-maven- - - name: Compile source and tests - run: make compile-all + - name: Create settings-security.xml + run: echo '' > ~/.m2/settings-security.xml + + - name: Install all modules + run: make install-all + env: + MAVEN_OFFLINE_FLAG: '' - name: Download test dependencies - if: steps.java-cache.outputs.cache-hit != 'true' run: make download-all-dependencies + env: + MAVEN_OFFLINE_FLAG: '' # Override CI offline mode to allow fetching dependencies - name: Save maven repository cache uses: actions/cache/save@v4 - if: steps.java-cache.outputs.cache-hit != 'true' with: path: ~/.m2/repository - key: ${{ runner.os }}-${{ matrix.java-version }}-maven-${{ steps.get-pom-hash.outputs.value }} + key: ${{ steps.java-cache.outputs.cache-primary-key }} + + - name: Upload build targets + uses: actions/upload-artifact@v4 + with: + name: build-targets + path: | + */target/ + */*/target/ + retention-days: 3 + + - name: Upload scylladb JARs + uses: actions/upload-artifact@v4 + with: + name: scylladb-jars + path: ~/.m2/repository/com/scylladb/ + retention-days: 3 verify: name: Full verify runs-on: ubuntu-latest + needs: [build] timeout-minutes: 10 - strategy: - matrix: - java-version: [8] - fail-fast: false + env: + JAVA_VERSION: '8' steps: - name: Checkout source uses: actions/checkout@v5 - - name: Set up JDK ${{ matrix.java-version }} + - name: Set up JDK ${{ env.JAVA_VERSION }} uses: actions/setup-java@v5 with: - java-version: ${{ matrix.java-version }} + java-version: ${{ env.JAVA_VERSION }} distribution: 'temurin' - name: Restore maven repository cache uses: actions/cache/restore@v4 + id: java-cache with: path: ~/.m2/repository - key: ${{ runner.os }}-${{ matrix.java-version }}-maven-${{ hashFiles('**/pom.xml') }} + key: v3-${{ runner.os }}-${{ env.JAVA_VERSION }}-maven-${{ hashFiles('**/pom.xml') }} + restore-keys: | + v3-${{ runner.os }}-${{ env.JAVA_VERSION }}-maven- + + - name: Verify maven cache was restored + if: steps.java-cache.outputs.cache-matched-key == '' + run: | + echo "::error::Maven repository cache was not found. This can happen when the GitHub Actions cache is evicted (10 GB limit per repository). Re-run all jobs (not just failed jobs) so the build job repopulates the cache." + exit 1 + + - name: Download build targets + uses: actions/download-artifact@v4 + with: + name: build-targets + + - name: Download scylladb JARs + uses: actions/download-artifact@v4 + with: + name: scylladb-jars + path: ~/.m2/repository/com/scylladb/ + + - name: Touch build targets to prevent recompilation + run: find . -path '*/target/*' -type f -exec touch {} + + + - name: Create settings-security.xml + run: echo '' > ~/.m2/settings-security.xml - name: Full verify run: make check @@ -105,28 +154,53 @@ jobs: unit-tests: name: Unit tests runs-on: ubuntu-latest + needs: [build] timeout-minutes: 10 - strategy: - matrix: - java-version: [8] - fail-fast: false + env: + JAVA_VERSION: '8' steps: - name: Checkout source uses: actions/checkout@v5 - - name: Set up JDK 8 + - name: Set up JDK ${{ env.JAVA_VERSION }} uses: actions/setup-java@v5 with: - java-version: ${{ matrix.java-version }} + java-version: ${{ env.JAVA_VERSION }} distribution: 'temurin' - name: Restore maven repository cache uses: actions/cache/restore@v4 + id: java-cache with: path: ~/.m2/repository - key: ${{ runner.os }}-${{ matrix.java-version }}-maven-${{ hashFiles('**/pom.xml') }} + key: v3-${{ runner.os }}-${{ env.JAVA_VERSION }}-maven-${{ hashFiles('**/pom.xml') }} + restore-keys: | + v3-${{ runner.os }}-${{ env.JAVA_VERSION }}-maven- + + - name: Verify maven cache was restored + if: steps.java-cache.outputs.cache-matched-key == '' + run: | + echo "::error::Maven repository cache was not found. This can happen when the GitHub Actions cache is evicted (10 GB limit per repository). Re-run all jobs (not just failed jobs) so the build job repopulates the cache." + exit 1 + + - name: Download build targets + uses: actions/download-artifact@v4 + with: + name: build-targets + + - name: Download scylladb JARs + uses: actions/download-artifact@v4 + with: + name: scylladb-jars + path: ~/.m2/repository/com/scylladb/ + + - name: Touch build targets to prevent recompilation + run: find . -path '*/target/*' -type f -exec touch {} + + + - name: Create settings-security.xml + run: echo '' > ~/.m2/settings-security.xml - name: Run unit tests run: make test-unit @@ -135,8 +209,8 @@ jobs: if: always() run: | shopt -s globstar - mkdir unit - cp --parents ./**/target/*-reports/*.xml unit/ + mkdir -p unit + cp --parents ./**/target/*-reports/*.xml unit/ || true - name: Upload test results uses: actions/upload-artifact@v4 @@ -157,24 +231,10 @@ jobs: updateComment: false skip_annotations: true - setup-integration-tests: - name: Setup ITs - runs-on: ubuntu-latest - timeout-minutes: 2 - - steps: - - name: Checkout source - uses: actions/checkout@v5 - - - name: Setup Python 3 - uses: actions/setup-python@v6 - with: - python-version: '3.13' - cassandra-integration-tests: name: Cassandra ITs runs-on: ubuntu-latest - needs: [setup-integration-tests] + needs: [build] timeout-minutes: 90 strategy: @@ -196,9 +256,35 @@ jobs: - name: Restore maven repository cache uses: actions/cache/restore@v4 + id: java-cache with: path: ~/.m2/repository - key: ${{ runner.os }}-${{ matrix.java-version }}-maven-${{ hashFiles('**/pom.xml') }} + key: v3-${{ runner.os }}-${{ matrix.java-version }}-maven-${{ hashFiles('**/pom.xml') }} + restore-keys: | + v3-${{ runner.os }}-${{ matrix.java-version }}-maven- + + - name: Verify maven cache was restored + if: steps.java-cache.outputs.cache-matched-key == '' + run: | + echo "::error::Maven repository cache was not found. This can happen when the GitHub Actions cache is evicted (10 GB limit per repository). Re-run all jobs (not just failed jobs) so the build job repopulates the cache." + exit 1 + + - name: Download build targets + uses: actions/download-artifact@v4 + with: + name: build-targets + + - name: Download scylladb JARs + uses: actions/download-artifact@v4 + with: + name: scylladb-jars + path: ~/.m2/repository/com/scylladb/ + + - name: Touch build targets to prevent recompilation + run: find . -path '*/target/*' -type f -exec touch {} + + + - name: Create settings-security.xml + run: echo '' > ~/.m2/settings-security.xml - name: Setup Python 3 uses: actions/setup-python@v6 @@ -234,6 +320,9 @@ jobs: path: ~/.ccm/repository key: ccm-cassandra-${{ runner.os }}-${{ steps.cassandra-version.outputs.value }} + - name: Install Scylla CCM for running Cassandra + run: make install-scylla-ccm + - name: Determine test skip args id: test-skip-args run: | @@ -273,7 +362,7 @@ jobs: - name: Parse test results uses: mikepenz/action-junit-report@v5 - if: always() + if: steps.run-integration-tests.outcome != 'skipped' with: check_name: Integration tests report for Cassandra ${{ steps.cassandra-version.outputs.value }} (${{ matrix.test-group }}) require_tests: true @@ -286,7 +375,7 @@ jobs: scylla-integration-tests: name: Scylla ITs runs-on: ubuntu-latest - needs: [setup-integration-tests] + needs: [build] timeout-minutes: 90 strategy: @@ -308,9 +397,35 @@ jobs: - name: Restore maven repository cache uses: actions/cache/restore@v4 + id: java-cache with: path: ~/.m2/repository - key: ${{ runner.os }}-${{ matrix.java-version }}-maven-${{ hashFiles('**/pom.xml') }} + key: v3-${{ runner.os }}-${{ matrix.java-version }}-maven-${{ hashFiles('**/pom.xml') }} + restore-keys: | + v3-${{ runner.os }}-${{ matrix.java-version }}-maven- + + - name: Verify maven cache was restored + if: steps.java-cache.outputs.cache-matched-key == '' + run: | + echo "::error::Maven repository cache was not found. This can happen when the GitHub Actions cache is evicted (10 GB limit per repository). Re-run all jobs (not just failed jobs) so the build job repopulates the cache." + exit 1 + + - name: Download build targets + uses: actions/download-artifact@v4 + with: + name: build-targets + + - name: Download scylladb JARs + uses: actions/download-artifact@v4 + with: + name: scylladb-jars + path: ~/.m2/repository/com/scylladb/ + + - name: Touch build targets to prevent recompilation + run: find . -path '*/target/*' -type f -exec touch {} + + + - name: Create settings-security.xml + run: echo '' > ~/.m2/settings-security.xml - name: Setup Python 3 uses: actions/setup-python@v6 @@ -322,6 +437,8 @@ jobs: - name: Get scylla version id: scylla-version + env: + SCYLLA_VERSION: ${{ matrix.scylla-version }} run: make resolve-scylla-version - name: Pull CCM image from the cache @@ -383,7 +500,7 @@ jobs: - name: Parse test results uses: mikepenz/action-junit-report@v5 - if: always() + if: steps.run-integration-tests.outcome != 'skipped' with: check_name: Integration tests report for Scylla ${{ steps.scylla-version.outputs.value }} (${{ matrix.test-group }}) require_tests: true diff --git a/Makefile b/Makefile index 71688e22a33..d42a0ef1b9c 100644 --- a/Makefile +++ b/Makefile @@ -12,7 +12,26 @@ CCM_SCYLLA_REPO ?= github.com/scylladb/scylla-ccm CCM_SCYLLA_VERSION ?= master SCYLLA_EXT_OPTS ?= --smp=2 --memory=4G -MVNCMD ?= mvn -B -X -ntp +ifdef CI +# In CI, the build job runs `make install-all` once to compile, package, and install +# all modules into .m2/repository. Downstream jobs restore .m2 from cache and target/ +# dirs from artifact upload — no compilation prerequisites needed for test targets. + +MAVEN_OFFLINE_FLAG ?= -o +MAVEN_DEBUG_FLAG := +GUAVA_SHADED_DEP := +INSTALL_ALL_DEP := +PREPARE_CCM_DEP := +MAVEN_IT_PL_ARGS ?= -pl integration-tests +else +MAVEN_OFFLINE_FLAG ?= +MAVEN_DEBUG_FLAG = -X +GUAVA_SHADED_DEP := .install-guava-shaded +INSTALL_ALL_DEP := .install-all-modules +PREPARE_CCM_DEP = .prepare-cassandra-ccm +MAVEN_IT_PL_ARGS ?= +endif +MVNCMD ?= mvn -B $(MAVEN_DEBUG_FLAG) $(MAVEN_OFFLINE_FLAG) -ntp GET_VERSION_VERSION ?= 0.4.3 @@ -39,8 +58,14 @@ export PATH := $(MAKEFILE_PATH)/bin:$(PATH) .install-all-modules: $(MVNCMD) install -DskipTests -Dfmt.skip=true -Dclirr.skip=true -Danimal.sniffer.skip=true +# dependency:go-offline doesn't resolve surefire providers (surefire-junit4, surefire-testng) +# because they are lazy-loaded at test runtime. We explicitly download them for offline mode. +SUREFIRE_VERSION := $(shell grep '' pom.xml | sed 's/.*\(.*\)<\/surefire.version>.*/\1/') .download-test-dependencies: - $(MVNCMD) test -Dtest=TestThatDoesNotExists -Dfmt.skip=true -Dclirr.skip=true -Danimal.sniffer.skip=true || true + $(MVNCMD) dependency:go-offline -Dfmt.skip=true -Dclirr.skip=true -Danimal.sniffer.skip=true || true + $(MVNCMD) dependency:get -Dartifact=org.apache.maven.surefire:surefire-junit4:$(SUREFIRE_VERSION) -Dtransitive=true || true + $(MVNCMD) dependency:get -Dartifact=org.apache.maven.surefire:surefire-junit47:$(SUREFIRE_VERSION) -Dtransitive=true || true + $(MVNCMD) dependency:get -Dartifact=org.apache.maven.surefire:surefire-testng:$(SUREFIRE_VERSION) -Dtransitive=true || true .download-verify-dependencies: $(MVNCMD) verify -DskipTests || true @@ -82,6 +107,7 @@ export PATH := $(MAKEFILE_PATH)/bin:$(PATH) install-cassandra-ccm: @echo "Install CCM ${CCM_CASSANDRA_VERSION}" + pip install "setuptools<81" pip install "git+https://${CCM_CASSANDRA_REPO}.git@${CCM_CASSANDRA_VERSION}" mkdir ${CCM_CONFIG_DIR} 2>/dev/null || true echo CASSANDRA > ${CCM_CONFIG_DIR}/ccm-type @@ -94,7 +120,7 @@ install-scylla-ccm: echo SCYLLA > ${CCM_CONFIG_DIR}/ccm-type echo ${CCM_SCYLLA_VERSION} > ${CCM_CONFIG_DIR}/ccm-version -download-all-dependencies: compile-all .download-test-dependencies .download-verify-dependencies +download-all-dependencies: .download-test-dependencies .download-verify-dependencies CASSANDRA_VERSION_FILE=/tmp/cassandra-version-${CASSANDRA_VERSION}.resolved resolve-cassandra-version: .prepare-get-version @@ -139,7 +165,7 @@ resolve-scylla-version: .prepare-get-version elif [[ "${SCYLLA_VERSION}" == "LTS-PRIOR" ]]; then SCYLLA_VERSION_RESOLVED=$$(get-version --source dockerhub-imagetag --repo scylladb/scylla -filters "^[0-9]{4}$$.^[0-9]+$$.^[0-9]+$$ and LAST-1.1.LAST" | tr -d '\"') if [[ -z "$${SCYLLA_VERSION_RESOLVED}" ]]; then - SCYLLA_VERSION_RESOLVED=$$(get-version --source dockerhub-imagetag --repo scylladb/scylla-enterprise -filters "^[0-9]{4}$.^[0-9]+$.^[0-9]+$ and LAST-1.1.LAST" | tr -d '\"') + SCYLLA_VERSION_RESOLVED=$$(get-version --source dockerhub-imagetag --repo scylladb/scylla-enterprise -filters "^[0-9]{4}$$.^[0-9]+$$.^[0-9]+$$ and LAST-1.1.LAST" | tr -d '\"') fi elif [[ "${SCYLLA_VERSION}" == "LATEST" ]]; then SCYLLA_VERSION_RESOLVED=$$(get-version --source dockerhub-imagetag --repo scylladb/scylla -filters "^[0-9]{4}$$.^[0-9]+$$.^[0-9]+$$ and LAST.LAST.LAST" | tr -d '\"') @@ -171,7 +197,7 @@ checkout-one-commit-before: git tag -d ${RELEASE_TARGET_TAG} fi -download-cassandra: .prepare-scylla-ccm resolve-cassandra-version +download-cassandra: $(PREPARE_CCM_DEP) resolve-cassandra-version @if [[ -z "$${CASSANDRA_VERSION_RESOLVED}" ]]; then CASSANDRA_VERSION_RESOLVED=$$(cat '${CASSANDRA_VERSION_FILE}') fi @@ -241,19 +267,25 @@ release-dry-run: .require-release-env mkdir /tmp/java-driver-release-logs/ 2>/dev/null || true $(MVNCMD) release:perform > >(tee /tmp/java-driver-release-logs/stdout.log) 2> >(tee /tmp/java-driver-release-logs/stderr.log) -compile-all: .install-guava-shaded - mvn -B compile test-compile -Dfmt.skip=true -Dclirr.skip=true -Danimal.sniffer.skip=true +# Full install: builds all modules and installs JARs to .m2/repository. +# In CI, the build job calls this once; downstream jobs skip it and rely on +# cached .m2/repository + uploaded target/ dirs from the build job. +install-all: .install-all-modules + +compile-all: $(GUAVA_SHADED_DEP) + $(MVNCMD) compile test-compile -Dfmt.skip=true -Dclirr.skip=true -Danimal.sniffer.skip=true +# Full verification: runs all plugins (fmt, clirr, animal-sniffer) that install-all skips. check: $(MVNCMD) verify -DskipTests fix: $(MVNCMD) fmt:format -test-unit: .install-guava-shaded +test-unit: $(GUAVA_SHADED_DEP) $(MVNCMD) test -Dfmt.skip=true -Dclirr.skip=true -Danimal.sniffer.skip=true -test-integration-scylla: .install-all-modules .prepare-scylla-ccm resolve-scylla-version .prepare-environment-update-aio-max-nr +test-integration-scylla: $(INSTALL_ALL_DEP) .prepare-scylla-ccm resolve-scylla-version .prepare-environment-update-aio-max-nr @if [[ -z "$${SCYLLA_VERSION_RESOLVED}" ]]; then SCYLLA_VERSION_RESOLVED=`cat '${SCYLLA_VERSION_FILE}'` fi @@ -261,9 +293,9 @@ test-integration-scylla: .install-all-modules .prepare-scylla-ccm resolve-scylla echo "ScyllaDB version ${SCYLLA_VERSION} was not resolved" exit 1 fi - mvn -B -e verify -pl integration-tests -Dccm.version=$${SCYLLA_VERSION_RESOLVED} -Dccm.distribution=scylla -Dfmt.skip=true -Dclirr.skip=true -Danimal.sniffer.skip=true $(MAVEN_EXTRA_ARGS) + $(MVNCMD) -e verify $(MAVEN_IT_PL_ARGS) -Dccm.version=$${SCYLLA_VERSION_RESOLVED} -Dccm.distribution=scylla -Dfmt.skip=true -Dclirr.skip=true -Danimal.sniffer.skip=true $(MAVEN_EXTRA_ARGS) -test-integration-cassandra: .install-all-modules .prepare-scylla-ccm resolve-cassandra-version +test-integration-cassandra: $(INSTALL_ALL_DEP) $(PREPARE_CCM_DEP) resolve-cassandra-version @if [[ -z "$${CASSANDRA_VERSION_RESOLVED}" ]]; then CASSANDRA_VERSION_RESOLVED=`cat '${CASSANDRA_VERSION_FILE}'` fi @@ -271,7 +303,7 @@ test-integration-cassandra: .install-all-modules .prepare-scylla-ccm resolve-cas echo "Cassandra version ${CASSANDRA_VERSION} was not resolved" exit 1 fi - mvn -B -e verify -pl integration-tests -Dccm.version=$${CASSANDRA_VERSION_RESOLVED} -Dfmt.skip=true -Dclirr.skip=true -Danimal.sniffer.skip=true $(MAVEN_EXTRA_ARGS) + $(MVNCMD) -e verify $(MAVEN_IT_PL_ARGS) -Dccm.version=$${CASSANDRA_VERSION_RESOLVED} -Dfmt.skip=true -Dclirr.skip=true -Danimal.sniffer.skip=true $(MAVEN_EXTRA_ARGS) check-no-compile-warnings: @$(MAKE) compile-all | grep WARNING >/tmp/all-compile-warnings.log || true diff --git a/integration-tests/pom.xml b/integration-tests/pom.xml index fa343078e68..d0e5613d03d 100644 --- a/integration-tests/pom.xml +++ b/integration-tests/pom.xml @@ -226,6 +226,11 @@ + org.apache.maven.plugins maven-failsafe-plugin @@ -236,14 +241,12 @@ integration-test - ${testing.jvm}/bin/java com.datastax.oss.driver.categories.ParallelizableTests classes ${test.parallel.threads} ${project.build.directory}/failsafe-reports/failsafe-summary-parallelized.xml ${skipParallelizableITs} ${blockhound.argline} - ${testing.jvm}/bin/java @@ -256,7 +259,6 @@ ${project.build.directory}/failsafe-reports/failsafe-summary-serial.xml ${skipSerialITs} ${blockhound.argline} - ${testing.jvm}/bin/java @@ -272,7 +274,6 @@ ${project.build.directory}/failsafe-reports/failsafe-summary-isolated.xml ${skipIsolatedITs} ${blockhound.argline} - ${testing.jvm}/bin/java