diff --git a/.github/workflows/sdk-platform-java-downstream_protobuf_compatibility_check_nightly.yaml b/.github/workflows/sdk-platform-java-downstream_protobuf_compatibility_check_nightly.yaml new file mode 100644 index 000000000000..cbcfe2fed8c9 --- /dev/null +++ b/.github/workflows/sdk-platform-java-downstream_protobuf_compatibility_check_nightly.yaml @@ -0,0 +1,88 @@ +permissions: + contents: read + +on: + pull_request: + # Runs on PRs targeting main, but will be filtered for Release PRs + branches: + - 'main' + workflow_dispatch: + inputs: + protobuf_runtime_versions: + description: 'Comma separated list of Protobuf-Java versions (i.e. "3.25.x","4.x.y"). + Note: Surround each version in the list with quotes ("")' + required: true + schedule: + - cron: '0 1 * * *' # Nightly at 1am + +# This job intends to test the compatibility of Protobuf runtime version against modules in the monorepo. +name: sdk-platform-java Downstream Protobuf Compatibility Check Nightly +jobs: + downstream-protobuf-test: + # This job runs if any of the three conditions match: + # 1. PR is raised from Release-Please (PR comes from branch: release-please--branches-main) + # 2. Job is invoked by the nightly job (scheduled event) + # 3. Job is manually invoked via Github UI (workflow_dispatch event) + if: github.head_ref == 'release-please--branches--main' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' + runs-on: ubuntu-22.04 + permissions: + contents: read + strategy: + fail-fast: false + matrix: + module: + - java-bigtable + - java-bigquery + - java-bigquerystorage + - java-datastore + - java-firestore + - java-grafeas + - java-logging + - java-logging-logback + - java-pubsub + - java-resourcemanager + - java-showcase + - java-spanner + - java-spanner-jdbc + - java-storage + - java-storage-nio + - java-translate + # Default Protobuf-Java versions to use are specified here. Without this, the nightly workflow won't know + # which values to use and would resolve to ''. + protobuf-version: ${{ fromJSON(format('[{0}]', inputs.protobuf_runtime_versions || '"4.35.1"')) }} + steps: + - name: Checkout monorepo + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: false + - uses: actions/setup-java@c1e323688fd81a25caa38c78aa6df2d33d3e20d9 # v4 + with: + java-version: 8 + distribution: temurin + - run: echo "JAVA8_HOME=${JAVA_HOME}" >> $GITHUB_ENV + # Java Client Libraries are compiled with Java 11 and target Java 8. Java 11 is required because GraalVM + # minimum support is for Java 11. + - uses: actions/setup-java@c1e323688fd81a25caa38c78aa6df2d33d3e20d9 # v4 + with: + java-version: 11 + distribution: temurin + - name: Install monorepo modules + shell: bash + run: .kokoro/build.sh + env: + BUILD_SUBDIR: ${{ matrix.module }} + JOB_TYPE: install + - name: Print Protobuf-Java testing version + run: echo "Testing with Protobuf-Java v${PROTOBUF_RUNTIME_VERSION}" + env: + PROTOBUF_RUNTIME_VERSION: ${{ matrix.protobuf-version }} + - name: Perform downstream source compatibility testing + run: ./sdk-platform-java/.kokoro/nightly/downstream-protobuf-source-compatibility.sh + env: + MODULES_UNDER_TEST: ${{ matrix.module }} + PROTOBUF_RUNTIME_VERSION: ${{ matrix.protobuf-version }} + - name: Perform downstream binary compatibility testing + run: ./sdk-platform-java/.kokoro/nightly/downstream-protobuf-binary-compatibility.sh + env: + MODULES_UNDER_TEST: ${{ matrix.module }} + PROTOBUF_RUNTIME_VERSION: ${{ matrix.protobuf-version }} diff --git a/.kokoro/common.sh b/.kokoro/common.sh index a1dad45d963d..9a56b378131a 100644 --- a/.kokoro/common.sh +++ b/.kokoro/common.sh @@ -418,6 +418,9 @@ function install_modules() { always_install_deps_list=( 'java-monitoring/google-cloud-monitoring' 'java-monitoring/google-cloud-monitoring-bom' + 'java-kms/google-cloud-kms' + 'java-kms/proto-google-cloud-kms-v1' + 'java-kms/grpc-google-cloud-kms-v1' 'google-auth-library-java/appengine' 'google-auth-library-java/bom' 'google-auth-library-java/cab-token-generator' diff --git a/java-storage/google-cloud-storage/pom.xml b/java-storage/google-cloud-storage/pom.xml index bde210ee1cb7..f9243ea0af3a 100644 --- a/java-storage/google-cloud-storage/pom.xml +++ b/java-storage/google-cloud-storage/pom.xml @@ -514,6 +514,7 @@ **/Crc32cValueTest.java **/ITSyncingFileChannelTest.java **/RewindableByteBufferContentTest.java + **/StorageITRunnerTest.java diff --git a/sdk-platform-java/.kokoro/nightly/common.sh b/sdk-platform-java/.kokoro/nightly/common.sh index 589ee00bfbf7..c5de227610d8 100644 --- a/sdk-platform-java/.kokoro/nightly/common.sh +++ b/sdk-platform-java/.kokoro/nightly/common.sh @@ -13,18 +13,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -# For google-cloud-java, only test specific handwritten libraries included in the monorepo. This is to -# help speed up the execution as building the entire repo is an expensive operation. Specify the nested -# `google-cloud-*` path (except for grafeas as it doesn't have one) because maven -pl will only build the -# specified folder (i.e. parent folder) and ignore all the related sub-modules inside -google_cloud_java_handwritten_maven_args="java-grafeas,java-vertexai/google-cloud-vertexai,java-resourcemanager/google-cloud-resourcemanager,java-translate/google-cloud-translate" - # Checks that the protobuf compatibility scripts provide non-empty input function validate_protobuf_compatibility_script_inputs { - # Comma-delimited list of repos to test - if [ -z "${REPOS_UNDER_TEST}" ]; then - echo "REPOS_UNDER_TEST Env Var must be set. This script expects a" - echo "comma-delimited list: i.e REPOS_UNDER_TEST=\"java-bigtable,java-bigquery\"" + # Comma-delimited list of modules to test + if [ -z "${MODULES_UNDER_TEST}" ]; then + echo "MODULES_UNDER_TEST Env Var must be set. This script expects a" + echo "comma-delimited list: i.e MODULES_UNDER_TEST=\"java-bigtable,java-bigquery\"" exit 1 fi diff --git a/sdk-platform-java/.kokoro/nightly/downstream-protobuf-binary-compatibility.sh b/sdk-platform-java/.kokoro/nightly/downstream-protobuf-binary-compatibility.sh index 5877eb614c73..366f59c64e03 100755 --- a/sdk-platform-java/.kokoro/nightly/downstream-protobuf-binary-compatibility.sh +++ b/sdk-platform-java/.kokoro/nightly/downstream-protobuf-binary-compatibility.sh @@ -20,44 +20,53 @@ source "${scriptDir}/common.sh" validate_protobuf_compatibility_script_inputs +monorepoRoot=$(realpath "${scriptDir}/../../..") + # Declare a map of downstream handwritten libraries and the relevant artifacts to test. The map stores a -# K/V pairing of (Key: repo name, Value: comma separate list of Group ID:Artifact ID pairings). Note: The -# value list doesn't hold the version and this needs to be parsed from the repo's versions.txt file -declare -A repo_linkage_checker_arguments -repo_linkage_checker_arguments["google-cloud-java"]="io.grafeas:grafeas,com.google.cloud:google-cloud-vertexai,com.google.cloud:google-cloud-resourcemanager,com.google.cloud:google-cloud-translate,com.google.api.grpc:grpc-google-cloud-vertexai-v1,com.google.api.grpc:grpc-google-cloud-vertexai-v1beta1,com.google.api.grpc:grpc-google-cloud-resourcemanager-v3,com.google.api.grpc:grpc-google-cloud-translate-v3,com.google.api.grpc:grpc-google-cloud-translate-v3beta1" -repo_linkage_checker_arguments["java-bigtable"]="com.google.cloud:google-cloud-bigtable,com.google.api.grpc:grpc-google-cloud-bigtable-admin-v2,com.google.api.grpc:grpc-google-cloud-bigtable-v2" -repo_linkage_checker_arguments["java-bigquery"]="com.google.cloud:google-cloud-bigquery" -repo_linkage_checker_arguments["java-bigquerystorage"]="com.google.cloud:google-cloud-bigquerystorage,com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1,com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2,com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1,com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1alpha" -repo_linkage_checker_arguments["java-datastore"]="com.google.cloud:google-cloud-datastore,com.google.cloud.datastre:datastore-v1-proto-client,com.google.api.grpc:grpc-google-cloud-datastore-admin-v1" -repo_linkage_checker_arguments["java-firestore"]="com.google.cloud:google-cloud-firestore,com.google.cloud:google-cloud-firestore-admin,com.google.api.grpc:grpc-google-cloud-firestore-admin-v1,com.google.api.grpc:grpc-google-cloud-firestore-v1" -repo_linkage_checker_arguments["java-logging"]="com.google.cloud:google-cloud-logging,com.google.api.grpc:grpc-google-cloud-logging-v2" -repo_linkage_checker_arguments["java-logging-logback"]="com.google.cloud:google-cloud-logging-logback" -repo_linkage_checker_arguments["java-pubsub"]="com.google.cloud:google-cloud-pubsub,com.google.api.grpc:grpc-google-cloud-pubsub-v1" -repo_linkage_checker_arguments["java-pubsublite"]="com.google.cloud:google-cloud-pubsublite,com.google.api.grpc:grpc-google-cloud-pubsublite-v1" -repo_linkage_checker_arguments["java-spanner-jdbc"]="com.google.cloud:google-cloud-spanner-jdbc" -repo_linkage_checker_arguments["java-spanner"]="com.google.cloud:google-cloud-spanner,com.google.cloud:google-cloud-spanner-executor,com.google.api.grpc:grpc-google-cloud-spanner-v1,com.google.api.grpc:grpc-google-cloud-spanner-admin-instance-v1,com.google.api.grpc:grpc-google-cloud-spanner-admin-database-v1,com.google.api.grpc:grpc-google-cloud-spanner-executor-v1" -repo_linkage_checker_arguments["java-storage"]="com.google.cloud:google-cloud-storage,com.google.api.grpc:gapic-google-cloud-storage-v2,com.google.api.grpc:grpc-google-cloud-storage-v2,com.google.cloud:google-cloud-storage-control,com.google.api.grpc:grpc-google-cloud-storage-control-v2" -repo_linkage_checker_arguments["java-storage-nio"]="com.google.cloud:google-cloud-nio" +# K/V pairing of (Key: module name, Value: comma separate list of Group ID:Artifact ID pairings). Note: The +# value list doesn't hold the version and this needs to be parsed from the monorepo's versions.txt file +declare -A module_linkage_checker_arguments +module_linkage_checker_arguments["java-grafeas"]="io.grafeas:grafeas" +module_linkage_checker_arguments["java-resourcemanager"]="com.google.cloud:google-cloud-resourcemanager,com.google.api.grpc:grpc-google-cloud-resourcemanager-v3" +module_linkage_checker_arguments["java-translate"]="com.google.cloud:google-cloud-translate,com.google.api.grpc:grpc-google-cloud-translate-v3,com.google.api.grpc:grpc-google-cloud-translate-v3beta1" +module_linkage_checker_arguments["java-bigtable"]="com.google.cloud:google-cloud-bigtable,com.google.api.grpc:grpc-google-cloud-bigtable-admin-v2,com.google.api.grpc:grpc-google-cloud-bigtable-v2" +module_linkage_checker_arguments["java-bigquery"]="com.google.cloud:google-cloud-bigquery" +module_linkage_checker_arguments["java-bigquerystorage"]="com.google.cloud:google-cloud-bigquerystorage,com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta1,com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1beta2,com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1,com.google.api.grpc:grpc-google-cloud-bigquerystorage-v1alpha" +module_linkage_checker_arguments["java-datastore"]="com.google.cloud:google-cloud-datastore,com.google.cloud.datastre:datastore-v1-proto-client,com.google.api.grpc:grpc-google-cloud-datastore-admin-v1" +module_linkage_checker_arguments["java-firestore"]="com.google.cloud:google-cloud-firestore,com.google.cloud:google-cloud-firestore-admin,com.google.api.grpc:grpc-google-cloud-firestore-admin-v1,com.google.api.grpc:grpc-google-cloud-firestore-v1" +module_linkage_checker_arguments["java-logging"]="com.google.cloud:google-cloud-logging,com.google.api.grpc:grpc-google-cloud-logging-v2" +module_linkage_checker_arguments["java-logging-logback"]="com.google.cloud:google-cloud-logging-logback" +module_linkage_checker_arguments["java-pubsub"]="com.google.cloud:google-cloud-pubsub,com.google.api.grpc:grpc-google-cloud-pubsub-v1" +module_linkage_checker_arguments["java-showcase"]="com.google.showcase:gapic-showcase,com.google.api.grpc:grpc-gapic-showcase-v1beta1" +module_linkage_checker_arguments["java-spanner-jdbc"]="com.google.cloud:google-cloud-spanner-jdbc" +module_linkage_checker_arguments["java-spanner"]="com.google.cloud:google-cloud-spanner,com.google.cloud:google-cloud-spanner-executor,com.google.api.grpc:grpc-google-cloud-spanner-v1,com.google.api.grpc:grpc-google-cloud-spanner-admin-instance-v1,com.google.api.grpc:grpc-google-cloud-spanner-admin-database-v1,com.google.api.grpc:grpc-google-cloud-spanner-executor-v1" +module_linkage_checker_arguments["java-storage"]="com.google.cloud:google-cloud-storage,com.google.api.grpc:gapic-google-cloud-storage-v2,com.google.api.grpc:grpc-google-cloud-storage-v2,com.google.cloud:google-cloud-storage-control,com.google.api.grpc:grpc-google-cloud-storage-control-v2" +module_linkage_checker_arguments["java-storage-nio"]="com.google.cloud:google-cloud-nio" # This function requires access to the versions.txt to retrieve the versions for the artifacts # It will try to match the artifact_id in the versions.txt file and attach it to form the GAV # The GAV list is required by Linkage Checker as program arguments function build_program_arguments() { - artifact_list="${repo_linkage_checker_arguments[$1]}" + artifact_list="${module_linkage_checker_arguments[$1]}" for artifact in ${artifact_list//,/ }; do # Split on comma artifact_id=$(echo "${artifact}" | cut -d ':' -f2) # The grep query tries to match `{artifact_id}:{released_version}:{current_version}`. # The artifact_id must be exact otherwise multiple entries may match - version=$(cat "versions.txt" | grep -E "^${artifact_id}:.*:.*$" | cut -d ':' -f3) - repo_gav_coordinate="${artifact}:${version}" + version=$(cat "${monorepoRoot}/versions.txt" | grep -E "^${artifact_id}:.*:.*$" | cut -d ':' -f3 || true) + # Unreleased internal test modules like java-showcase are not tracked in versions.txt, + # so fallback to 0.0.1-SNAPSHOT for linkage checking. + if [ -z "${version}" ]; then + version="0.0.1-SNAPSHOT" + fi + module_gav_coordinate="${artifact}:${version}" # The first entry added is not separated with a comma. Avoids generating `,{ARTIFACT_LIST}` if [ -z "${linkage_checker_arguments}" ]; then - linkage_checker_arguments="${repo_gav_coordinate}" + linkage_checker_arguments="${module_gav_coordinate}" else - linkage_checker_arguments="${linkage_checker_arguments},${repo_gav_coordinate}" + linkage_checker_arguments="${linkage_checker_arguments},${module_gav_coordinate}" fi done } @@ -70,26 +79,22 @@ mvn -B -ntp clean compile -T 1C # Linkage Checker tool resides in the /dependencies subfolder pushd dependencies -# REPOS_UNDER_TEST Env Var accepts a comma separated list of googleapis repos to test. For Github CI, -# this will be a single repo as Github will build a matrix of repos with each repo being tested in parallel. -# For local invocation, you can pass a list of repos to test multiple repos together. -for repo in ${REPOS_UNDER_TEST//,/ }; do # Split on comma - # Perform testing on main (with latest changes). Shallow copy as history is not important - git clone "https://github.com/googleapis/${repo}.git" --depth=1 - pushd "${repo}" - - if [ "${repo}" == "google-cloud-java" ]; then - # The `-am` command also builds anything these libraries depend on (i.e. proto-* and grpc-* sub modules) - mvn clean install -B -V -ntp -T 1C -DskipTests -Dclirr.skip -Denforcer.skip -Dmaven.javadoc.skip \ - -pl "${google_cloud_java_handwritten_maven_args}" -am - else - # Install all repo modules to ~/.m2 (there can be multiple relevant artifacts to test i.e. core, admin, control) - mvn clean install -B -V -ntp -T 1C -DskipTests -Dclirr.skip -Denforcer.skip -Dmaven.javadoc.skip +# MODULES_UNDER_TEST Env Var accepts a comma separated list of monorepo submodules to test. For Github CI, +# this will be a single module as Github will build a matrix of modules with each being tested in parallel. +# For local invocation, you can pass a list of modules to test multiple modules together. +for module in ${MODULES_UNDER_TEST//,/ }; do # Split on comma + module_dir="${monorepoRoot}/${module}" + if [ ! -d "${module_dir}" ]; then + echo "Directory ${module_dir} does not exist. Skipping or failed." >&2 + exit 1 fi + pushd "${module_dir}" + # Install all module artifacts to ~/.m2 (there can be multiple relevant artifacts to test i.e. core, admin, control) + mvn clean install -B -V -ntp -T 1C -DskipTests -Dclirr.skip -Denforcer.skip -Dmaven.javadoc.skip linkage_checker_arguments="" - build_program_arguments "${repo}" + build_program_arguments "${module}" # Linkage Checker /dependencies popd @@ -97,7 +102,7 @@ for repo in ${REPOS_UNDER_TEST//,/ }; do # Split on comma echo "Artifact List: ${linkage_checker_arguments}" # The `-s` argument filters the linkage check problems that stem from the artifact program_args="-r --artifacts ${linkage_checker_arguments},com.google.protobuf:protobuf-java:${PROTOBUF_RUNTIME_VERSION},com.google.protobuf:protobuf-java-util:${PROTOBUF_RUNTIME_VERSION} -s ${linkage_checker_arguments}" - echo "Running Linkage Checker on the repo's handwritten modules" + echo "Running Linkage Checker on the module's handwritten artifacts" echo "Linkage Checker Program Arguments: ${program_args}" mvn -B -ntp exec:java -Dexec.args="${program_args}" -P exec-linkage-checker done diff --git a/sdk-platform-java/.kokoro/nightly/downstream-protobuf-source-compatibility.sh b/sdk-platform-java/.kokoro/nightly/downstream-protobuf-source-compatibility.sh index aaa65b82087e..32379fae2e35 100755 --- a/sdk-platform-java/.kokoro/nightly/downstream-protobuf-source-compatibility.sh +++ b/sdk-platform-java/.kokoro/nightly/downstream-protobuf-source-compatibility.sh @@ -20,48 +20,44 @@ source "${scriptDir}/common.sh" validate_protobuf_compatibility_script_inputs -# REPOS_UNDER_TEST Env Var accepts a comma separated list of googleapis repos to test. For Github CI, -# this will be a single repo as Github will build a matrix of repos with each repo being tested in parallel. -# For local invocation, you can pass a list of repos to test multiple repos together. -for repo in ${REPOS_UNDER_TEST//,/ }; do # Split on comma - # Perform source-compatibility testing on main (latest changes) - git clone "https://github.com/googleapis/$repo.git" --depth=1 - pushd "$repo" +monorepoRoot=$(realpath "${scriptDir}/../../..") + +# MODULES_UNDER_TEST Env Var accepts a comma separated list of monorepo submodules to test. For Github CI, +# this will be a single module as Github will build a matrix of modules with each being tested in parallel. +# For local invocation, you can pass a list of modules to test multiple modules together. +for module in ${MODULES_UNDER_TEST//,/ }; do # Split on comma + module_dir="${monorepoRoot}/${module}" + if [ ! -d "${module_dir}" ]; then + echo "Directory ${module_dir} does not exist. Skipping or failed." >&2 + exit 1 + fi + + pushd "${module_dir}" # Compile with Java 11 and run the tests with Java 8 JVM mvn compile -T 1C # JAVA8_HOME is set by the GH Actions CI if [ -n "${JAVA8_HOME}" ]; then - surefire_opt="-Djvm=${JAVA8_HOME}/bin/java" + jvm_opt="-Djvm=${JAVA8_HOME}/bin/java" + # Required for JDK 8 to bypass Java 11 LogbackServiceProvider and exclude JDK 11 Jqwik property tests + test_opts="-Dslf4j.provider=org.slf4j.helpers.NOP_FallbackServiceProvider -P!jqwik-tests -Pexclude-jqwik-on-java8" else # Provide a default value for local executions that don't configure JAVA8_HOME - surefire_opt="-Djvm=${JAVA_HOME}/bin/java" + jvm_opt="-Djvm=${JAVA_HOME}/bin/java" + test_opts="" fi - # Compile the Handwritten Library with the Protobuf-Java version to test source compatibility + # Compile the library with the Protobuf-Java version to test source compatibility # Run unit tests to help check for any behavior differences (dependant on coverage) - if [ "${repo}" == "google-cloud-java" ]; then - # The `-am` command also builds anything these libraries depend on (i.e. proto-* and grpc-* sub modules) - mvn test -B -V -ntp \ - -Dclirr.skip \ - -Denforcer.skip \ - -Dmaven.javadoc.skip \ - -Denforcer.skip \ - -Dprotobuf.version=${PROTOBUF_RUNTIME_VERSION} \ - -pl "${google_cloud_java_handwritten_maven_args}" -am \ - "${surefire_opt}" \ - -T 1C - else - mvn test -B -V -ntp \ - -Dclirr.skip \ - -Denforcer.skip \ - -Dmaven.javadoc.skip \ - -Denforcer.skip \ - -Dprotobuf.version=${PROTOBUF_RUNTIME_VERSION} \ - "${surefire_opt}" \ - -T 1C - fi + mvn test -B -V -ntp \ + -Dclirr.skip \ + -Denforcer.skip \ + -Dmaven.javadoc.skip \ + -Dprotobuf.version=${PROTOBUF_RUNTIME_VERSION} \ + "${jvm_opt}" \ + ${test_opts} \ + -T 1C popd done \ No newline at end of file