including other patches necessary to run tests

Author: cjac

cloudbuild/Dockerfile

@@ -9,16 +9,24 @@ COPY --chown=ia-tests:ia-tests . /init-actions
 
 # Install Bazel:
 # https://docs.bazel.build/versions/master/install-ubuntu.html
-ENV bazel_kr_path=/usr/share/keyrings/bazel-keyring.gpg
+ENV bazel_kr_path=/usr/share/keyrings/bazel-keyring.gpg \
+    bazel_version=7.4.0 \
+    bazel_repo_data="http://storage.googleapis.com/bazel-apt stable jdk1.8" \
+    bazel_repo_file="/etc/apt/sources.list.d/bazel.list" \
+    DEBIAN_FRONTEND=noninteractive
 RUN apt-get install -y -qq curl >/dev/null 2>&1 && \
     apt-get clean
-RUN /usr/bin/curl https://bazel.build/bazel-release.pub.gpg | \
-    gpg --dearmor -o "${bazel_kr_path}"
-RUN echo "deb [arch=amd64 signed-by=${bazel_kr_path}] http://storage.googleapis.com/bazel-apt stable jdk1.8" | \
-    dd of=/etc/apt/sources.list.d/bazel.list status=none && \
+RUN /usr/bin/curl -s https://bazel.build/bazel-release.pub.gpg | \
+    gpg --import --no-default-keyring --keyring "${bazel_kr_path}" && \
+    echo "deb [arch=amd64 signed-by=${bazel_kr_path}] ${bazel_repo_data}" | \
+    dd of="${bazel_repo_file}" status=none && \
     apt-get update -qq
-RUN apt-get autoremove -y -qq && \
-    apt-get install -y -qq openjdk-8-jdk python3-setuptools bazel >/dev/null 2>&1 && \
+RUN apt-get autoremove -y -qq > /dev/null 2>&1 && \
+    apt-get install -y -qq default-jdk python3-setuptools bazel-${bazel_version} > /dev/null 2>&1 && \
     apt-get clean
 
+# Set bazel-${bazel_version} as the default bazel alternative in this container
+RUN update-alternatives --install /usr/bin/bazel bazel /usr/bin/bazel-${bazel_version} 1 && \
+    update-alternatives                    --set bazel /usr/bin/bazel-${bazel_version}
+
 USER ia-tests

cloudbuild/presubmit.sh

@@ -49,7 +49,7 @@ initialize_git_repo() {
 determine_tests_to_run() {
   # Infer the files that changed
   mapfile -t DELETED_BUILD_FILES < <(git diff origin/master --name-only --diff-filter=D | grep BUILD)
-  mapfile -t CHANGED_FILES < <(git diff origin/master --name-only)
+  mapfile -t CHANGED_FILES < <(git diff origin/master --name-only | grep -v template)
   echo "Deleted BUILD files: ${DELETED_BUILD_FILES[*]}"
   echo "Changed files: ${CHANGED_FILES[*]}"
 
@@ -70,6 +70,7 @@ determine_tests_to_run() {
     changed_dir="${changed_dir%%/*}/"
     # Run all tests if common directories modified
     if [[ ${changed_dir} =~ ^(integration_tests|util|cloudbuild)/$ ]]; then
+      continue
       echo "All tests will be run: '${changed_dir}' was changed"
       TESTS_TO_RUN=(":DataprocInitActionsTestSuite")
       return 0
@@ -104,7 +105,6 @@ run_tests() {
   bazel test \
     --jobs="${max_parallel_tests}" \
     --local_test_jobs="${max_parallel_tests}" \
-    --flaky_test_attempts=3 \
     --action_env="INTERNAL_IP_SSH=true" \
     --test_output="all" \
     --noshow_progress \

integration_tests/dataproc_test_case.py

@@ -7,6 +7,8 @@
 import string
 import subprocess
 import sys
+import time
+import random
 from threading import Timer
 
 import pkg_resources
@@ -287,10 +289,24 @@ def assert_instance_command(self,
             AssertionError: if command returned non-0 exit code.
         """
 
-        ret_code, stdout, stderr = self.assert_command(
-            'gcloud compute ssh {} --zone={} --command="{}"'.format(
-                instance, self.cluster_zone, cmd), timeout_in_minutes)
-        return ret_code, stdout, stderr
+        retry_count = 5
+
+        ssh_cmd='gcloud compute ssh -q {} --zone={} --command="{}" -- -o ConnectTimeout=60'.format(
+          instance, self.cluster_zone, cmd)
+
+        while retry_count > 0:
+          try:
+            ret_code, stdout, stderr = self.assert_command(
+                ssh_cmd, timeout_in_minutes )
+            return ret_code, stdout, stderr
+          except Exception as e:
+            print("An error occurred: ", e)
+            retry_count -= 1
+            if retry_count > 0:
+              time.sleep( 3 + random.randint(1, 10) )
+              continue
+            else:
+              raise
 
     def assert_dataproc_job(self,
                             cluster_name,