DEVOPS-6131: route sbt resolution through virtana-zing AR mirror (PR pipeline)

.gitignore

@@ -116,3 +116,7 @@ spark-warehouse/
 # For venv
 *.venv
 .venv/
+/credentials.sbt
+/project/credentials.sbt
+/.sbtopts
+/.sbt-mirror-repositories

ci/Jenkinsfile

@@ -22,7 +22,15 @@ node('docker-big') {
         try {
             stage('Build image') {
                 ansiColor('xterm') {
-                    sh("${MAKE} build")
+                    // DEVOPS-6131: sbt can't reach Maven Central from jenkins-eng (shared egress IP
+                    // is 429'd). ci/sbt-mirror/with-mirror.sh runs the build with sbt pointed at the
+                    // virtana-zing Artifact Registry proxy: it mints a short-lived token from
+                    // gcr_push_key (in-process; gcloud uses a throwaway config so the agent account
+                    // is untouched) and cleans up its transient config on exit. Logic + templates
+                    // all live in ci/sbt-mirror/.
+                    withCredentials([file(credentialsId: 'gcr_push_key', variable: 'AR_KEY_FILE')]) {
+                        sh("ci/sbt-mirror/with-mirror.sh ${MAKE} build")
+                    }
                 }
             }
 
@@ -42,6 +50,7 @@ node('docker-big') {
 
         } finally {
             stage ('Clean test environment') {
+                // with-mirror.sh removes its own auth files via an EXIT trap; just clean the build.
                 sh("${MAKE} clean")
             }
         }

ci/sbt-mirror/credentials.properties.tmpl

@@ -0,0 +1,4 @@
+virtana-zing.host=us-maven.pkg.dev
+virtana-zing.username=oauth2accesstoken
+virtana-zing.password=${AR_TOKEN}
+virtana-zing.auto=true

ci/sbt-mirror/credentials.sbt

@@ -0,0 +1,14 @@
+// DEVOPS-6131 (generated copy - do not edit here; source: ci/sbt-mirror/credentials.sbt).
+// sbt's lm-coursier only authenticates via the native `credentials` setting (not the coursier
+// properties file / COURSIER_CREDENTIALS env). ci/sbt-mirror/setup.sh drops this at both the build
+// root and project/ (meta build) so library AND plugin resolution can reach the virtana-zing mirror.
+// The token comes from the AR_TOKEN env var (exported by sourcing .ar-token.env before the build),
+// so no secret lives in this file; with AR_TOKEN unset (local dev, mirror not used) the password is
+// empty and this credential is simply never matched/used.
+//
+// Scope = ThisBuild so EVERY subproject (server/client/spark) sees the credential. A plain
+// `credentials += ...` is root-project-only, so Ivy resolution for the subprojects (scalastyle,
+// makePom) found no credential and logged "Unable to find credentials for [... @ us-maven.pkg.dev]"
+// even though coursier (which aggregates credentials build-wide) still downloaded everything.
+// Realm matches exactly what AR sends in WWW-Authenticate: Basic realm="https://us-maven.pkg.dev".
+ThisBuild / credentials += Credentials("https://us-maven.pkg.dev", "us-maven.pkg.dev", "oauth2accesstoken", sys.env.getOrElse("AR_TOKEN", ""))

ci/sbt-mirror/repositories

@@ -0,0 +1,6 @@
+[repositories]
+  local
+  maven-central: https://us-maven.pkg.dev/zing-registry-188222/virtana-zing
+  sbt-plugin-releases: https://repo.scala-sbt.org/scalasbt/sbt-plugin-releases/, [organization]/[module]/(scala_[scalaVersion]/)(sbt_[sbtVersion]/)[revision]/[type]s/[artifact](-[classifier]).[ext]
+  typesafe-ivy-releases: https://repo.typesafe.com/typesafe/ivy-releases/, [organization]/[module]/[revision]/[type]s/[artifact](-[classifier]).[ext], bootOnly
+  sbt-ivy-snapshots: https://repo.scala-sbt.org/scalasbt/ivy-snapshots/, [organization]/[module]/[revision]/[type]s/[artifact](-[classifier]).[ext], bootOnly

ci/sbt-mirror/with-mirror.sh

@@ -0,0 +1,82 @@
+#!/usr/bin/env bash
+#
+# DEVOPS-6131 - run a build command with sbt resolving Maven Central through the virtana-zing
+# Artifact Registry proxy instead of hitting repo1.maven.org directly (jenkins-eng's shared egress
+# IP gets 429'd by Central's Cloudflare front-end, which breaks sbt's launcher/dependency/plugin
+# downloads). This is the sbt analog of the mirrorMavenCentral shared step - sbt doesn't read
+# Maven's settings.xml, so we render sbt's own config from the templates next to this script.
+#
+# Usage (from a pipeline, gcr_push_key bound as AR_KEY_FILE):
+#     withCredentials([file(credentialsId: 'gcr_push_key', variable: 'AR_KEY_FILE')]) {
+#         sh 'ci/sbt-mirror/with-mirror.sh make -f ci/Makefile build'
+#     }
+#
+# Design notes addressing the two things we care about:
+#   * The AR OAuth token stays IN-PROCESS (exported to the child build only). Nothing writes it to a
+#     file that gets passed around; the one transient file that must hold it (coursier's boot creds)
+#     is removed by the EXIT trap below, pass or fail - no separate pipeline cleanup step.
+#   * gcloud auth runs in a THROWAWAY CLOUDSDK_CONFIG dir, so the agent's own gcloud account is never
+#     activated/replaced (agents are reused across builds). This is NOT `gcloud auth login`.
+#
+# AR_KEY_FILE must point at a GCP service-account key with artifactregistry.reader on
+# zing-registry-188222. sbt runs on the agent, so no container mount is needed.
+
+set -eu
+: "${AR_KEY_FILE:?with-mirror.sh: AR_KEY_FILE must be set (bind gcr_push_key as a file credential)}"
+[ "$#" -ge 1 ] || { echo "with-mirror.sh: usage: with-mirror.sh <build command...>" >&2; exit 2; }
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)"
+MIRROR="https://us-maven.pkg.dev/zing-registry-188222/virtana-zing"
+SBT_VERSION="$(awk -F= '/sbt.version/{gsub(/ /,"",$2);print $2}' "${REPO_ROOT}/project/build.properties")"
+
+COURSIER_CREDS="${HOME}/.config/coursier/credentials.properties"
+SBT_REPOS="${REPO_ROOT}/.sbt-mirror-repositories"
+
+# Remove every transient auth/config file on exit - the token never outlives this build.
+cleanup() {
+    rm -f "${COURSIER_CREDS}" "${SBT_REPOS}" "${REPO_ROOT}/.sbtopts" \
+          "${REPO_ROOT}/credentials.sbt" "${REPO_ROOT}/project/credentials.sbt"
+}
+trap cleanup EXIT
+
+# Mint a short-lived AR token in a throwaway gcloud config dir (agent's own account untouched).
+# xtrace guarded off so the token is never echoed even if the caller runs us under `bash -x`.
+{ set +x; } 2>/dev/null
+AR_TOKEN="$(
+    CLOUDSDK_CONFIG="$(mktemp -d)"; export CLOUDSDK_CONFIG
+    gcloud auth activate-service-account --key-file="${AR_KEY_FILE}" --quiet 1>&2
+    gcloud auth print-access-token
+    rm -rf "${CLOUDSDK_CONFIG}"
+)"
+export AR_TOKEN
+
+# repositories: workspace-local (so we don't clobber a shared ~/.sbt/repositories on the agent).
+cp "${SCRIPT_DIR}/repositories" "${SBT_REPOS}"
+
+# coursier boot creds (token populated from template) - for the sbt LAUNCHER's shaded coursier.
+mkdir -p "$(dirname "${COURSIER_CREDS}")"
+while IFS= read -r line || [ -n "${line}" ]; do
+    printf '%s\n' "${line//'${AR_TOKEN}'/${AR_TOKEN}}"
+done < "${SCRIPT_DIR}/credentials.properties.tmpl" > "${COURSIER_CREDS}"
+chmod 600 "${COURSIER_CREDS}"
+
+# lm-coursier + Ivy creds for deps + plugins - read $AR_TOKEN from the env (no secret on disk).
+cp "${SCRIPT_DIR}/credentials.sbt" "${REPO_ROOT}/credentials.sbt"
+cp "${SCRIPT_DIR}/credentials.sbt" "${REPO_ROOT}/project/credentials.sbt"
+
+# force sbt to use ONLY the mirror repositories (no fallback to Central).
+cat > "${REPO_ROOT}/.sbtopts" <<EOF
+-Dsbt.repository.config=${SBT_REPOS}
+-Dsbt.override.build.repos=true
+EOF
+
+# pre-fetch the sbt launch jar through the mirror (the bare curl in build/sbt-launch-lib.bash
+# can't send an auth header itself).
+mkdir -p "${REPO_ROOT}/build"
+curl --fail --location --silent -H "Authorization: Bearer ${AR_TOKEN}" \
+     "${MIRROR}/org/scala-sbt/sbt-launch/${SBT_VERSION}/sbt-launch-${SBT_VERSION}.jar" \
+     -o "${REPO_ROOT}/build/sbt-launch-${SBT_VERSION}.jar"
+
+echo "with-mirror.sh: sbt -> ${MIRROR} (sbt ${SBT_VERSION}); running: $*"
+"$@"