diff --git a/.gitignore b/.gitignore index e3b431ec2..144de683f 100644 --- a/.gitignore +++ b/.gitignore @@ -116,3 +116,7 @@ spark-warehouse/ # For venv *.venv .venv/ +/credentials.sbt +/project/credentials.sbt +/.sbtopts +/.sbt-mirror-repositories diff --git a/ci/Jenkinsfile b/ci/Jenkinsfile index 68c98ebd2..cae8f61b2 100644 --- a/ci/Jenkinsfile +++ b/ci/Jenkinsfile @@ -22,7 +22,15 @@ node('docker-big') { try { stage('Build image') { ansiColor('xterm') { - sh("${MAKE} build") + // DEVOPS-6131: sbt can't reach Maven Central from jenkins-eng (shared egress IP + // is 429'd). ci/sbt-mirror/with-mirror.sh runs the build with sbt pointed at the + // virtana-zing Artifact Registry proxy: it mints a short-lived token from + // gcr_push_key (in-process; gcloud uses a throwaway config so the agent account + // is untouched) and cleans up its transient config on exit. Logic + templates + // all live in ci/sbt-mirror/. + withCredentials([file(credentialsId: 'gcr_push_key', variable: 'AR_KEY_FILE')]) { + sh("ci/sbt-mirror/with-mirror.sh ${MAKE} build") + } } } @@ -42,6 +50,7 @@ node('docker-big') { } finally { stage ('Clean test environment') { + // with-mirror.sh removes its own auth files via an EXIT trap; just clean the build. sh("${MAKE} clean") } } diff --git a/ci/sbt-mirror/credentials.properties.tmpl b/ci/sbt-mirror/credentials.properties.tmpl new file mode 100644 index 000000000..cbace2718 --- /dev/null +++ b/ci/sbt-mirror/credentials.properties.tmpl @@ -0,0 +1,4 @@ +virtana-zing.host=us-maven.pkg.dev +virtana-zing.username=oauth2accesstoken +virtana-zing.password=${AR_TOKEN} +virtana-zing.auto=true diff --git a/ci/sbt-mirror/credentials.sbt b/ci/sbt-mirror/credentials.sbt new file mode 100644 index 000000000..b1ba78384 --- /dev/null +++ b/ci/sbt-mirror/credentials.sbt @@ -0,0 +1,14 @@ +// DEVOPS-6131 (generated copy - do not edit here; source: ci/sbt-mirror/credentials.sbt). +// sbt's lm-coursier only authenticates via the native `credentials` setting (not the coursier +// properties file / COURSIER_CREDENTIALS env). ci/sbt-mirror/setup.sh drops this at both the build +// root and project/ (meta build) so library AND plugin resolution can reach the virtana-zing mirror. +// The token comes from the AR_TOKEN env var (exported by sourcing .ar-token.env before the build), +// so no secret lives in this file; with AR_TOKEN unset (local dev, mirror not used) the password is +// empty and this credential is simply never matched/used. +// +// Scope = ThisBuild so EVERY subproject (server/client/spark) sees the credential. A plain +// `credentials += ...` is root-project-only, so Ivy resolution for the subprojects (scalastyle, +// makePom) found no credential and logged "Unable to find credentials for [... @ us-maven.pkg.dev]" +// even though coursier (which aggregates credentials build-wide) still downloaded everything. +// Realm matches exactly what AR sends in WWW-Authenticate: Basic realm="https://us-maven.pkg.dev". +ThisBuild / credentials += Credentials("https://us-maven.pkg.dev", "us-maven.pkg.dev", "oauth2accesstoken", sys.env.getOrElse("AR_TOKEN", "")) diff --git a/ci/sbt-mirror/repositories b/ci/sbt-mirror/repositories new file mode 100644 index 000000000..1df7d6cf5 --- /dev/null +++ b/ci/sbt-mirror/repositories @@ -0,0 +1,6 @@ +[repositories] + local + maven-central: https://us-maven.pkg.dev/zing-registry-188222/virtana-zing + sbt-plugin-releases: https://repo.scala-sbt.org/scalasbt/sbt-plugin-releases/, [organization]/[module]/(scala_[scalaVersion]/)(sbt_[sbtVersion]/)[revision]/[type]s/[artifact](-[classifier]).[ext] + typesafe-ivy-releases: https://repo.typesafe.com/typesafe/ivy-releases/, [organization]/[module]/[revision]/[type]s/[artifact](-[classifier]).[ext], bootOnly + sbt-ivy-snapshots: https://repo.scala-sbt.org/scalasbt/ivy-snapshots/, [organization]/[module]/[revision]/[type]s/[artifact](-[classifier]).[ext], bootOnly diff --git a/ci/sbt-mirror/with-mirror.sh b/ci/sbt-mirror/with-mirror.sh new file mode 100755 index 000000000..df44e07de --- /dev/null +++ b/ci/sbt-mirror/with-mirror.sh @@ -0,0 +1,82 @@ +#!/usr/bin/env bash +# +# DEVOPS-6131 - run a build command with sbt resolving Maven Central through the virtana-zing +# Artifact Registry proxy instead of hitting repo1.maven.org directly (jenkins-eng's shared egress +# IP gets 429'd by Central's Cloudflare front-end, which breaks sbt's launcher/dependency/plugin +# downloads). This is the sbt analog of the mirrorMavenCentral shared step - sbt doesn't read +# Maven's settings.xml, so we render sbt's own config from the templates next to this script. +# +# Usage (from a pipeline, gcr_push_key bound as AR_KEY_FILE): +# withCredentials([file(credentialsId: 'gcr_push_key', variable: 'AR_KEY_FILE')]) { +# sh 'ci/sbt-mirror/with-mirror.sh make -f ci/Makefile build' +# } +# +# Design notes addressing the two things we care about: +# * The AR OAuth token stays IN-PROCESS (exported to the child build only). Nothing writes it to a +# file that gets passed around; the one transient file that must hold it (coursier's boot creds) +# is removed by the EXIT trap below, pass or fail - no separate pipeline cleanup step. +# * gcloud auth runs in a THROWAWAY CLOUDSDK_CONFIG dir, so the agent's own gcloud account is never +# activated/replaced (agents are reused across builds). This is NOT `gcloud auth login`. +# +# AR_KEY_FILE must point at a GCP service-account key with artifactregistry.reader on +# zing-registry-188222. sbt runs on the agent, so no container mount is needed. + +set -eu +: "${AR_KEY_FILE:?with-mirror.sh: AR_KEY_FILE must be set (bind gcr_push_key as a file credential)}" +[ "$#" -ge 1 ] || { echo "with-mirror.sh: usage: with-mirror.sh " >&2; exit 2; } + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" +MIRROR="https://us-maven.pkg.dev/zing-registry-188222/virtana-zing" +SBT_VERSION="$(awk -F= '/sbt.version/{gsub(/ /,"",$2);print $2}' "${REPO_ROOT}/project/build.properties")" + +COURSIER_CREDS="${HOME}/.config/coursier/credentials.properties" +SBT_REPOS="${REPO_ROOT}/.sbt-mirror-repositories" + +# Remove every transient auth/config file on exit - the token never outlives this build. +cleanup() { + rm -f "${COURSIER_CREDS}" "${SBT_REPOS}" "${REPO_ROOT}/.sbtopts" \ + "${REPO_ROOT}/credentials.sbt" "${REPO_ROOT}/project/credentials.sbt" +} +trap cleanup EXIT + +# Mint a short-lived AR token in a throwaway gcloud config dir (agent's own account untouched). +# xtrace guarded off so the token is never echoed even if the caller runs us under `bash -x`. +{ set +x; } 2>/dev/null +AR_TOKEN="$( + CLOUDSDK_CONFIG="$(mktemp -d)"; export CLOUDSDK_CONFIG + gcloud auth activate-service-account --key-file="${AR_KEY_FILE}" --quiet 1>&2 + gcloud auth print-access-token + rm -rf "${CLOUDSDK_CONFIG}" +)" +export AR_TOKEN + +# repositories: workspace-local (so we don't clobber a shared ~/.sbt/repositories on the agent). +cp "${SCRIPT_DIR}/repositories" "${SBT_REPOS}" + +# coursier boot creds (token populated from template) - for the sbt LAUNCHER's shaded coursier. +mkdir -p "$(dirname "${COURSIER_CREDS}")" +while IFS= read -r line || [ -n "${line}" ]; do + printf '%s\n' "${line//'${AR_TOKEN}'/${AR_TOKEN}}" +done < "${SCRIPT_DIR}/credentials.properties.tmpl" > "${COURSIER_CREDS}" +chmod 600 "${COURSIER_CREDS}" + +# lm-coursier + Ivy creds for deps + plugins - read $AR_TOKEN from the env (no secret on disk). +cp "${SCRIPT_DIR}/credentials.sbt" "${REPO_ROOT}/credentials.sbt" +cp "${SCRIPT_DIR}/credentials.sbt" "${REPO_ROOT}/project/credentials.sbt" + +# force sbt to use ONLY the mirror repositories (no fallback to Central). +cat > "${REPO_ROOT}/.sbtopts" < ${MIRROR} (sbt ${SBT_VERSION}); running: $*" +"$@"