stackrox · kurlov · Jul 2, 2026
diff --git a/.github/workflows/acs-smoke-test.yml b/.github/workflows/acs-smoke-test.yml
@@ -0,0 +1,139 @@
+name: "ACS Smoke Test"
+
+on:
+  workflow_dispatch:
+    inputs:
+      operator-index-image:
+        description: 'ACS Operator index image (e.g., quay.io/rhacs-eng/stackrox-operator-index:ocp-v4-22-...)'
+        required: true
+        type: string
+      acs-version:
+        description: 'ACS minor version under test (e.g. 4.10). Determines channel and Y-2 oldest.'
+        required: true
+        type: string
+      cluster-lifespan:
+        description: 'Cluster lifespan (e.g., 2h, 4h, 8h)'
+        required: false
+        default: '2h'
+        type: string
+
+env:
+  CLUSTER_NAME: smoke-${{ github.run_number }}
+  GH_TOKEN: ${{ github.token }}
+  GH_NO_UPDATE_NOTIFIER: 1
+
+run-name: >-
+  ${{ format('ACS Smoke Test - {0}', inputs.operator-index-image) }}
+
+jobs:
+  create-cluster:
+    name: Create OpenShift cluster
+    runs-on: ubuntu-latest
+    outputs:
+      cluster-name: ${{ steps.info.outputs.name }}
+    steps:
+      - name: Set cluster name
+        id: info
+        run: echo "name=${{ env.CLUSTER_NAME }}" >> "$GITHUB_OUTPUT"
+
+      - name: Create OpenShift cluster
+        uses: stackrox/actions/infra/create-cluster@v1
+        with:
+          token: ${{ secrets.INFRA_TOKEN }}
+          flavor: ocp-default
+          name: ${{ env.CLUSTER_NAME }}
+          lifespan: ${{ inputs.cluster-lifespan }}
+          args: nodes=3,machine-type=e2-standard-8
+          wait: true
+
+  run-smoke-tests:
+    name: Run ACS smoke tests
+    needs: create-cluster
+    runs-on: ubuntu-latest
+    container:
+      image: quay.io/stackrox-io/apollo-ci:stackrox-test-0.5.11@sha256:df3459cf038e73775314b3a5d36d14dda41e6df438a0273c9efa8d23e0f2358a
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v7
+
+      - name: Install infractl
+        uses: stackrox/actions/infra/install-infractl@v1
+
+      - name: Authenticate to GCP
+        uses: google-github-actions/auth@v3
+        with:
+          credentials_json: ${{ secrets.GCP_RELEASE_AUTOMATION_SA }}
+
+      - name: Set up Cloud SDK
+        uses: google-github-actions/setup-gcloud@v3
+
+      - name: Download cluster kubeconfig
+        run: |
+          infractl artifacts \
+            --cluster-name=${{ needs.create-cluster.outputs.cluster-name }} \
+            --download-dir=./artifacts
+
+      - name: Set KUBECONFIG
+        run: echo "KUBECONFIG=${{ github.workspace }}/artifacts/kubeconfig" >> "$GITHUB_ENV"
+
+      - name: "Test 1: upgrade oldest-supported → provided"
+        working-directory: ./smoke-test
+        env:
+          OPERATOR_INDEX_IMAGE: ${{ inputs.operator-index-image }}
+          ACS_VERSION: ${{ inputs.acs-version }}
+        run: bash upgrade-oldest-test.sh
+
+      - name: Reset operator state between tests
+        working-directory: ./smoke-test
+        run: |
+          source lib.sh
+          reset_operator
+        shell: bash
+
+      - name: "Test 2: install provided → optionally upgrade to latest GA"
+        working-directory: ./smoke-test
+        env:
+          OPERATOR_INDEX_IMAGE: ${{ inputs.operator-index-image }}
+          ACS_VERSION: ${{ inputs.acs-version }}
+        run: bash upgrade-latest-test.sh
+
+  cleanup-cluster:
+    name: Cleanup cluster
+    needs: [create-cluster, run-smoke-tests]
+    runs-on: ubuntu-latest
+    if: always() && needs.create-cluster.result != 'cancelled'
+    steps:
+      - name: Install infractl
+        uses: stackrox/actions/infra/install-infractl@v1
+
+      - name: Authenticate to GCP
+        uses: google-github-actions/auth@v3
+        with:
+          credentials_json: ${{ secrets.GCP_RELEASE_AUTOMATION_SA }}
+
+      - name: Delete cluster
+        run: infractl delete --cluster-name=${{ needs.create-cluster.outputs.cluster-name }}
+
+  report-status:
+    name: Report test status
+    needs: [create-cluster, run-smoke-tests, cleanup-cluster]
+    runs-on: ubuntu-latest
+    if: always()
+    steps:
+      - name: Write summary
+        run: |
+          cat <<'EOF' >> "$GITHUB_STEP_SUMMARY"
+          # ACS Smoke Test Results
+
+          | Field | Value |
+          |-------|-------|
+          | **Operator image** | `${{ inputs.operator-index-image }}` |
+          | **Cluster** | `${{ needs.create-cluster.outputs.cluster-name }}` |
+          | **Create cluster** | ${{ needs.create-cluster.result }} |
+          | **Smoke tests** | ${{ needs.run-smoke-tests.result }} |
+          | **Cleanup** | ${{ needs.cleanup-cluster.result }} |
+
+          ## Tests run:
+          1. **Test 1** — install oldest supported ACS version, upgrade to the provided operator-index image
+          2. **Test 2** — install provided operator-index image, upgrade to latest GA if minor is behind
+          EOF
diff --git a/smoke-test/lib.sh b/smoke-test/lib.sh
@@ -0,0 +1,230 @@
+#!/usr/bin/env bash
+# Shared functions for ACS operator smoke tests.
+
+set -euo pipefail
+
+# Logging helpers
+step() { echo; echo "══════════════════════════════════════════════════════"; echo "  $*"; echo "══════════════════════════════════════════════════════"; }
+info() { echo "  $*"; }
+ok()   { echo "  ✅ $*"; }
+warn() { echo "  ⚠️  $*"; }
+fail() { echo "  ❌ $*" >&2; return 1; }
+
+# Splits "4.10" → ACS_MAJOR=4, ACS_MINOR=10
+parse_acs_version() {
+  local ver="$1"
+  ACS_MAJOR=$(echo "$ver" | cut -d. -f1)
+  ACS_MINOR=$(echo "$ver" | cut -d. -f2)
+  [[ "$ACS_MAJOR" =~ ^[0-9]+$ && "$ACS_MINOR" =~ ^[0-9]+$ ]] \
+    || fail "ACS_VERSION must be MAJOR.MINOR (e.g. 4.10), got: ${ver}"
+}
+
+# Prints the highest minor version in rhacs-MAJOR.N channels from the official catalog.
+get_latest_official_minor() {
+  local major="${1:-4}"
+  oc get packagemanifest -n openshift-marketplace \
+    -l catalog=redhat-operators \
+    -o jsonpath='{range .items[?(@.metadata.name=="rhacs-operator")].status.channels[*]}{.name}{"\n"}{end}' \
+    | grep -oE "rhacs-${major}\.[0-9]+" \
+    | grep -oE '[0-9]+$' \
+    | sort -n \
+    | tail -1 \
+    || true
+}
+
+disable_default_sources() {
+  info "Disabling default OperatorHub sources..."
+  oc patch OperatorHub cluster --type json \
+    -p '[{"op":"add","path":"/spec/disableAllDefaultSources","value":true}]'
+  ok "Default sources disabled"
+}
+
+enable_default_sources() {
+  info "Enabling default OperatorHub sources..."
+  oc patch OperatorHub cluster --type json \
+    -p '[{"op":"add","path":"/spec/disableAllDefaultSources","value":false}]'
+  ok "Default sources enabled"
+}
+
+apply_custom_catalog() {
+  local index_image="$1"
+  info "Applying custom CatalogSource (image: ${index_image})..."
+  oc apply -f - <<EOF
+apiVersion: operators.coreos.com/v1alpha1
+kind: CatalogSource
+metadata:
+  name: my-operator-catalog
+  namespace: openshift-marketplace
+spec:
+  sourceType: grpc
+  image: ${index_image}
+  displayName: My Operator Catalog
+  publisher: grpc
+EOF
+  ok "Custom CatalogSource applied"
+}
+
+wait_for_catalog() {
+  local name="$1"
+  local ns="${2:-openshift-marketplace}"
+  local timeout="${3:-180}"
+  local deadline
+  deadline=$(( $(date +%s) + timeout ))
+  info "Waiting for CatalogSource/${name} to be READY (timeout: ${timeout}s)..."
+  while (( $(date +%s) < deadline )); do
+    local state
+    state=$(oc get catalogsource "$name" -n "$ns" \
+      -o jsonpath='{.status.connectionState.lastObservedState}' 2>/dev/null || true)
+    if [[ "$state" == "READY" ]]; then
+      ok "CatalogSource/${name} is READY"
+      return 0
+    fi
+    info "  state=${state:-pending}, retrying in 10s..."
+    sleep 10
+  done
+  fail "CatalogSource/${name} not READY within ${timeout}s"
+}
+
+apply_subscription() {
+  local channel="$1" source="$2" source_ns="${3:-openshift-marketplace}"
+  info "Applying Subscription (channel=${channel}, source=${source})..."
+  oc apply -f - <<EOF
+apiVersion: operators.coreos.com/v1alpha1
+kind: Subscription
+metadata:
+  name: rhacs-operator
+  namespace: openshift-operators
+spec:
+  channel: ${channel}
+  installPlanApproval: Automatic
+  name: rhacs-operator
+  source: ${source}
+  sourceNamespace: ${source_ns}
+EOF
+  ok "Subscription applied"
+}
+
+# Sets TARGET_CSV to the currentCSV for a given channel from the specified catalog label.
+# Retries for up to 60s because packagemanifests can lag behind catalog READY state.
+_resolve_target_csv() {
+  local catalog_label="$1" channel="$2"
+  local deadline
+  deadline=$(( $(date +%s) + 60 ))
+  info "Resolving currentCSV for channel ${channel} from ${catalog_label}..."
+  while (( $(date +%s) < deadline )); do
+    TARGET_CSV=$(oc get packagemanifest -n openshift-marketplace \
+      -l "catalog=${catalog_label}" \
+      -o jsonpath="{range .items[?(@.metadata.name==\"rhacs-operator\")].status.channels[?(@.name==\"${channel}\")]}{.currentCSV}{end}" \
+      2>/dev/null || true)
+    if [[ -n "$TARGET_CSV" ]]; then
+      info "Target CSV: ${TARGET_CSV}"
+      return 0
+    fi
+    info "  packagemanifest not ready yet, retrying in 10s..."
+    sleep 10
+  done
+  fail "Could not resolve currentCSV for channel ${channel} from ${catalog_label} after 60s"
+}
+
+# Install from official redhat-operators at channel rhacs-MAJOR.MINOR.
+# Sets TARGET_CSV to the channel's currentCSV.
+install_from_official() {
+  local major="$1" minor="$2"
+  local channel="rhacs-${major}.${minor}"
+  info "Installing ACS Operator from redhat-operators, channel ${channel}..."
+  wait_for_catalog "redhat-operators" "openshift-marketplace" 120
+  _resolve_target_csv "redhat-operators" "$channel"
+  apply_subscription "$channel" "redhat-operators"
+}
+
+# Install from the custom index image on the given channel (disables default sources).
+# Sets TARGET_CSV to the channel's currentCSV.
+install_from_custom() {
+  local index_image="$1" channel="$2"
+  info "Installing ACS Operator from custom index, channel ${channel}..."
+  disable_default_sources
+  apply_custom_catalog "$index_image"
+  wait_for_catalog "my-operator-catalog"
+  _resolve_target_csv "my-operator-catalog" "$channel"
+  apply_subscription "$channel" "my-operator-catalog"
+}
+
+# Upgrade by refreshing the custom CatalogSource and pointing the subscription at it.
+# Sets TARGET_CSV to the channel's currentCSV.
+upgrade_via_custom() {
+  local index_image="$1" channel="$2"
+  info "Upgrading via custom catalog (channel: ${channel})..."
+  apply_custom_catalog "$index_image"
+  disable_default_sources
+  wait_for_catalog "my-operator-catalog"
+  _resolve_target_csv "my-operator-catalog" "$channel"
+  apply_subscription "$channel" "my-operator-catalog"
+  ok "Subscription updated — OLM will upgrade to ${TARGET_CSV}"
+}
+
+# Upgrade to latest official GA via redhat-operators.
+# Sets TARGET_CSV to the latest channel's currentCSV.
+upgrade_to_latest_official() {
+  local major="${1:-4}"
+  enable_default_sources
+  wait_for_catalog "redhat-operators" "openshift-marketplace" 180
+  local latest_minor
+  latest_minor=$(get_latest_official_minor "$major") || true
+  [[ -n "$latest_minor" ]] || fail "Could not determine latest GA minor from redhat-operators"
+  local channel="rhacs-${major}.${latest_minor}"
+  info "Upgrading to latest GA channel: ${channel}..."
+  _resolve_target_csv "redhat-operators" "$channel"
+  apply_subscription "$channel" "redhat-operators"
+  ok "Subscription updated to ${channel}, target: ${TARGET_CSV}"
+}
+
+get_current_csv() {
+  oc get csv -n openshift-operators --no-headers 2>/dev/null \
+    | awk '/rhacs-operator/ && /Succeeded/ {print $1}' \
+    | head -1
+}
+
+# Waits for a specific CSV to reach Succeeded phase.
+# Use after install/upgrade to wait for the exact target version, not just any change.
+# This correctly handles multi-hop OLM upgrade graphs (e.g. 4.8→4.9→4.10).
+wait_for_csv() {
+  local target_csv="$1" timeout="${2:-600}"
+  local deadline
+  deadline=$(( $(date +%s) + timeout ))
+  info "Waiting for ${target_csv} to reach Succeeded (timeout: ${timeout}s)..."
+  while (( $(date +%s) < deadline )); do
+    local phase
+    phase=$(oc get csv "$target_csv" -n openshift-operators \
+      -o jsonpath='{.status.phase}' 2>/dev/null || true)
+    if [[ "$phase" == "Succeeded" ]]; then
+      ok "${target_csv} is Succeeded"
+      return 0
+    fi
+    # Show in-progress CSVs so logs show the hop chain
+    local progress
+    progress=$(oc get csv -n openshift-operators --no-headers 2>/dev/null \
+      | awk '/rhacs-operator/ {printf "%s(%s) ", $1, $7}' || true)
+    info "  ${progress:+CSVs: ${progress}}phase=${phase:-not found}, waiting 15s..."
+    sleep 15
+  done
+  fail "${target_csv} did not reach Succeeded within ${timeout}s"
+}
+
+log_csv() {
+  local csv
+  csv=$(oc get csv -n openshift-operators --no-headers 2>/dev/null \
+    | grep rhacs-operator || echo "<none>")
+  info "Current CSV: ${csv}"
+}
+
+# Reset between tests
+reset_operator() {
+  info "Resetting operator state for next test..."
+  oc delete subscription rhacs-operator -n openshift-operators --ignore-not-found
+  oc get csv -n openshift-operators --no-headers 2>/dev/null \
+    | awk '/rhacs-operator/ {print $1}' \
+    | xargs -r oc delete csv -n openshift-operators --ignore-not-found || true
+  oc delete catalogsource my-operator-catalog -n openshift-marketplace --ignore-not-found || true
+  enable_default_sources
+  ok "Operator state reset"
+}