Make setup idempotent for re-runs

.devcontainer/setup.sh

@@ -1,12 +1,17 @@
 #!/usr/bin/env bash
 
-set -e
+set -ex
 
 # Persist workflow env vars so they survive across terminal sessions
 ZENKO_ENV_FILE="$HOME/.zenko.env"
 yq eval '.env | to_entries | .[] | "export " + .key + "=" + (.value | tostring | @sh)' .github/workflows/end2end.yaml \
     | sed 's/\${{[^}]*}}//g' > "$ZENKO_ENV_FILE"
 echo 'export GIT_ACCESS_TOKEN="${GITHUB_TOKEN}"' >> "$ZENKO_ENV_FILE"
+
+echo 'export VOLUME_ROOT=$PWD/artifacts' >> "$ZENKO_ENV_FILE"
+echo 'export ZENKO_MONGODB_DATABASE=${ZENKO_MONGODB_DATABASE:-zenko-database}' >> "$ZENKO_ENV_FILE"
+echo "export HOST_DNS=$(awk '/^nameserver/{print \$2; exit}' /etc/resolv.conf)" >> "$ZENKO_ENV_FILE"
+mkdir -p "$PWD/artifacts/data"
 # Disable GCP tests as we don't have credentials setup in devcontainer
 echo 'export GCP_BACKEND_DESTINATION_LOCATION=' >> "$ZENKO_ENV_FILE"
 
@@ -26,28 +31,31 @@ for i in $(seq 0 $array_length); do
     working_dir=$(yq ".runs.steps[$i].working-directory" .github/actions/deploy/action.yaml)
     run_command=$(yq ".runs.steps[$i].run" .github/actions/deploy/action.yaml)
 
-    # We can't run `configure-e2e.sh` here because it needs services to be ready first, will be run after
-    # User will run tests manually after deployment
     (
-        if [[ "$run_command" != "null" && "$run_command" != *"configure-e2e.sh"* ]]; then
-            # Inject env 'generated' from previous steps
-            source "$GITHUB_ENV"
-
-            # Inject variables
-            # We use `sed` to replace github variable references and avoid bad substitution error from bash
-            env_variables=$(yq '.runs.steps['$i'].env | to_entries | .[] | .key + "=" + .value' .github/actions/deploy/action.yaml \
-                | sed -e 's/${{ *inputs.\([[:graph:]]*\) *}}/$GITHUB_INPUTS_\1/' -e 's/\${{.*}}//' \
-                | envsubst )
-            [ -n "$env_variables" ] && export $env_variables
-
-            if [ "$working_dir" != "null" ]; then
-                echo "Changing working dir: $working_dir"
-                cd $working_dir
-            fi
-
-            echo "Run command: $run_command"
-            eval "$run_command";
+        [[ "$run_command" == "null" ]] && exit 0
+        # We can't run `configure-e2e.sh` here because it needs services to be ready first, will be run after
+        [[ "$run_command" == *"configure-e2e.sh"* ]] && exit 0
+        # We don't want to run `run-e2e-test.sh` because it is used for linting here, user will run it manually if needed after deployment
+        [[ "$run_command" == *"run-e2e-test.sh"* ]] && exit 0
+        [[ "$run_command" == *"deploy-metadata.sh"* && "${ENABLE_RING_TESTS}" == "false" ]] && exit 0
+
+        # Inject env 'generated' from previous steps
+        source "$GITHUB_ENV"
+
+        # Inject variables
+        # We use `sed` to replace github variable references and avoid bad substitution error from bash
+        env_variables=$(yq '.runs.steps['$i'].env | to_entries | .[] | .key + "=" + .value' .github/actions/deploy/action.yaml \
+            | sed -e 's/${{ *inputs.\([[:graph:]]*\) *}}/$GITHUB_INPUTS_\1/' -e 's/\${{.*}}//' \
+            | envsubst )
+        [ -n "$env_variables" ] && export $env_variables
+
+        if [ "$working_dir" != "null" ]; then
+            echo "Changing working dir: $working_dir"
+            cd $working_dir
         fi
+
+        echo "Run command: $run_command"
+        eval "$run_command";
     )
 done
 
@@ -59,11 +67,10 @@ done
     bash configure-e2e-ctst.sh
 )
 
-docker image prune -af
-
-# Build CTST image from current branch 
+# Build CTST image from current branch
 SORBET_TAG=$(yq eval '.sorbet.tag' solution/deps.yaml)
 DRCTL_TAG=$(yq eval '.drctl.tag' solution/deps.yaml)
 TAG_NAME=ctst_codespace_setup
 GIT_AUTH_TOKEN=$GITHUB_TOKEN docker build --secret id=GIT_AUTH_TOKEN --build-arg SORBET_TAG=$SORBET_TAG --build-arg DRCTL_TAG=$DRCTL_TAG -t $E2E_CTST_IMAGE_NAME:$TAG_NAME ./tests/ctst
 kind load docker-image ${E2E_CTST_IMAGE_NAME}:$TAG_NAME
+docker rmi ${E2E_CTST_IMAGE_NAME}:$TAG_NAME

.github/actions/deploy/action.yaml

@@ -40,7 +40,9 @@ runs:
           ${{ runner.os }}-helm-
     - name: Generate MongoDB database name
       shell: bash
-      run: echo "ZENKO_MONGODB_DATABASE=$(cat /proc/sys/kernel/random/uuid)" >> "$GITHUB_ENV"
+      run: |
+        [ -z "$ZENKO_MONGODB_DATABASE" ] && ZENKO_MONGODB_DATABASE=$(cat /proc/sys/kernel/random/uuid)
+        echo "ZENKO_MONGODB_DATABASE=$ZENKO_MONGODB_DATABASE" >> "$GITHUB_ENV"
     - name: Install kind cluster dependencies
       shell: bash
       run: bash install-kind-dependencies.sh
@@ -71,14 +73,14 @@ runs:
       run: sh tests/smoke/deploy-sorbet-resources.sh end2end
       env:
         SORBETD_NAME: mock-sorbet
+        PVC_NAME: sorbet-data
       working-directory: ./.github/scripts/end2end/operator
     - name: Start mock miria service
       shell: bash
-      run: |
-        sed -i 's/sorbet-data/miria-data/g' tests/smoke/deploy-sorbet-resources.sh
-        tests/smoke/deploy-sorbet-resources.sh end2end
+      run: tests/smoke/deploy-sorbet-resources.sh end2end
       env:
         SORBETD_NAME: mock-miria
+        PVC_NAME: miria-data
       working-directory: ./.github/scripts/end2end/operator
     - name: Deploy metadata
       shell: bash

.github/scripts/end2end/configure-e2e-ctst.sh

@@ -32,8 +32,6 @@ export NOTIF_KAFKA_SCRAM_PORT=9095
 kubectl get zookeepercluster "${ZENKO_NAME}-base-quorum" -o json | jq '.
 | .metadata |= {namespace, name: "\(.name)-auth" }
 | del(.spec.labels)
-| del(.spec.persistence)
-| .spec.storageType |= "ephemeral"
 | del(.spec.pod.affinity)
 | del(.spec.pod.labels)
 | del(.status)

.github/scripts/end2end/deploy-metadata.sh

@@ -5,7 +5,7 @@ set -exu
 . "$(dirname $0)/common.sh"
 
 # create a separate namespace for metadata
-kubectl create namespace metadata
+kubectl create namespace metadata --dry-run=client -o yaml | kubectl apply -f -
 
 # clone the metadata repository
 git init metadata

.github/scripts/end2end/deploy-zenko.sh

@@ -101,34 +101,39 @@ create_encryption_secret()
     PRIVATE=$(mktemp zenko-key.XXXXXX)
     trap 'rm -f "$PUBLIC" "$PRIVATE"' EXIT INT HUP TERM
 
-    # Get the OpenSSL version
-    OPENSSL_VERSION=$(openssl version | awk '{print $2}')
-
-    # Check if OpenSSL 3.x is being used
-    if [[ $OPENSSL_VERSION =~ ^3\..* ]]; then
-        # Use the "-traditional" flag for OpenSSL 3.x
-        openssl genrsa -out "$PRIVATE" -traditional
+    if kubectl get secret ${ZENKO_NAME}-keypair.v0 --namespace ${NAMESPACE} >/dev/null 2>/dev/null; then
+        kubectl get secret ${ZENKO_NAME}-keypair.v0 --namespace ${NAMESPACE} \
+            -o jsonpath='{.data.publicKey}' | base64 -d > "$PUBLIC"
     else
-        openssl genrsa -out "$PRIVATE"
+        # Get the OpenSSL version
+        OPENSSL_VERSION=$(openssl version | awk '{print $2}')
+
+        # Check if OpenSSL 3.x is being used
+        if [[ $OPENSSL_VERSION =~ ^3\..* ]]; then
+            # Use the "-traditional" flag for OpenSSL 3.x
+            openssl genrsa -out "$PRIVATE" -traditional
+        else
+            openssl genrsa -out "$PRIVATE"
+        fi
+
+        openssl rsa -in "$PRIVATE" -pubout -out "$PUBLIC"
+
+        # Zkop expects PKCS#1 format, but with a type of 'PRIVATE KEY' as generated with older openssl
+        sed -i 's/RSA PRIVATE KEY/PRIVATE KEY/' "$PRIVATE"
+
+        kubectl create secret generic ${ZENKO_NAME}-keypair.v0 \
+            --namespace ${NAMESPACE} \
+            --from-file=publicKey="$PUBLIC" \
+            --from-file=privateKey="$PRIVATE"
     fi
 
-    openssl rsa -in "$PRIVATE" -pubout -out "$PUBLIC"
-
     AZURE_SECRET_KEY_ENCRYPTED="$(
         printf '%s' "${AZURE_SECRET_KEY}" \
         | openssl pkeyutl -encrypt -pubin -inkey "$PUBLIC" \
                   -pkeyopt rsa_padding_mode:oaep -pkeyopt rsa_oaep_md:sha256 -pkeyopt rsa_mgf1_md:sha256 \
         | base64 -w 0
     )"
 
-    # Zkop expects PKCS#1 format, but with a type of 'PRIVATE KEY' as generated with older openssl
-    sed -i 's/RSA PRIVATE KEY/PRIVATE KEY/' "$PRIVATE"
-
-    kubectl create secret generic ${ZENKO_NAME}-keypair.v0 \
-        --namespace ${NAMESPACE} \
-        --from-file=publicKey="$PUBLIC" \
-        --from-file=privateKey="$PRIVATE"
-
     export AZURE_SECRET_KEY_ENCRYPTED
 }
 

.github/scripts/end2end/deploy-zkop.sh

@@ -2,12 +2,8 @@
 
 set -ex
 
-[ -z "${OPERATOR_IMAGE_NAME}" ] && OPERATOR_IMAGE_NAME="$(yq eval '."zenko-operator" | .sourceRegistry + "/" + .image' solution/deps.yaml)"
 [ -z "${OPERATOR_IMAGE_TAG}" ] && OPERATOR_IMAGE_TAG="$(yq eval '."zenko-operator".tag' solution/deps.yaml)"
 
-docker pull "${OPERATOR_IMAGE_NAME}:${OPERATOR_IMAGE_TAG}"
-kind load docker-image "${OPERATOR_IMAGE_NAME}:${OPERATOR_IMAGE_TAG}"
-
 OPERATOR_PATH=./.github/scripts/end2end/operator
 git init $OPERATOR_PATH
 cd $OPERATOR_PATH

.github/scripts/end2end/install-kind-dependencies.sh

@@ -43,17 +43,23 @@ else
     KEYCLOAK_INGRESS_OPTIONS="$DIR/configs/keycloak_ingress_http.yaml"
 fi
 
-helm repo add --force-update bitnami https://charts.bitnami.com/bitnami
-helm repo add --force-update pravega https://charts.pravega.io
-helm repo add --force-update codecentric https://codecentric.github.io/helm-charts/
+helm_repo_add() {
+    helm repo list -o json 2>/dev/null | jq -e --arg n "$1" '.[] | select(.name == $n)' >/dev/null 2>&1 || helm repo add "$1" "$2"
+}
+
+helm_repo_add bitnami https://charts.bitnami.com/bitnami
+helm_repo_add pravega https://charts.pravega.io
+helm_repo_add codecentric https://codecentric.github.io/helm-charts/
 # BanzaiCloud repo may not work, c.f. https://scality.atlassian.net/browse/AN-225
-helm repo add --force-update banzaicloud-stable https://kubernetes-charts.banzaicloud.com || {
+helm_repo_add banzaicloud-stable https://kubernetes-charts.banzaicloud.com || {
 		echo -n "::notice file=$(basename $0),line=$LINENO,title=Banzaicloud Charts not available::"
 		echo "Failed to add banzaicloud-stable repo, using local checkout"
 
-		kafka_operator="$(mktemp -d)"
-		git -c advice.detachedHead=false clone -q --depth 1 -b "v${KAFKA_OPERATOR_VERSION}" \
-            https://github.com/banzaicloud/koperator "${kafka_operator}"
+		kafka_operator="${DIR}/kafka-operator"
+		if [ ! -d "${kafka_operator}" ]; then
+			git -c advice.detachedHead=false clone -q --depth 1 -b "v${KAFKA_OPERATOR_VERSION}" \
+				https://github.com/banzaicloud/koperator "${kafka_operator}"
+		fi
 
 		KAFKA_CHART="${kafka_operator}/charts/kafka-operator"
 	}
@@ -119,15 +125,13 @@ kubectl wait --for=condition=Ready --timeout=240s clusterissuer/artesca-root-ca-
 
 # Copy root CA secret to default namespace for applications to use
 echo "Copying root CA certificate to default namespace..."
-kubectl get secret root-ca -n cert-manager -o json | 
-  jq '.metadata.namespace="default" | .metadata.name="zenko-root-ca"' | 
+kubectl get secret root-ca -n cert-manager -o json |
+  jq '.metadata = {namespace: "default", name: "zenko-root-ca"}' |
   kubectl apply -f -
 
 # prometheus
-# last-applied-configuration can end up larger than 256kB  which is too large for an annotation
-# so if apply fails, replace can work
 prom_url=https://raw.githubusercontent.com/coreos/prometheus-operator/${PROMETHEUS_VERSION}/bundle.yaml
-kubectl create -f $prom_url || kubectl replace -f $prom_url --wait
+kubectl apply --server-side -f $prom_url
 # wait for the resource to exist
 kubectl wait --for=condition=established --timeout=10m crd/alertmanagers.monitoring.coreos.com
 envsubst < configs/prometheus.yaml | kubectl apply -f -
@@ -137,14 +141,14 @@ helm upgrade --install --version ${ZK_OPERATOR_VERSION} -n default zk-operator p
 
 # kafka
 kafka_crd_url=https://github.com/banzaicloud/koperator/releases/download/v${KAFKA_OPERATOR_VERSION}/kafka-operator.crds.yaml
-kubectl create -f $kafka_crd_url || kubectl replace -f $kafka_crd_url
+kubectl apply --server-side -f $kafka_crd_url
 helm upgrade --install --version ${KAFKA_OPERATOR_VERSION} -n default kafka-operator ${KAFKA_CHART} \
     --set prometheusMetrics.authProxy.image.repository=quay.io/brancz/kube-rbac-proxy \
     --set prometheusMetrics.authProxy.image.tag=v0.21.0
 
 # keycloak
 envsubst < $DIR/configs/keycloak_config.json > $DIR/configs/keycloak-realm.json
-kubectl create configmap keycloak-realm --from-file=$DIR/configs/keycloak-realm.json
+kubectl create configmap keycloak-realm --from-file=$DIR/configs/keycloak-realm.json --dry-run=client -o yaml | kubectl apply -f -
 helm upgrade --install --version ${KEYCLOAK_VERSION} keycloak codecentric/keycloak -f "$DIR/configs/keycloak_options.yaml" -f "${KEYCLOAK_INGRESS_OPTIONS}"
 
 kubectl rollout status sts/keycloak --timeout=10m

.github/scripts/end2end/install-mocks.sh

@@ -7,9 +7,10 @@ NAMESPACE=${1:-default}
 kubectl create \
     configmap aws-mock \
     --from-file=../mocks/aws/mock-metadata.tar.gz \
-    --namespace ${NAMESPACE}
+    --namespace ${NAMESPACE} \
+    --dry-run=client -o yaml | kubectl apply -f -
 
-kubectl create \
+kubectl apply \
     -f ../mocks/azure-mock.yaml \
     -f ../mocks/aws-mock.yaml \
     --namespace ${NAMESPACE} && \

.github/scripts/end2end/run-e2e-ctst.sh

@@ -159,7 +159,8 @@ CTST_VERSION=$(sed 's/.*"cli-testing": ".*#\(.*\)".*/\1/;t;d' ../../../tests/cts
 # Grant access to Kube API (insecure, only for testing)
 kubectl create clusterrolebinding serviceaccounts-cluster-admin \
   --clusterrole=cluster-admin \
-  --group=system:serviceaccounts
+  --group=system:serviceaccounts \
+  --dry-run=client -o yaml | kubectl apply -f -
 
 # Running end2end ctst tests
 # Using overrides as we need to attach a local folder to the pod

tests/ctst/run-ctst-locally.sh

@@ -14,14 +14,17 @@ IMAGE_NAME="${2:-ghcr.io/scality/zenko/zenko-e2e-ctst:ctst_codespace_setup}"
 # certain tests based on their @version tag.
 VERSION=$(cat ../../VERSION | grep -Po 'VERSION="\K[^"]*')
 POD_NAME="ctst-end2end"
+ZENKO_NAME="${ZENKO_NAME:-end2end}"
+ADMIN_ACCESS_KEY=$(kubectl get secret "${ZENKO_NAME}-management-vault-admin-creds.v1" -o jsonpath='{.data.accessKey}' | base64 -d)
+ADMIN_SECRET_KEY=$(kubectl get secret "${ZENKO_NAME}-management-vault-admin-creds.v1" -o jsonpath='{.data.secretKey}' | base64 -d)
 WORLD_PARAMETERS="$(jq -c <<EOF
 {
   "subdomain": "zenko.local",
   "ssl": false,
   "port": "80",
-  "AccountName": "zenko",
-  "AdminAccessKey": "R6JN4OJ998ZBY99DD56X",
-  "AdminSecretKey": "OEow2DytG0sr7mK3844vb/hKDBWiU+UWc+4+FVfr"
+  "AccountName": "zenko-ctst",
+  "AdminAccessKey": "$ADMIN_ACCESS_KEY",
+  "AdminSecretKey": "$ADMIN_SECRET_KEY"
 }
 EOF
 )"