WIP: Add SM3 e2e setup

Makefile

@@ -36,6 +36,10 @@ install-serving-with-mesh: install-tools
 	UNINSTALL_MESH="false" ./hack/mesh.sh
 	MESH=true SCALE_UP=4 INSTALL_SERVING=true INSTALL_EVENTING="false" ./hack/install.sh
 
+install-serving-with-mesh3: install-tools
+	UNINSTALL_MESH="false" MESH_VERSION=3 ./hack/mesh.sh
+	MESH=true MESH_VERSION=3 SCALE_UP=4 INSTALL_SERVING=true INSTALL_EVENTING="false" ./hack/install.sh
+
 install-eventing: install-tools
 	INSTALL_SERVING="false" ./hack/install.sh
 
@@ -82,6 +86,12 @@ install-mesh:
 uninstall-mesh:
 	UNINSTALL_MESH="true" ./hack/mesh.sh
 
+install-mesh3:
+	UNINSTALL_MESH="false" MESH_VERSION=3 ./hack/mesh.sh
+
+uninstall-mesh3:
+	UNINSTALL_MESH="true" MESH_VERSION=3 ./hack/mesh.sh
+
 install-tracing-zipkin:
 	TRACING_BACKEND=zipkin ./hack/tracing.sh
 
@@ -150,6 +160,17 @@ test-e2e-with-mesh: install-tools
 	MESH=true SCALE_UP=4 INSTALL_KAFKA="true" ENABLE_TRACING=true ./hack/install.sh
 	MESH=true TEST_KNATIVE_KAFKA=true ./test/e2e-tests.sh
 
+# Run E2E tests from the current repo for serving+eventing+mesh3
+test-e2e-with-mesh3-testonly:
+	MESH=true MESH_VERSION=3 ./test/e2e-tests.sh
+
+test-e2e-with-mesh3: install-tools
+	UNINSTALL_MESH="false" MESH_VERSION=3 ./hack/mesh.sh
+	./hack/tracing.sh
+	UNINSTALL_STRIMZI="false" ./hack/strimzi.sh
+	MESH=true MESH_VERSION=3 SCALE_UP=4 INSTALL_KAFKA="true" ENABLE_TRACING=true ./hack/install.sh
+	MESH=true MESH_VERSION=3 TEST_KNATIVE_KAFKA=true ./test/e2e-tests.sh
+
 # Run both unit and E2E tests from the current repo.
 test-operator: test-unit test-e2e
 
@@ -205,11 +226,11 @@ test-upgrade: install-tools
 	TEST_KNATIVE_KAFKA=true TEST_KNATIVE_E2E=false TEST_KNATIVE_UPGRADE=true ./test/upstream-e2e-tests.sh
 
 mesh-upgrade: install-tools
-	UNINSTALL_MESH=false ./hack/mesh.sh
+	UNINSTALL_MESH=false MESH_VERSION=2 ./hack/mesh.sh
 	TRACING_BACKEND=zipkin ./hack/tracing.sh
 	UNINSTALL_STRIMZI=false ./hack/strimzi.sh
-	MESH=true INSTALL_PREVIOUS_VERSION=true INSTALL_KAFKA=true TRACING_BACKEND=zipkin ENABLE_TRACING=true SCALE_UP=6 ./hack/install.sh
-	MESH=true TEST_KNATIVE_KAFKA=true TEST_KNATIVE_E2E=false TEST_KNATIVE_UPGRADE=true ./test/upstream-e2e-tests.sh
+	MESH=true MESH_VERSION=2 INSTALL_PREVIOUS_VERSION=true INSTALL_KAFKA=true TRACING_BACKEND=zipkin ENABLE_TRACING=true SCALE_UP=6 ./hack/install.sh
+	MESH=true MESH_VERSION=2 TEST_KNATIVE_KAFKA=true TEST_KNATIVE_E2E=false TEST_KNATIVE_UPGRADE=true ./test/upstream-e2e-tests.sh
 
 test-upgrade-with-mesh: mesh-upgrade
 

hack/lib/__sources__.bash

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-declare -a __sources=(metadata vars images common ui scaleup namespaces serverless catalog olmv0_catalog olmv1_catalog tracing mesh certmanager strimzi keda tracing clusterlogging testselect)
+declare -a __sources=(metadata vars images common ui scaleup namespaces serverless catalog olmv0_catalog olmv1_catalog tracing mesh mesh3 certmanager strimzi keda tracing clusterlogging testselect)
 
 for source in "${__sources[@]}"; do
   # shellcheck disable=SC1091,SC1090

hack/lib/mesh3.bash

@@ -0,0 +1,152 @@
+#!/usr/bin/env bash
+
+mesh_v3_resources_dir="$(dirname "${BASH_SOURCE[0]}")/mesh_v3_resources"
+
+function install_mesh3 {
+  ensure_catalog_pods_running
+  deploy_sail_operator
+  deploy_istio
+  deploy_mesh3_gateways
+}
+
+function uninstall_mesh3 {
+  undeploy_mesh3_gateways
+  undeploy_istio
+  undeploy_sail_operator
+}
+
+function deploy_sail_operator {
+  if [[ ${SKIP_OPERATOR_SUBSCRIPTION:-} != "true" ]]; then
+    logger.info "Installing Service Mesh 3 operator in namespace openshift-operators"
+    oc apply -f "${mesh_v3_resources_dir}"/01_subscription.yaml || return $?
+  fi
+
+  logger.info "Waiting until Service Mesh 3 operator is available"
+  timeout 600 "[[ \$(oc get deploy -n openshift-operators servicemesh-operator3 --no-headers 2>/dev/null | wc -l) != 1 ]]" || return 1
+  oc wait --for=condition=Available deployment servicemesh-operator3 --timeout=300s -n openshift-operators || return $?
+}
+
+function undeploy_sail_operator {
+  logger.info "Deleting Service Mesh 3 operator subscription"
+  oc delete subscriptions.operators.coreos.com -n openshift-operators servicemeshoperator3 --ignore-not-found
+
+  logger.info 'Deleting ClusterServiceVersion'
+  for csv in $(set +o pipefail && oc get csv -n openshift-operators --no-headers 2>/dev/null \
+      | grep 'servicemeshoperator3' | cut -f1 -d' '); do
+    oc delete csv -n openshift-operators "${csv}"
+  done
+
+  logger.info 'Ensure no operators present'
+  timeout 600 "[[ \$(oc get deployments -n openshift-operators -oname | grep -c 'servicemeshoperator3') != 0 ]]"
+
+  logger.info 'Ensure no CRDs left'
+  if [[ ! $(oc get crd -oname | grep -c 'sailoperator.io') -eq 0 ]]; then
+    oc get crd -oname | grep 'sailoperator.io' | xargs oc delete --timeout=60s
+  fi
+  logger.success "Service Mesh 3 operator has been uninstalled"
+}
+
+function deploy_istio {
+  logger.info "Installing Istio and IstioCNI"
+
+  # Make sure istios.sailoperator.io CRD is available.
+  timeout 120 "[[ \$(oc get crd istios.sailoperator.io --no-headers 2>/dev/null | wc -l) != 1 ]]" || return 1
+  oc wait --for=condition=Established crd istios.sailoperator.io
+
+  # Create namespaces for Istio and IstioCNI.
+  oc get ns istio-system || oc create namespace istio-system
+  oc get ns istio-cni || oc create namespace istio-cni
+
+  # Substitute the MESH3_ISTIO_VERSION placeholder and apply Istio CR.
+  local istio_cr
+  istio_cr="$(mktemp -t istio-XXXXX.yaml)"
+  sed "s/MESH3_ISTIO_VERSION/${MESH3_ISTIO_VERSION}/g" "${mesh_v3_resources_dir}/02_istio.yaml" > "${istio_cr}"
+  oc apply -f "${istio_cr}" -n istio-system || return $?
+
+  # Substitute the MESH3_ISTIO_VERSION placeholder and apply IstioCNI CR.
+  local istiocni_cr
+  istiocni_cr="$(mktemp -t istiocni-XXXXX.yaml)"
+  sed "s/MESH3_ISTIO_VERSION/${MESH3_ISTIO_VERSION}/g" "${mesh_v3_resources_dir}/03_istiocni.yaml" > "${istiocni_cr}"
+  oc apply -f "${istiocni_cr}" -n istio-cni || return $?
+
+  timeout 120 "[[ \$(oc get istio -n istio-system default --no-headers 2>/dev/null | wc -l) != 1 ]]" || return 1
+
+  oc wait --timeout=180s --for=condition=Ready istio -n istio-system default || oc get istio -n istio-system default -o yaml
+  oc wait --timeout=180s --for=condition=Ready istiocni -n istio-cni default || oc get istiocni -n istio-cni default -o yaml
+
+  rm -f "${istio_cr}" "${istiocni_cr}"
+}
+
+function undeploy_istio {
+  logger.info "Deleting Istio and IstioCNI"
+  oc delete istiocni -n istio-cni default --ignore-not-found || return $?
+  oc delete istio -n istio-system default --ignore-not-found || return $?
+}
+
+function deploy_mesh3_gateways {
+  # Generate wildcard certs with cluster's subdomain.
+  local out_dir
+  out_dir="$(mktemp -d /tmp/certs-XXX)"
+
+  openssl req -x509 -sha256 -nodes -days 365 -newkey rsa:2048 \
+    -subj "/O=Example Inc./CN=Example" \
+    -keyout "${out_dir}"/root.key \
+    -out "${out_dir}"/root.crt
+
+  subdomain=$(oc get ingresses.config.openshift.io cluster -o jsonpath="{.spec.domain}")
+  openssl req -nodes -newkey rsa:2048 \
+      -subj "/O=Example Inc./CN=Example" \
+      -reqexts san \
+      -config <(printf "[req]\ndistinguished_name=req\n[san]\nsubjectAltName=DNS:*.%s" "$subdomain") \
+      -keyout "${out_dir}"/wildcard.key \
+      -out "${out_dir}"/wildcard.csr
+
+  openssl x509 -req -days 365 -set_serial 0 \
+      -extfile <(printf "subjectAltName=DNS:*.%s" "$subdomain") \
+      -CA "${out_dir}"/root.crt \
+      -CAkey "${out_dir}"/root.key \
+      -in "${out_dir}"/wildcard.csr \
+      -out "${out_dir}"/wildcard.crt
+
+  oc get ns knative-serving-ingress || oc create namespace knative-serving-ingress
+
+  # Wildcard certs go into knative-serving-ingress for SM3.
+  oc create -n knative-serving-ingress secret tls wildcard-certs \
+      --key="${out_dir}"/wildcard.key \
+      --cert="${out_dir}"/wildcard.crt --dry-run=client -o yaml | oc apply -f -
+
+  # ca-key-pair secret in cert-manager namespace needed for upstream e2e test with https option.
+  oc get ns cert-manager || oc create namespace cert-manager
+  oc create -n cert-manager secret tls ca-key-pair \
+      --key="${out_dir}"/wildcard.key \
+      --cert="${out_dir}"/wildcard.crt --dry-run=client -o yaml | oc apply -f -
+
+  oc apply -f "${mesh_v3_resources_dir}"/04_namespace.yaml || return $?
+  oc apply -f "${mesh_v3_resources_dir}"/05_gateway_deploy.yaml || return $?
+  oc apply -f "${mesh_v3_resources_dir}"/06_serving_gateways.yaml || return $?
+  oc apply -f "${mesh_v3_resources_dir}"/07_peer_authentication.yaml || return $?
+
+  oc apply -f "${mesh_v3_resources_dir}"/authorization-policies/setup || return $?
+  oc apply -f "${mesh_v3_resources_dir}"/authorization-policies/helm || return $?
+
+  oc apply -n "${EVENTING_NAMESPACE}" -f "${mesh_v3_resources_dir}"/kafka-service-entry.yaml || return $?
+  for ns in serverless-tests eventing-e2e0 eventing-e2e1 eventing-e2e2 eventing-e2e3 eventing-e2e4; do
+    oc apply -n "$ns" -f "${mesh_v3_resources_dir}"/kafka-service-entry.yaml || return $?
+  done
+  oc apply -n "serverless-tests" -f "${mesh_v3_resources_dir}"/network-policy-monitoring.yaml || return $?
+}
+
+function undeploy_mesh3_gateways {
+  oc delete -n serverless-tests -f "${mesh_v3_resources_dir}"/network-policy-monitoring.yaml --ignore-not-found || return $?
+  for ns in serverless-tests eventing-e2e0 eventing-e2e1 eventing-e2e2 eventing-e2e3 eventing-e2e4; do
+    oc delete -n "$ns" -f "${mesh_v3_resources_dir}"/kafka-service-entry.yaml --ignore-not-found || return $?
+  done
+  oc delete -n "${EVENTING_NAMESPACE}" -f "${mesh_v3_resources_dir}"/kafka-service-entry.yaml --ignore-not-found || return $?
+  oc delete -f "${mesh_v3_resources_dir}"/authorization-policies/helm --ignore-not-found || return $?
+  oc delete -f "${mesh_v3_resources_dir}"/authorization-policies/setup --ignore-not-found || return $?
+  oc delete -f "${mesh_v3_resources_dir}"/07_peer_authentication.yaml --ignore-not-found || return $?
+  oc delete -f "${mesh_v3_resources_dir}"/06_serving_gateways.yaml --ignore-not-found || return $?
+  oc delete -f "${mesh_v3_resources_dir}"/05_gateway_deploy.yaml --ignore-not-found || return $?
+  oc delete -n cert-manager secret ca-key-pair --ignore-not-found || return $?
+  oc delete -n knative-serving-ingress secret wildcard-certs --ignore-not-found || return $?
+}

hack/lib/mesh_v3_resources/01_subscription.yaml

@@ -0,0 +1,14 @@
+apiVersion: operators.coreos.com/v1alpha1
+kind: Subscription
+metadata:
+  labels:
+    operators.coreos.com/servicemeshoperator3.openshift-operators: ""
+  name: servicemeshoperator3
+  namespace: openshift-operators
+spec:
+  channel: stable
+  installPlanApproval: Automatic
+  name: servicemeshoperator3
+  source: redhat-operators
+  sourceNamespace: openshift-marketplace
+  startingCSV: servicemeshoperator3.v3.2.2

hack/lib/mesh_v3_resources/02_istio.yaml

@@ -0,0 +1,19 @@
+apiVersion: sailoperator.io/v1
+kind: Istio
+metadata:
+  name: default
+spec:
+  values:
+    global:
+      proxy:
+        excludeInboundPorts: "8444,8022"
+    meshConfig:
+      accessLogFile: /dev/stdout
+      accessLogFormat: "{ \"authority\": \"%REQ(:AUTHORITY)%\", \"bytes_received\": %BYTES_RECEIVED%, \"bytes_sent\": %BYTES_SENT%, \"downstream_local_address\": \"%DOWNSTREAM_LOCAL_ADDRESS%\", \"downstream_peer_cert_v_end\": \"%DOWNSTREAM_PEER_CERT_V_END%\", \"downstream_peer_cert_v_start\": \"%DOWNSTREAM_PEER_CERT_V_START%\", \"downstream_remote_address\": \"%DOWNSTREAM_REMOTE_ADDRESS%\", \"downstream_tls_cipher\": \"%DOWNSTREAM_TLS_CIPHER%\", \"downstream_tls_version\": \"%DOWNSTREAM_TLS_VERSION%\", \"duration\": %DURATION%, \"hostname\": \"%HOSTNAME%\", \"istio_policy_status\": \"%DYNAMIC_METADATA(istio.mixer:status)%\", \"method\": \"%REQ(:METHOD)%\", \"path\": \"%REQ(X-ENVOY-ORIGINAL-PATH?:PATH)%\", \"protocol\": \"%PROTOCOL%\", \"request_duration\": %REQUEST_DURATION%, \"request_id\": \"%REQ(X-REQUEST-ID)%\", \"requested_server_name\": \"%REQUESTED_SERVER_NAME%\", \"response_code\": \"%RESPONSE_CODE%\", \"response_duration\": %RESPONSE_DURATION%, \"response_tx_duration\": %RESPONSE_TX_DURATION%, \"response_flags\": \"%RESPONSE_FLAGS%\", \"route_name\": \"%ROUTE_NAME%\", \"start_time\": \"%START_TIME%\", \"upstream_cluster\": \"%UPSTREAM_CLUSTER%\", \"upstream_host\": \"%UPSTREAM_HOST%\", \"upstream_local_address\": \"%UPSTREAM_LOCAL_ADDRESS%\", \"upstream_service_time\": %RESP(X-ENVOY-UPSTREAM-SERVICE-TIME)%, \"upstream_transport_failure_reason\": \"%UPSTREAM_TRANSPORT_FAILURE_REASON%\", \"user_agent\": \"%REQ(USER-AGENT)%\", \"x_forwarded_for\": \"%REQ(X-FORWARDED-FOR)%\" }\n"
+      defaultConfig:
+        terminationDrainDuration: 35s # needed to make QP stop hook work
+  updateStrategy:
+    inactiveRevisionDeletionGracePeriodSeconds: 30
+    type: InPlace
+  namespace: istio-system
+  version: MESH3_ISTIO_VERSION

hack/lib/mesh_v3_resources/03_istiocni.yaml

@@ -0,0 +1,7 @@
+apiVersion: sailoperator.io/v1
+kind: IstioCNI
+metadata:
+  name: default
+spec:
+  namespace: istio-cni
+  version: MESH3_ISTIO_VERSION

hack/lib/mesh_v3_resources/04_namespace.yaml

@@ -0,0 +1,90 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: knative-serving
+  labels:
+    istio-injection: enabled
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: knative-serving-ingress
+  labels:
+    istio-injection: enabled
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: knative-eventing
+  labels:
+    istio-injection: enabled
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: serving-tests
+  labels:
+    istio-injection: enabled
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: serving-tests-alt
+  labels:
+    istio-injection: enabled
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: serverless-tests
+  labels:
+    istio-injection: enabled
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: eventing-e2e0
+  labels:
+    istio-injection: enabled
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: eventing-e2e1
+  labels:
+    istio-injection: enabled
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: eventing-e2e2
+  labels:
+    istio-injection: enabled
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: eventing-e2e3
+  labels:
+    istio-injection: enabled
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: eventing-e2e4
+  labels:
+    istio-injection: enabled
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: tenant-1
+  labels:
+    istio-injection: enabled
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: tenant-2
+  labels:
+    istio-injection: enabled

hack/lib/mesh_v3_resources/05_gateway_deploy.yaml

@@ -0,0 +1,45 @@
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: knative-istio-ingressgateway
+  namespace: knative-serving-ingress
+spec:
+  selector:
+    matchLabels:
+      knative: ingressgateway
+  template:
+    metadata:
+      annotations:
+        inject.istio.io/templates: gateway
+      labels:
+        knative: ingressgateway
+        sidecar.istio.io/inject: "true"
+    spec:
+      containers:
+        - name: istio-proxy
+          image: auto
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: istio-ingressgateway-sds
+  namespace: knative-serving-ingress
+rules:
+  - apiGroups: [""]
+    resources: ["secrets"]
+    verbs: ["get", "watch", "list"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: istio-ingressgateway-sds
+  namespace: knative-serving-ingress
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: istio-ingressgateway-sds
+subjects:
+  - kind: ServiceAccount
+    name: default
+---