Makefile.test

# This makefile holds "end-to-end" tests, i.e. function usage with kpt and real manifest manipulation

SHELL=/bin/bash -o pipefail

FN_MODE ?= exec

ifeq ($(CONTAINER_TAG),)
APPLY_SETTERS_IMAGE := ghcr.io/krm-functions/apply-setters@sha256:5807049387f4f775464e7e41da251fe224ac8229fd307c32773949fd6187f256
DIGESTER_IMAGE := ghcr.io/krm-functions/digester@sha256:1285722a33e42a25a6c63067ab3b6c4be084d085754465329cb875d01dd140a1
GATEKEEPER_SET_ENFORCEMENT_ACTION_IMAGE := ghcr.io/krm-functions/gatekeeper-set-enforcement-action@sha256:cd2
HELM_RENDER_IMAGE := ghcr.io/krm-functions/render-helm-chart@sha256:ef7666e8ea762cdc32a53c50e00307cbe10e914f1aaf8cfd9b0bb5c7d278dfe7
HELM_SOURCE_IMAGE := ghcr.io/krm-functions/source-helm-chart@sha256:865232a46bca8e1294c64d0de35437e18936b049159d8329000b44fd4c84c99f
HELM_UPGRADER_IMAGE := ghcr.io/krm-functions/helm-upgrader@sha256:c32150ceb6661532e85793db39749f7091a14af02eedb9d1685bda936ad03598
KUBECONFORM_IMAGE := ghcr.io/krm-functions/kubeconform@sha256:10b02b73da6e02957471957bded732fc8f4bb02045b6187a378fcf761251be43
PACKAGE_COMPOSITOR_IMAGE := ghcr.io/krm-functions/package-compositor@sha256:04bdd68ece1e7f87c505394f56f82c761b65f42e8e063c9996b4764366fec195
REMOVE_LOCAL_CONFIG_RESOURCES_IMAGE := ghcr.io/krm-functions/remove-local-config-resources@sha256:af7e984553154265adb2782924efe0687b19f4b8d8961e95e379f9bfd040dad8
SET_ANNOTATIONS_IMAGE := ghcr.io/krm-functions/set-annotations@sha256:0d65e3637dbdc6b3397ccfb93f9d6864a263c92cb21524cf0f3ed4178cd32fe1
SET_LABELS_IMAGE := ghcr.io/krm-functions/set-labels@sha256:e49f8927f83d286d626c50f6c6df1e9e7896ec8f3192eb8bfdc3837c0098cadc
else
APPLY_SETTERS_IMAGE := ghcr.io/krm-functions/apply-setters:$(CONTAINER_TAG)
DIGESTER_IMAGE := ghcr.io/krm-functions/digester:$(CONTAINER_TAG)
GATEKEEPER_SET_ENFORCEMENT_ACTION_IMAGE := ghcr.io/krm-functions/gatekeeper-set-enforcement-action:$(CONTAINER_TAG)
HELM_RENDER_IMAGE := ghcr.io/krm-functions/render-helm-chart:$(CONTAINER_TAG)
HELM_SOURCE_IMAGE := ghcr.io/krm-functions/source-helm-chart:$(CONTAINER_TAG)
HELM_UPGRADER_IMAGE := ghcr.io/krm-functions/helm-upgrader:$(CONTAINER_TAG)
KUBECONFORM_IMAGE := ghcr.io/krm-functions/kubeconform:$(CONTAINER_TAG)
PACKAGE_COMPOSITOR_IMAGE := ghcr.io/krm-functions/package-compositor:$(CONTAINER_TAG)
REMOVE_LOCAL_CONFIG_RESOURCES := ghcr.io/krm-functions/remove-local-config-resources:$(CONTAINER_TAG)
SET_ANNOTATIONS_IMAGE := ghcr.io/krm-functions/set-annotations:$(CONTAINER_TAG)
SET_LABELS_IMAGE := ghcr.io/krm-functions/set-labels:$(CONTAINER_TAG)
endif

ifeq ($(FN_MODE),exec)
APPLY_SETTERS := --exec bin/linux_amd64/apply-setters
DIGESTER := --exec bin/linux_amd64/digester
GATEKEEPER_SET_ENFORCEMENT_ACTION := --exec bin/linux_amd64/gatekeeper-set-enforcement-action
HELM_RENDER := --exec bin/linux_amd64/render-helm-chart
HELM_SOURCE := --exec bin/linux_amd64/source-helm-chart
HELM_UPGRADER := --exec bin/linux_amd64/helm-upgrader
KUBECONFORM := --exec bin/linux_amd64/kubeconform
TEMPLATE_KYAML := --exec bin/linux_amd64/template-kyaml
PACKAGE_COMPOSITOR := --exec bin/linux_amd64/package-compositor
REMOVE_LOCAL_CONFIG_RESOURCES := --exec bin/linux_amd64/remove-local-config-resources
SET_ANNOTATIONS := --exec bin/linux_amd64/set-annotations
SET_LABELS := --exec bin/linux_amd64/set-labels
else
APPLY_SETTERS := --image $(APPLY_SETTERS_IMAGE)
DIGESTER := --network --image $(DIGESTER_IMAGE)
GATEKEEPER_SET_ENFORCEMENT_ACTION := --image $(GATEKEEPER_SET_ENFORCEMENT_ACTION_IMAGE)
HELM_RENDER := --network --image $(HELM_RENDER_IMAGE)
HELM_SOURCE := --network --image $(HELM_SOURCE_IMAGE)
HELM_UPGRADER := --network --image $(HELM_UPGRADER_IMAGE)
KUBECONFORM := --network --image $(KUBECONFORM_IMAGE)
PACKAGE_COMPOSITOR := --network --image $(PACKAGE_COMPOSITOR_IMAGE)
REMOVE_LOCAL_CONFIG_RESOURCES := --network --image $(REMOVE_LOCAL_CONFIG_RESOURCES_IMAGE)
SET_ANNOTATIONS := --network --image $(SET_ANNOTATIONS_IMAGE)
SET_LABELS := --network --image $(SET_LABELS_IMAGE)
endif

BASELINE_RENDER_HELM_CHART := ghcr.io/kptdev/krm-functions-catalog/render-helm-chart:v0.2.2

e2e-tests: render-helm-chart-example \
	   render-helm-chart-example2 \
	   render-with-kube-version \
	   test-apply-setters \
	   test-digester \
	   test-gatekeeper-set-enforcement-action \
	   test-helm-upgrader \
	   test-kubeconform \
	   test-package-compositor-e2e \
	   test-remove-local-config-resources \
	   test-render-helm-chart \
	   test-set-annotations \
	   test-set-labels \
	   test-source-helm-chart

# BEWARE: These tests are fragile - we want to test upgrades, but without upgrade-constraints tests might fail when charts are updated
.PHONY: test-helm-upgrader
test-helm-upgrader:
	rm -rf tmp-results
	kpt fn source examples/helm-upgrader | kpt fn eval - --fn-config example-function-configs/config-upgrade-helm-version-inline.yaml --truncate-output=false --results-dir tmp-results $(HELM_UPGRADER) > test-out.yaml
	grep -e '.*upgrade-available: .*cert-manager:v1.8.2' test-out.yaml
	grep -e '.*upgrade-available.0: .*cert-manager:v1.12.1' test-out.yaml
	grep -e '.*upgrade-available.0: .*metacontroller-helm:v4.10.0' test-out.yaml
	grep -e '.*upgrade-available.0: .*karpenter:0.35.0' test-out.yaml
	grep -e '.*upgrade-available.1: .*/external-secrets:2.2.0' test-out.yaml

	rm test-out.yaml
	rm -rf tmp-results

.PHONY: test-render-helm-chart
test-render-helm-chart:
	# For reference, render chart using baseline function
	rm -rf rendered-output
	echo "" | kpt fn eval - --network --fn-config examples/render-helm-chart2/argo-workflows.yaml -i ${BASELINE_RENDER_HELM_CHART} | kpt fn sink rendered-output
	# Source step, fetch chart from upstream, but do not render
	rm -rf sourced-output
	kpt fn source examples/render-helm-chart2 | kpt fn eval - ${HELM_RENDER} --results-dir results2 | kpt fn sink sourced-output
	grep -e 'message: sourcing with render-helm-chart is deprecated' results2/results.yaml
	# Render from previously sourced chart
	rm -rf rendered-output2
	kpt fn source sourced-output | kpt fn eval - ${HELM_RENDER} | kpt fn sink rendered-output2
	diff -q rendered-output rendered-output2
	rm -rf rendered-output sourced-output rendered-output2 results2

# Example usage from docs/render-helm-chart.md
.PHONY: render-helm-chart-example
render-helm-chart-example:
	kpt fn source examples/render-helm-chart | kpt fn eval - ${HELM_SOURCE} | kpt fn sink my-cert-manager-package
	grep -q 'apiVersion: experimental.helm.sh/v1alpha1' my-cert-manager-package/cert-manager-chart.yaml
	cp examples/render-helm-chart/Kptfile my-cert-manager-package/
	kpt fn render my-cert-manager-package -o stdout | grep -q 'team-name: blue-team'
	rm -rf my-cert-manager-package

.PHONY: render-helm-chart-example2
render-helm-chart-example2:
	kpt fn source examples/render-helm-chart/non-stardard-tar-filename | kpt fn eval - ${HELM_RENDER} | kpt fn sink my-non-stardard-tar-filename
	rm -rf my-non-stardard-tar-filename

# Test rendering using Capabilities.KubeVersion
.PHONY: render-with-kube-version
render-with-kube-version:
	mkdir _tmp
	helm package test-data/test-chart
	scripts/source-chart.sh test-data/render-test-chart.yaml test-chart-0.1.0.tgz > _tmp/render.yaml
	rm test-chart-0.1.0.tgz
	# Note, no patch version
	sed -i 's/kubeVersion: .*/kubeVersion: 1.29.0/' _tmp/render.yaml
	kpt fn source _tmp | kpt fn eval - ${HELM_RENDER} | kpt fn sink _tmp-output
	cat _tmp-output/configmap_foo.yaml
	grep -q 'kubeVersion: "v1.29.0"' _tmp-output/configmap_foo.yaml
	grep -q 'kubeVersionMinor: "29"' _tmp-output/configmap_foo.yaml
	rm -rf _tmp-output
	sed -i 's/kubeVersion: .*/kubeVersion: 1.30.0/' _tmp/render.yaml
	kpt fn source _tmp | kpt fn eval - ${HELM_RENDER} | kpt fn sink _tmp-output
	cat _tmp-output/configmap_foo.yaml
	grep -q 'kubeVersion: "v1.30.0"' _tmp-output/configmap_foo.yaml
	grep -q 'kubeVersionMinor: "30"' _tmp-output/configmap_foo.yaml
	rm -rf _tmp _tmp-output

.PHONY: test-source-helm-chart
test-source-helm-chart:
	kpt fn source examples/render-helm-chart3 | kpt fn eval - --truncate-output=false $(HELM_SOURCE) > test-out.txt
	grep -e 'experimental.helm.sh/chart-sum/karpenter' test-out.txt
	grep -e 'experimental.helm.sh/chart-sum/cert-manager' test-out.txt
	rm test-out.txt

.PHONY: test-apply-setters
test-apply-setters: test-apply-setters1 test-apply-setters2

.PHONY: test-apply-setters1
test-apply-setters1:
	kpt fn source examples/apply-setters | kpt fn eval - --truncate-output=false $(APPLY_SETTERS) --fn-config example-function-configs/apply-setters/cm-setters.yaml | kpt fn eval - -i $(REMOVE_LOCAL_CONFIG_RESOURCES_IMAGE) -o unwrap > test-out.yaml
	grep -e 'foo: "valueFoo"' test-out.yaml
	grep -e 'bar: "valueBar"' test-out.yaml
	grep -e 'baz: "valueBaz"' test-out.yaml
	grep -e 'replicas: ""' test-out.yaml
	grep -e 'combined: "valueBar-valueBaz"' test-out.yaml
	grep -e 'app.kubernetes.io/version: ""' test-out.yaml
	rm test-out.yaml

.PHONY: test-apply-setters2
test-apply-setters2:
	kpt fn source examples/apply-setters | kpt fn eval - --truncate-output=false $(APPLY_SETTERS) --fn-config example-function-configs/apply-setters/apply-setters-fn-config.yaml | kpt fn eval - -i $(REMOVE_LOCAL_CONFIG_RESOURCES_IMAGE) -o unwrap > test-out.yaml
	grep -e 'foo: ""' test-out.yaml
	grep -e 'bar: "valueBar"' test-out.yaml
	grep -e 'baz: "valueBaz"' test-out.yaml
	grep -e 'replicas: "4"' test-out.yaml
	grep -e 'combined: "valueBar-valueBaz"' test-out.yaml
	grep -e 'app.kubernetes.io/version: "a1b2c3d4e5e6"' test-out.yaml
	rm test-out.yaml

.PHONY: test-digester
test-digester: test-digester-step1 test-digester-step3 test-digester-output

.PHONY: test-digester-step1
test-digester-step1:
	rm -rf digester-tmp0
	kpt fn source examples/digester | kpt fn eval - --truncate-output=false $(HELM_SOURCE) | kpt fn sink digester-tmp0

.PHONY: test-digester-step2
test-digester-step2:
	kpt fn source digester-tmp0 | kpt fn eval - --truncate-output=false $(DIGESTER) > test-out.txt
	head -n 60 test-out.txt

.PHONY: test-digester-step3
test-digester-step3:
	rm -rf digester-tmp1
	kpt fn source digester-tmp0 | kpt fn eval - --truncate-output=false $(DIGESTER) | kpt fn sink digester-tmp1
	kpt fn render digester-tmp1 -o unwrap > test-out.yaml

.PHONY: test-digester-output
test-digester-output:
	grep 'image: "quay.io/jetstack/cert-manager-controller@sha256:.*' test-out.yaml
	grep 'image: "quay.io/jetstack/cert-manager-cainjector@sha256:.*' test-out.yaml
	grep 'image: "quay.io/jetstack/cert-manager-webhook@sha256:.*' test-out.yaml
	grep 'image: "quay.io/jetstack/cert-manager-ctl@sha256:.*' test-out.yaml

.PHONY: test-digester-w-containers
test-digester-w-containers:
	echo "** sourcing step"
	rm -rf cert-manager-package
	kpt fn source examples/digester \
	  | kpt fn eval - $(HELM_SOURCE) \
	  | kpt fn eval - --truncate-output=false $(DIGESTER) \
	  | kpt fn sink cert-manager-package

.PHONY: test-digester-w-containers2
test-digester-w-containers2:
	echo "** rendering using pipeline in Ktfile"
	rm -rf cert-manager-rendered
	kpt fn render cert-manager-package -o stdout | kpt fn sink cert-manager-rendered

PHONY: test-gatekeeper-set-enforcement-action
test-gatekeeper-set-enforcement-action:
	kpt fn source examples/gatekeeper-set-enforcement-action | kpt fn eval - --truncate-output=false $(GATEKEEPER_SET_ENFORCEMENT_ACTION) -o unwrap -- enforcementAction=deny > test-out.yaml
	if [ "$$(grep 'enforcementAction: deny' test-out.yaml | wc -l)" != "2" ]; then echo "*** error setting enforcementAction"; exit 1; fi

#KUBECONFORM_SCHEMA_LOCATIONS ?= "default"
KUBECONFORM_SCHEMA_LOCATIONS ?= 'examples/kubeconform/schema-bundle/,examples/kubeconform/schema-bundle/CRDs-catalog/{{.Group}}/{{.ResourceKind}}_{{.ResourceAPIVersion}}.json'

.PHONY: test-kubeconform
test-kubeconform:
	rm -rf tmp-results
	#kpt fn source examples/kubeconform/manifests | kpt fn eval - --results-dir tmp-results --truncate-output=false $(KUBECONFORM) -- ignore_missing_schemas=true kubernetes_version=1.29.1 schema_locations=$(KUBECONFORM_SCHEMA_LOCATIONS) > test-out.txt || true
	kpt fn source examples/kubeconform/manifests | kpt fn eval - --results-dir tmp-results --truncate-output=false $(KUBECONFORM) -- schema_locations=$(KUBECONFORM_SCHEMA_LOCATIONS) debug=true > test-out.txt || true
	make test-kubeconform-results

.PHONY: test-kubeconform-w-container
test-kubeconform-w-container:
	rm -rf tmp-results
	kpt fn source examples/kubeconform/manifests | kpt fn eval - --results-dir tmp-results --truncate-output=false -i $(KUBECONFORM_IMAGE) -- ignore_missing_schemas=true schema_locations=$(KUBECONFORM_SCHEMA_LOCATIONS) > test-out.txt || true
	make test-kubeconform-results

.PHONY: test-kubeconform-results
test-kubeconform-results:
	grep -e '.* /data/nested: got object, want null or string' tmp-results/results.yaml
	grep -e '.* name: invalid-nested-dict' tmp-results/results.yaml
	grep -e '.* /data/a-number: got number, want null or string' tmp-results/results.yaml
	grep -e '.* name: invalid-non-string-value' tmp-results/results.yaml

LOCAL_PACKAGES_DIR ?= /tmp/package-compositor

.PHONY: test-package-compositor-e2e
test-package-compositor-e2e: test-package-compositor test-package-compositor-ref
	# We don't set all annotations and comments exactly as kpt
	if [ "$(shell diff -rw fn-output/fleet-1/example-fleet example-fleet-ref | wc -l)" != "40" ]; then echo "*** source-package diff not as expected"; exit 1; fi

.PHONY: test-remove-local-config-resources
test-remove-local-config-resources:
	rm -rf tmp-results fn-output
	kpt fn source examples/remove-local-config-resources | kpt fn eval - --results-dir tmp-results --truncate-output=false $(REMOVE_LOCAL_CONFIG_RESOURCES) | kpt fn sink fn-output
	kpt pkg tree fn-output
	tree fn-output
	if grep -q 'name: cm2' fn-output/*; then echo "*** found resource that should have been removed"; exit 1; fi

.PHONY: test-package-compositor
test-package-compositor:
	rm -rf tmp-results fn-output
	kpt fn source examples/package-compositor/specs | kpt fn eval - --results-dir tmp-results --truncate-output=false $(PACKAGE_COMPOSITOR) | kpt fn sink fn-output
	kpt pkg tree fn-output
	tree fn-output

.PHONY: test-package-compositor-ref
test-package-compositor-ref:
	# This is identical to 'examples/package-compositor/specs/fleet-1/packages.yaml'
	rm -rf example-fleet-ref && mkdir -p example-fleet-ref/olo
	kpt pkg get https://github.com/krm-functions/catalog.git/examples/package-compositor/pkg1 example-fleet-ref/foo --for-deployment
	kpt pkg get https://github.com/krm-functions/catalog.git/examples/package-compositor/pkg2 example-fleet-ref/bar --for-deployment
	kpt pkg get https://github.com/krm-functions/catalog.git/examples/package-compositor/pkg3 example-fleet-ref/bar/baz1 --for-deployment
	kpt pkg get https://github.com/krm-functions/catalog.git/examples/package-compositor/pkg1 example-fleet-ref/bar/baz2 --for-deployment
	kpt pkg get https://github.com/krm-functions/catalog.git/examples/package-compositor/pkg4 example-fleet-ref/olo/baz --for-deployment

.PHONY: test-set-annotations
test-set-annotations:
	kpt fn source examples/set-labels | kpt fn eval - --truncate-output=false $(SET_ANNOTATIONS) -o unwrap -- l1=v1 l2=v2 | \
		yq -e '.metadata.annotations.l1|contains("v1")'
	kpt fn source examples/set-labels | kpt fn eval - --truncate-output=false $(SET_ANNOTATIONS) --fn-config example-function-configs/set-annotations/configmap.yaml -o unwrap | \
		yq -e '.metadata.annotations.foo|contains("bar")'
	kpt fn source examples/set-labels | kpt fn eval - --truncate-output=false $(SET_ANNOTATIONS) --fn-config example-function-configs/set-annotations/setannotations.yaml -o unwrap | \
		yq -e '.metadata.annotations.baz|contains("olo")'

.PHONY: test-set-labels
test-set-labels:
	kpt fn source examples/set-labels | kpt fn eval - --truncate-output=false $(SET_LABELS) -o unwrap -- l1=v1 l2=v2 | \
		yq -e '.metadata.labels.l1|contains("v1")'
	kpt fn source examples/set-labels | kpt fn eval - --truncate-output=false $(SET_LABELS) --fn-config example-function-configs/set-labels/configmap.yaml -o unwrap | \
		yq -e '.metadata.labels.foo|contains("bar")'
	kpt fn source examples/set-labels | kpt fn eval - --truncate-output=false $(SET_LABELS) --fn-config example-function-configs/set-labels/setlabels.yaml -o unwrap | \
		yq -e '.metadata.labels.baz|contains("olo")'

.PHONY: test-template-kyaml
test-template-kyaml:
	kpt fn source examples/template | kpt fn eval - --truncate-output=false $(TEMPLATE_KYAML) -- someConfig=aTestConfigString > test-out.txt
	kpt fn source examples/template | kpt fn eval - --truncate-output=false $(TEMPLATE_KYAML) > test-out.txt