From 9bdf4effd102b006509c980d279e453beda27588 Mon Sep 17 00:00:00 2001 From: Vincent Dely Date: Tue, 5 May 2026 02:04:10 +0200 Subject: [PATCH] feat(api): allow dependencies on any Kubernetes resource Kustomizations can now depend on any Kubernetes object, improving flexibility in release ordering and readiness behavior. Signed-off-by: Vincent Dely --- api/v1/kustomization_types.go | 2 +- api/v1/reference_types.go | 23 +- api/v1/zz_generated.deepcopy.go | 9 +- ...mize.toolkit.fluxcd.io_kustomizations.yaml | 27 ++- docs/api/v1/kustomize.md | 51 +++- .../controller/kustomization_controller.go | 90 ++++++-- .../kustomization_dependson_test.go | 217 +++++++++++++++++- 7 files changed, 380 insertions(+), 39 deletions(-) diff --git a/api/v1/kustomization_types.go b/api/v1/kustomization_types.go index 6bd217c3a..8e4f50d0f 100644 --- a/api/v1/kustomization_types.go +++ b/api/v1/kustomization_types.go @@ -50,7 +50,7 @@ type KustomizationSpec struct { CommonMetadata *CommonMetadata `json:"commonMetadata,omitempty"` // DependsOn may contain a DependencyReference slice - // with references to Kustomization resources that must be ready before this + // with references to Kubernetes resources that must be ready before this // Kustomization can be reconciled. // +optional DependsOn []DependencyReference `json:"dependsOn,omitempty"` diff --git a/api/v1/reference_types.go b/api/v1/reference_types.go index 07eb3b701..0ebdd9723 100644 --- a/api/v1/reference_types.go +++ b/api/v1/reference_types.go @@ -51,17 +51,32 @@ func (s *CrossNamespaceSourceReference) String() string { return fmt.Sprintf("%s/%s", s.Kind, s.Name) } -// DependencyReference defines a Kustomization dependency on another Kustomization resource. +// DependencyReference defines a Kustomization dependency on a Kubernetes resource. +// When the dependency is a Kustomization, defaults are applied during reconciliation. type DependencyReference struct { - // Name of the referent. + // APIVersion of the resource to depend on, defaults to the Kustomization API + // group version when the dependency is a Kustomization. + // +optional + APIVersion string `json:"apiVersion,omitempty"` + + // Kind of the resource to depend on, defaults to Kustomization. + // +optional + Kind string `json:"kind,omitempty"` + + // Name of the resource to depend on. // +required Name string `json:"name"` - // Namespace of the referent, defaults to the namespace of the Kustomization - // resource object that contains the reference. + // Namespace of the resource to depend on, defaults to the namespace of the + // Kustomization resource object that contains the reference. // +optional Namespace string `json:"namespace,omitempty"` + // Ready checks if the resource Ready status condition is true, defaults to + // true when the dependency is a Kustomization. + // +optional + Ready *bool `json:"ready,omitempty"` + // ReadyExpr is a CEL expression that can be used to assess the readiness // of a dependency. When specified, the built-in readiness check // is replaced by the logic defined in the CEL expression. diff --git a/api/v1/zz_generated.deepcopy.go b/api/v1/zz_generated.deepcopy.go index 231b5b525..f63570e18 100644 --- a/api/v1/zz_generated.deepcopy.go +++ b/api/v1/zz_generated.deepcopy.go @@ -94,6 +94,11 @@ func (in *Decryption) DeepCopy() *Decryption { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *DependencyReference) DeepCopyInto(out *DependencyReference) { *out = *in + if in.Ready != nil { + in, out := &in.Ready, &out.Ready + *out = new(bool) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DependencyReference. @@ -176,7 +181,9 @@ func (in *KustomizationSpec) DeepCopyInto(out *KustomizationSpec) { if in.DependsOn != nil { in, out := &in.DependsOn, &out.DependsOn *out = make([]DependencyReference, len(*in)) - copy(*out, *in) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } } if in.Decryption != nil { in, out := &in.Decryption, &out.Decryption diff --git a/config/crd/bases/kustomize.toolkit.fluxcd.io_kustomizations.yaml b/config/crd/bases/kustomize.toolkit.fluxcd.io_kustomizations.yaml index beeff81f0..20c6feafa 100644 --- a/config/crd/bases/kustomize.toolkit.fluxcd.io_kustomizations.yaml +++ b/config/crd/bases/kustomize.toolkit.fluxcd.io_kustomizations.yaml @@ -137,20 +137,35 @@ spec: dependsOn: description: |- DependsOn may contain a DependencyReference slice - with references to Kustomization resources that must be ready before this + with references to Kubernetes resources that must be ready before this Kustomization can be reconciled. items: - description: DependencyReference defines a Kustomization dependency - on another Kustomization resource. + description: |- + DependencyReference defines a Kustomization dependency on a Kubernetes resource. + When the dependency is a Kustomization, defaults are applied during reconciliation. properties: + apiVersion: + description: |- + APIVersion of the resource to depend on, defaults to the Kustomization API + group version when the dependency is a Kustomization. + type: string + kind: + description: Kind of the resource to depend on, defaults to + Kustomization. + type: string name: - description: Name of the referent. + description: Name of the resource to depend on. type: string namespace: description: |- - Namespace of the referent, defaults to the namespace of the Kustomization - resource object that contains the reference. + Namespace of the resource to depend on, defaults to the namespace of the + Kustomization resource object that contains the reference. type: string + ready: + description: |- + Ready checks if the resource Ready status condition is true, defaults to + true when the dependency is a Kustomization. + type: boolean readyExpr: description: |- ReadyExpr is a CEL expression that can be used to assess the readiness diff --git a/docs/api/v1/kustomize.md b/docs/api/v1/kustomize.md index 11624c6c1..260d485f2 100644 --- a/docs/api/v1/kustomize.md +++ b/docs/api/v1/kustomize.md @@ -97,7 +97,7 @@ overridden if its key matches a common one.

(Optional)

DependsOn may contain a DependencyReference slice -with references to Kustomization resources that must be ready before this +with references to Kubernetes resources that must be ready before this Kustomization can be reconciled.

@@ -653,7 +653,8 @@ field.

(Appears on: KustomizationSpec)

-

DependencyReference defines a Kustomization dependency on another Kustomization resource.

+

DependencyReference defines a Kustomization dependency on a Kubernetes resource. +When the dependency is a Kustomization, defaults are applied during reconciliation.

@@ -666,13 +667,38 @@ field.

+ + + + + + + + @@ -684,8 +710,21 @@ string + + + + @@ -754,7 +793,7 @@ overridden if its key matches a common one.

diff --git a/internal/controller/kustomization_controller.go b/internal/controller/kustomization_controller.go index 1bf2204cf..8db0188a9 100644 --- a/internal/controller/kustomization_controller.go +++ b/internal/controller/kustomization_controller.go @@ -45,6 +45,7 @@ import ( "github.com/fluxcd/cli-utils/pkg/kstatus/polling" "github.com/fluxcd/cli-utils/pkg/kstatus/polling/engine" + "github.com/fluxcd/cli-utils/pkg/kstatus/status" "github.com/fluxcd/cli-utils/pkg/object" apiacl "github.com/fluxcd/pkg/apis/acl" eventv1 "github.com/fluxcd/pkg/apis/event/v1beta1" @@ -560,29 +561,62 @@ func (r *KustomizationReconciler) checkDependencies(ctx context.Context, } for _, depRef := range obj.Spec.DependsOn { - // Check if the dependency exists by querying - // the API server bypassing the cache. - if depRef.Namespace == "" { - depRef.Namespace = obj.GetNamespace() + // Default the dependency Kind to Kustomization if unset. + if depRef.Kind == "" { + depRef.Kind = kustomizev1.KustomizationKind } - depName := types.NamespacedName{ - Namespace: depRef.Namespace, + + // Apply Kustomization defaults if the dependency is a Kustomization. + if depRef.Kind == kustomizev1.KustomizationKind { + // Default APIVersion to Kustomization if unset. + if depRef.APIVersion == "" { + depRef.APIVersion = kustomizev1.GroupVersion.String() + } + // Default namespace to the dependent's namespace if unset. + if depRef.Namespace == "" { + depRef.Namespace = obj.GetNamespace() + } + // Default readiness check to true if unset. + if depRef.Ready == nil { + depRef.Ready = new(true) + } + } + + depMd := object.ObjMetadata{ + GroupKind: schema.GroupKind{Kind: depRef.Kind}, Name: depRef.Name, + Namespace: depRef.Namespace, } - var dep kustomizev1.Kustomization - err := r.APIReader.Get(ctx, depName, &dep) - if err != nil { - return fmt.Errorf("dependency '%s' not found: %w", depName, err) + depObj := &unstructured.Unstructured{ + Object: map[string]any{ + "apiVersion": depRef.APIVersion, + "kind": depRef.Kind, + "metadata": map[string]any{ + "name": depRef.Name, + "namespace": depRef.Namespace, + }, + }, + } + + // Check if the dependency exists by querying + // the API server bypassing the cache. + if err := r.APIReader.Get(ctx, client.ObjectKeyFromObject(depObj), depObj); err != nil { + return fmt.Errorf("dependency '%s/%s' not found: %w", depRef.APIVersion, ssautil.FmtObjMetadata(depMd), err) + } + + // Skip all readiness checks if unset or set to false. + if depRef.Ready == nil || !*depRef.Ready { + continue } // Evaluate the CEL expression (if specified) to determine if the dependency is ready. if depRef.ReadyExpr != "" { - ready, err := r.evalReadyExpr(ctx, depRef.ReadyExpr, objMap, &dep) + ready, err := r.evalReadyExpr(ctx, depRef.ReadyExpr, objMap, depObj) if err != nil { return err } if !ready { - return fmt.Errorf("dependency '%s' is not ready according to readyExpr eval", depName) + return fmt.Errorf("dependency '%s/%s' is not ready according to readyExpr eval", depRef.APIVersion, ssautil.FmtObjMetadata(depMd)) } } @@ -594,16 +628,32 @@ func (r *KustomizationReconciler) checkDependencies(ctx context.Context, // Check if the dependency observed generation is up to date // and if the dependency is in a ready state. - if len(dep.Status.Conditions) == 0 || dep.Generation != dep.Status.ObservedGeneration { - return fmt.Errorf("dependency '%s' is not ready", depName) + stat, err := status.Compute(depObj) + if err != nil { + return fmt.Errorf("dependency '%s/%s' is not ready: %w", depRef.APIVersion, ssautil.FmtObjMetadata(depMd), err) } - if !apimeta.IsStatusConditionTrue(dep.Status.Conditions, meta.ReadyCondition) { - return fmt.Errorf("dependency '%s' is not ready", depName) + if stat.Status != status.CurrentStatus { + return fmt.Errorf("dependency '%s/%s' is not ready: status %s", depRef.APIVersion, ssautil.FmtObjMetadata(depMd), stat.Status) } - // Check if the dependency source matches the current source + // These checks only apply to Kustomization dependencies. + // Check if the Kustomization dependency source matches the current source // and if so, verify that the last applied revision of the dependency // matches the current source artifact revision. + // Additionally check Kustomization dependencies for readiness. + // kstatus.Compute() tolerates missing conditions, but Kustomizations are required to have a Ready condition. + if depRef.Kind != kustomizev1.KustomizationKind { + continue + } + + var dep kustomizev1.Kustomization + if err := runtime.DefaultUnstructuredConverter.FromUnstructured(depObj.Object, &dep); err != nil { + return fmt.Errorf("failed to convert unstructured to Kustomization: %w", err) + } + if !apimeta.IsStatusConditionTrue(dep.Status.Conditions, meta.ReadyCondition) { + return fmt.Errorf("dependency '%s/%s' is not ready", depRef.APIVersion, ssautil.FmtObjMetadata(depMd)) + } + srcNamespace := dep.Spec.SourceRef.Namespace if srcNamespace == "" { srcNamespace = dep.GetNamespace() @@ -616,7 +666,7 @@ func (r *KustomizationReconciler) checkDependencies(ctx context.Context, srcNamespace == depSrcNamespace && dep.Spec.SourceRef.Kind == obj.Spec.SourceRef.Kind && !source.GetArtifact().HasRevision(dep.Status.LastAppliedRevision) { - return fmt.Errorf("dependency '%s' revision is not up to date", depName) + return fmt.Errorf("dependency '%s/%s' revision is not up to date", depRef.APIVersion, ssautil.FmtObjMetadata(depMd)) } } @@ -628,7 +678,7 @@ func (r *KustomizationReconciler) evalReadyExpr( ctx context.Context, expr string, selfMap map[string]any, - dep *kustomizev1.Kustomization, + dep *unstructured.Unstructured, ) (bool, error) { const ( selfName = "self" @@ -640,7 +690,7 @@ func (r *KustomizationReconciler) evalReadyExpr( cel.WithOutputType(celtypes.BoolType), cel.WithStructVariables(selfName, depName)) if err != nil { - return false, reconcile.TerminalError(fmt.Errorf("failed to evaluate dependency %s: %w", dep.Name, err)) + return false, reconcile.TerminalError(fmt.Errorf("failed to evaluate dependency %s: %w", dep.GetName(), err)) } depMap, err := runtime.DefaultUnstructuredConverter.ToUnstructured(dep) diff --git a/internal/controller/kustomization_dependson_test.go b/internal/controller/kustomization_dependson_test.go index 14eac8032..dedb1b889 100644 --- a/internal/controller/kustomization_dependson_test.go +++ b/internal/controller/kustomization_dependson_test.go @@ -28,6 +28,8 @@ import ( "github.com/fluxcd/pkg/testserver" sourcev1 "github.com/fluxcd/source-controller/api/v1" . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" @@ -164,6 +166,26 @@ spec: }, timeout, time.Second).Should(BeTrue()) }) + t.Run("reconciles when dependency is found", func(t *testing.T) { + g := NewWithT(t) + g.Eventually(func() error { + _ = k8sClient.Get(context.Background(), client.ObjectKeyFromObject(kustomization), resultK) + resultK.Spec.DependsOn = []kustomizev1.DependencyReference{ + { + APIVersion: apiextensionsv1.SchemeGroupVersion.String(), + Kind: "CustomResourceDefinition", + Name: "kustomizations.kustomize.toolkit.fluxcd.io", + }, + } + return k8sClient.Update(context.Background(), resultK) + }, timeout, time.Second).Should(BeNil()) + + g.Eventually(func() bool { + _ = k8sClient.Get(context.Background(), client.ObjectKeyFromObject(kustomization), resultK) + return conditions.IsReady(resultK) + }, timeout, time.Second).Should(BeTrue()) + }) + t.Run("fails due to dependency not found", func(t *testing.T) { g := NewWithT(t) g.Eventually(func() error { @@ -182,6 +204,160 @@ spec: return conditions.HasAnyReason(resultK, meta.ReadyCondition, meta.DependencyNotReadyReason) }, timeout, time.Second).Should(BeTrue()) }) + + t.Run("fails due to other dependency not found", func(t *testing.T) { + g := NewWithT(t) + g.Eventually(func() error { + _ = k8sClient.Get(context.Background(), client.ObjectKeyFromObject(kustomization), resultK) + resultK.Spec.DependsOn = []kustomizev1.DependencyReference{ + { + APIVersion: apiextensionsv1.SchemeGroupVersion.String(), + Kind: "CustomResourceDefinition", + Name: "helmreleases.helm.toolkit.fluxcd.io", + }, + } + return k8sClient.Update(context.Background(), resultK) + }, timeout, time.Second).Should(BeNil()) + + g.Eventually(func() bool { + _ = k8sClient.Get(context.Background(), client.ObjectKeyFromObject(kustomization), resultK) + ready := conditions.Get(resultK, meta.ReadyCondition) + return ready.Reason == meta.DependencyNotReadyReason && + strings.Contains(ready.Message, "CustomResourceDefinition") && + strings.Contains(ready.Message, "not found") + }, timeout, time.Second).Should(BeTrue()) + }) + + t.Run("fails due to multiple dependencies not found", func(t *testing.T) { + g := NewWithT(t) + g.Eventually(func() error { + _ = k8sClient.Get(context.Background(), client.ObjectKeyFromObject(kustomization), resultK) + resultK.Spec.DependsOn = []kustomizev1.DependencyReference{ + { + APIVersion: corev1.SchemeGroupVersion.String(), + Kind: "ConfigMap", + Name: "root", + Namespace: id, + }, + { + APIVersion: corev1.SchemeGroupVersion.String(), + Kind: "Pod", + Name: "root", + Namespace: id, + }, + } + return k8sClient.Update(context.Background(), resultK) + }, timeout, time.Second).Should(BeNil()) + + g.Eventually(func() bool { + _ = k8sClient.Get(context.Background(), client.ObjectKeyFromObject(kustomization), resultK) + ready := conditions.Get(resultK, meta.ReadyCondition) + return ready.Reason == meta.DependencyNotReadyReason && + strings.Contains(ready.Message, "ConfigMap") && + strings.Contains(ready.Message, "not found") + }, timeout, time.Second).Should(BeTrue()) + }) + + podKey := types.NamespacedName{ + Name: fmt.Sprintf("dep-%s", randStringRunes(5)), + Namespace: id, + } + pod := &corev1.Pod{ + TypeMeta: metav1.TypeMeta{ + APIVersion: corev1.SchemeGroupVersion.String(), + Kind: "Pod", + }, + ObjectMeta: metav1.ObjectMeta{ + Generation: 3, + Name: podKey.Name, + Namespace: podKey.Namespace, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "test", + Image: "alpine:latest", + }, + }, + }, + Status: corev1.PodStatus{ + ObservedGeneration: 2, + Conditions: []corev1.PodCondition{ + {Type: corev1.PodReady, Status: corev1.ConditionTrue}, + }, + Phase: corev1.PodRunning, + }, + } + + g.Expect(k8sClient.Create(context.Background(), pod)).To(Succeed()) + + t.Run("fails due to dependency with ObservedGeneration < Generation", func(t *testing.T) { + g := NewWithT(t) + g.Eventually(func() error { + _ = k8sClient.Get(context.Background(), client.ObjectKeyFromObject(kustomization), resultK) + resultK.Spec.DependsOn = []kustomizev1.DependencyReference{ + { + APIVersion: corev1.SchemeGroupVersion.String(), + Kind: "Pod", + Name: podKey.Name, + Namespace: podKey.Namespace, + Ready: new(true), + }, + } + return k8sClient.Update(context.Background(), resultK) + }, timeout, time.Second).Should(BeNil()) + + g.Eventually(func() bool { + _ = k8sClient.Get(context.Background(), client.ObjectKeyFromObject(kustomization), resultK) + ready := conditions.Get(resultK, meta.ReadyCondition) + return ready.Reason == meta.DependencyNotReadyReason && + strings.Contains(ready.Message, "Pod") && + strings.Contains(ready.Message, podKey.Name) && + strings.Contains(ready.Message, "is not ready") + }, timeout, time.Second).Should(BeTrue()) + }) + + podKey.Name = fmt.Sprintf("dep-%s", randStringRunes(5)) + pod.ObjectMeta = metav1.ObjectMeta{ + Generation: 1, + Name: podKey.Name, + Namespace: podKey.Namespace, + } + pod.Status = corev1.PodStatus{ + ObservedGeneration: 1, + Conditions: []corev1.PodCondition{ + {Type: corev1.PodReady, Status: corev1.ConditionFalse}, + }, + Phase: corev1.PodRunning, + } + + g.Expect(k8sClient.Create(context.Background(), pod)).To(Succeed()) + + t.Run("fails due to dependency with ObservedGeneration = Generation and ReadyCondition = False", func(t *testing.T) { + g := NewWithT(t) + g.Eventually(func() error { + _ = k8sClient.Get(context.Background(), client.ObjectKeyFromObject(kustomization), resultK) + resultK.Spec.DependsOn = []kustomizev1.DependencyReference{ + { + APIVersion: corev1.SchemeGroupVersion.String(), + Kind: "Pod", + Name: podKey.Name, + Namespace: podKey.Namespace, + Ready: new(true), + }, + } + return k8sClient.Update(context.Background(), resultK) + }, timeout, time.Second).Should(BeNil()) + + g.Eventually(func() bool { + _ = k8sClient.Get(context.Background(), client.ObjectKeyFromObject(kustomization), resultK) + ready := conditions.Get(resultK, meta.ReadyCondition) + return ready.Reason == meta.DependencyNotReadyReason && + strings.Contains(ready.Message, "Pod") && + strings.Contains(ready.Message, podKey.Name) && + strings.Contains(ready.Message, "is not ready") + }, timeout, time.Second).Should(BeTrue()) + }) } func TestKustomizationReconciler_DependsOn_CEL(t *testing.T) { @@ -261,7 +437,7 @@ data: t.Run("succeeds with readyExpr dependency check", func(t *testing.T) { g := NewWithT(t) - // Create a dependency Kustomization with matching annotations + // Create a Kustomization dependency with matching annotations dependency := &kustomizev1.Kustomization{ ObjectMeta: metav1.ObjectMeta{ Name: depID, @@ -301,6 +477,14 @@ data: Name: dependency.Name, ReadyExpr: `self.metadata.annotations['app/version'] == dep.metadata.annotations['app/version']`, }, + { + APIVersion: corev1.SchemeGroupVersion.String(), + Kind: "ConfigMap", + Name: id, + Namespace: id, + Ready: new(true), + ReadyExpr: "has(dep.data)", + }, } return k8sClient.Update(context.Background(), resultK) }, timeout, time.Second).Should(BeNil()) @@ -342,6 +526,37 @@ data: g.Expect(conditions.IsStalled(resultK)).Should(BeFalse()) }) + t.Run("fails with readyExpr when other condition not met", func(t *testing.T) { + g := NewWithT(t) + + // Update the main kustomization with a ConfigMap dependency + g.Eventually(func() error { + _ = k8sClient.Get(context.Background(), client.ObjectKeyFromObject(kustomization), resultK) + resultK.Spec.DependsOn = []kustomizev1.DependencyReference{ + { + APIVersion: corev1.SchemeGroupVersion.String(), + Kind: "ConfigMap", + Name: id, + Namespace: id, + Ready: new(true), + ReadyExpr: "!has(dep.data)", + }, + } + return k8sClient.Update(context.Background(), resultK) + }, timeout, time.Second).Should(BeNil()) + + // Should fail because CEL expression evaluates to false + g.Eventually(func() bool { + _ = k8sClient.Get(context.Background(), client.ObjectKeyFromObject(kustomization), resultK) + ready := conditions.Get(resultK, meta.ReadyCondition) + return ready.Reason == meta.DependencyNotReadyReason && + strings.Contains(ready.Message, "ConfigMap") && + strings.Contains(ready.Message, "not ready according to readyExpr") + }, timeout, time.Second).Should(BeTrue()) + + g.Expect(conditions.IsStalled(resultK)).Should(BeFalse()) + }) + t.Run("fails terminally with invalid readyExpr", func(t *testing.T) { g := NewWithT(t)
+apiVersion
+ +string + +		 +(Optional) +

APIVersion of the resource to depend on, defaults to the Kustomization API +group version when the dependency is a Kustomization.

+
+kind
+ +string + +		 +(Optional) +

Kind of the resource to depend on, defaults to Kustomization.

+
name
string 		-

Name of the referent.

+

Name of the resource to depend on.
(Optional) -

Namespace of the referent, defaults to the namespace of the Kustomization -resource object that contains the reference.

+

Namespace of the resource to depend on, defaults to the namespace of the +Kustomization resource object that contains the reference.

+
+ready
+ +bool + +		 +(Optional) +

Ready checks if the resource Ready status condition is true, defaults to +true when the dependency is a Kustomization.
(Optional)

DependsOn may contain a DependencyReference slice -with references to Kustomization resources that must be ready before this +with references to Kubernetes resources that must be ready before this Kustomization can be reconciled.