From 6e279ac714825489aa4b871dcb7c41fc136a839a Mon Sep 17 00:00:00 2001 From: Gianluca Mardente Date: Mon, 3 Aug 2026 18:34:23 +0200 Subject: [PATCH] feat: add JobCheck ValidateHealth support Adds `JobCheck` to `ValidateHealth`: a Job, referenced from a ConfigMap/Secret via `JobRef`, usable alongside the existing Lua Script and CEL evaluation checks. Bumps libsveltos to pick up ValidateHealth.JobCheck: a Job, referenced via JobRef from a ConfigMap/Secret, that can be used as a health check alongside the existing Lua Script and CEL evaluation options. Removes this repo's dependency on the private `sveltos-enterprise` module entirely. `go.mod` no longer references it, under any build tag. --- Dockerfile | 5 +- Dockerfile.enterprise | 52 -- Dockerfile_WithGit | 5 +- Dockerfile_WithGit.enterprise | 52 -- Makefile | 10 - cmd/main.go | 804 +---------------- ...fig.projectsveltos.io_clusterprofiles.yaml | 180 +++- ...g.projectsveltos.io_clusterpromotions.yaml | 274 +++++- ...ig.projectsveltos.io_clustersummaries.yaml | 182 +++- .../config.projectsveltos.io_profiles.yaml | 180 +++- controllers/clusterpromotion_controller.go | 31 +- ...ion_oss.go => clusterpromotion_default.go} | 10 +- controllers/clusterpromotion_plugin.go | 49 -- controllers/delete_checks.go | 4 +- controllers/handlers_helm.go | 4 +- controllers/handlers_kustomize.go | 4 +- controllers/handlers_resources.go | 4 +- controllers/handlers_utils.go | 2 +- go.mod | 4 +- go.sum | 8 +- lib/clusterops/jobhealthcheck_default.go | 36 + lib/clusterops/validate_health.go | 45 +- lib/crd/clusterprofiles.go | 180 +++- lib/crd/clusterpromotions.go | 274 +++++- lib/crd/clustersummaries.go | 182 +++- lib/crd/profiles.go | 180 +++- manifest/manifest.yaml | 816 ++++++++++++++++- pkg/app/app.go | 826 ++++++++++++++++++ test/fv/job_health_check_test.go | 202 +++++ test/fv/promotion_test.go | 2 +- 30 files changed, 3535 insertions(+), 1072 deletions(-) delete mode 100644 Dockerfile.enterprise delete mode 100644 Dockerfile_WithGit.enterprise rename controllers/{clusterpromotion_oss.go => clusterpromotion_default.go} (72%) delete mode 100644 controllers/clusterpromotion_plugin.go create mode 100644 lib/clusterops/jobhealthcheck_default.go create mode 100644 pkg/app/app.go create mode 100644 test/fv/job_health_check_test.go diff --git a/Dockerfile b/Dockerfile index 1698994c..2b10151c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,10 +8,7 @@ WORKDIR /workspace # Copy the Go Modules manifests COPY go.mod go.mod COPY go.sum go.sum -# Deps are downloaded as part of the build step below, not in a separate "go mod download" -# layer: go.mod lists the private sveltos-enterprise module (needed only for `-tags -# enterprise` builds), and unlike `go build`, bare `go mod download` is not build-tag-aware -# and would eagerly try to resolve it even for this default (non-enterprise) build. +RUN go mod download # Copy the go source COPY cmd/main.go cmd/main.go diff --git a/Dockerfile.enterprise b/Dockerfile.enterprise deleted file mode 100644 index d93b811e..00000000 --- a/Dockerfile.enterprise +++ /dev/null @@ -1,52 +0,0 @@ -# syntax=docker/dockerfile:1.26 - -# Build the manager binary (Enterprise build - links in the private sveltos-enterprise module) -FROM golang:1.26.5@sha256:3aff6657219a4d9c14e27fb1d8976c49c29fddb70ba835014f477e1c70636647 AS builder - -ARG BUILDOS -ARG TARGETARCH - -WORKDIR /workspace -# Copy the Go Modules manifests -COPY go.mod go.mod -COPY go.sum go.sum - -# github.com/projectsveltos/sveltos-enterprise is a private module. Unlike the default -# Dockerfile, this build always needs it (it's the enterprise variant), so it's fetched here -# via git over SSH, authenticated with a key forwarded from the host through BuildKit's ssh -# mount (see Makefile's docker-buildx target: --ssh default=). -RUN git config --global url."git@github.com:".insteadOf "https://github.com/" && \ - mkdir -p -m 0700 /root/.ssh && ssh-keyscan github.com >> /root/.ssh/known_hosts -RUN --mount=type=ssh GOPRIVATE=github.com/projectsveltos/sveltos-enterprise go mod download - -# Copy the go source -COPY cmd/main.go cmd/main.go -COPY api/ api/ -COPY lib/ lib/ -COPY controllers/ controllers/ -COPY pkg/ pkg/ -COPY internal/ internal/ - -# Build -RUN CGO_ENABLED=0 GOOS=$BUILDOS GOARCH=$TARGETARCH go build -tags enterprise -a -o manager cmd/main.go - -# Use distroless as minimal base image to package the manager binary -# Refer to https://github.com/GoogleContainerTools/distroless for more details -FROM gcr.io/distroless/static:nonroot - -ARG GIT_VERSION=unknown - -LABEL org.opencontainers.image.source="https://github.com/projectsveltos/addon-controller" \ - org.opencontainers.image.url="https://projectsveltos.io" \ - org.opencontainers.image.licenses="Apache-2.0" \ - org.opencontainers.image.vendor="projectsveltos" \ - org.opencontainers.image.title="addon-controller" \ - org.opencontainers.image.description="Deploys Kubernetes add-ons and applications (Helm charts, Kustomize, raw YAML) across fleets of clusters, with built-in multi-tenancy support." \ - org.opencontainers.image.version="$GIT_VERSION" \ - org.opencontainers.image.revision="$GIT_VERSION" - -WORKDIR / -COPY --from=builder /workspace/manager . -USER 65532:65532 - -ENTRYPOINT ["/manager"] diff --git a/Dockerfile_WithGit b/Dockerfile_WithGit index 8184f265..19bd2aa1 100644 --- a/Dockerfile_WithGit +++ b/Dockerfile_WithGit @@ -8,10 +8,7 @@ WORKDIR /workspace # Copy the Go Modules manifests COPY go.mod go.mod COPY go.sum go.sum -# Deps are downloaded as part of the build step below, not in a separate "go mod download" -# layer: go.mod lists the private sveltos-enterprise module (needed only for `-tags -# enterprise` builds), and unlike `go build`, bare `go mod download` is not build-tag-aware -# and would eagerly try to resolve it even for this default (non-enterprise) build. +RUN go mod download # Copy the go source COPY cmd/main.go cmd/main.go diff --git a/Dockerfile_WithGit.enterprise b/Dockerfile_WithGit.enterprise deleted file mode 100644 index 8ce0da29..00000000 --- a/Dockerfile_WithGit.enterprise +++ /dev/null @@ -1,52 +0,0 @@ -# syntax=docker/dockerfile:1.26 - -# Build the manager binary (Enterprise build - links in the private sveltos-enterprise module) -FROM golang:1.26.5@sha256:3aff6657219a4d9c14e27fb1d8976c49c29fddb70ba835014f477e1c70636647 AS builder - -ARG BUILDOS -ARG TARGETARCH - -WORKDIR /workspace -# Copy the Go Modules manifests -COPY go.mod go.mod -COPY go.sum go.sum - -# github.com/projectsveltos/sveltos-enterprise is a private module. Unlike the default -# Dockerfile, this build always needs it (it's the enterprise variant), so it's fetched here -# via git over SSH, authenticated with a key forwarded from the host through BuildKit's ssh -# mount (see Makefile's docker-buildx target: --ssh default=). -RUN git config --global url."git@github.com:".insteadOf "https://github.com/" && \ - mkdir -p -m 0700 /root/.ssh && ssh-keyscan github.com >> /root/.ssh/known_hosts -RUN --mount=type=ssh GOPRIVATE=github.com/projectsveltos/sveltos-enterprise go mod download - -# Copy the go source -COPY cmd/main.go cmd/main.go -COPY api/ api/ -COPY lib/ lib/ -COPY controllers/ controllers/ -COPY pkg/ pkg/ -COPY internal/ internal/ - -# Build -RUN CGO_ENABLED=0 GOOS=$BUILDOS GOARCH=$TARGETARCH go build -tags enterprise -a -o manager cmd/main.go - -# This is needed to support kustomization that points to a oci/git repo that utilizes remote kustomization references. -FROM alpine:3.24.1 - -ARG GIT_VERSION=unknown - -LABEL org.opencontainers.image.source="https://github.com/projectsveltos/addon-controller" \ - org.opencontainers.image.url="https://projectsveltos.io" \ - org.opencontainers.image.licenses="Apache-2.0" \ - org.opencontainers.image.vendor="projectsveltos" \ - org.opencontainers.image.title="addon-controller" \ - org.opencontainers.image.description="Deploys Kubernetes add-ons and applications (Helm charts, Kustomize, raw YAML) across fleets of clusters, with built-in multi-tenancy support. This variant bundles git for Kustomize remote-reference support." \ - org.opencontainers.image.version="$GIT_VERSION" \ - org.opencontainers.image.revision="$GIT_VERSION" - -RUN apk add --no-cache git -WORKDIR / -COPY --from=builder /workspace/manager . -USER 65532:65532 - -ENTRYPOINT ["/manager"] diff --git a/Makefile b/Makefile index 31cb19ea..f0027859 100644 --- a/Makefile +++ b/Makefile @@ -36,11 +36,6 @@ K8S_LATEST_VER ?= $(shell curl -s https://dl.k8s.io/release/stable.txt) export CONTROLLER_IMG ?= $(REGISTRY)/$(IMAGE_NAME) TAG ?= main -# SSH key with read access to the private github.com/projectsveltos/sveltos-enterprise repo, -# forwarded into the enterprise docker-buildx build (see Dockerfile.enterprise) so it can -# fetch that module. Override with e.g. `make docker-buildx SVELTOS_ENTERPRISE_SSH_KEY=~/.ssh/other_key`. -SVELTOS_ENTERPRISE_SSH_KEY ?= $(HOME)/.ssh/id_ed25519 - .PHONY: all all: build @@ -555,11 +550,6 @@ docker-build: ## Build docker image with the manager. docker-push: ## Push docker image with the manager. docker push $(CONTROLLER_IMG):$(TAG) -.PHONY: docker-buildx -docker-buildx: ## docker build for multiple arch and push to docker hub (enterprise build - requires SSH access to sveltos-enterprise) - docker buildx build --push --platform linux/amd64,linux/arm64 --ssh default=$(SVELTOS_ENTERPRISE_SSH_KEY) --build-arg GIT_VERSION=$(TAG) -t $(CONTROLLER_IMG):$(TAG) -f Dockerfile.enterprise . - docker buildx build --push --platform linux/amd64,linux/arm64 --ssh default=$(SVELTOS_ENTERPRISE_SSH_KEY) --build-arg GIT_VERSION=$(TAG) -t $(CONTROLLER_IMG)-git:$(TAG) -f Dockerfile_WithGit.enterprise . - .PHONY: load-image load-image: docker-build $(KIND) $(KIND) load docker-image $(CONTROLLER_IMG):$(TAG) --name $(CONTROL_CLUSTER_NAME) diff --git a/cmd/main.go b/cmd/main.go index db9ecef7..f04f5742 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -17,809 +17,9 @@ limitations under the License. package main import ( - "context" - "flag" - "fmt" - "net/http" - "net/http/pprof" - "os" - "runtime" - "runtime/debug" - "sync" - "syscall" - "time" - - _ "embed" - - sourcev1 "github.com/fluxcd/source-controller/api/v1" - "github.com/go-logr/logr" - "github.com/spf13/pflag" - lua "github.com/yuin/gopher-lua" - corev1 "k8s.io/api/core/v1" - apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/api/meta" - "k8s.io/apimachinery/pkg/fields" - apimachineryruntime "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/apimachinery/pkg/types" - "k8s.io/client-go/discovery" - _ "k8s.io/client-go/plugin/pkg/client/auth" - "k8s.io/client-go/rest" - cliflag "k8s.io/component-base/cli/flag" - "k8s.io/klog/v2" - clusterv1 "sigs.k8s.io/cluster-api/api/core/v1beta2" - "sigs.k8s.io/cluster-api/controllers/remote" - "sigs.k8s.io/cluster-api/util/apiwarnings" - ctrl "sigs.k8s.io/controller-runtime" - "sigs.k8s.io/controller-runtime/pkg/cache" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/controller" - "sigs.k8s.io/controller-runtime/pkg/healthz" - "sigs.k8s.io/controller-runtime/pkg/manager" - "sigs.k8s.io/controller-runtime/pkg/metrics/filters" - metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" - "sigs.k8s.io/controller-runtime/pkg/webhook" - - configv1beta1 "github.com/projectsveltos/addon-controller/api/v1beta1" - "github.com/projectsveltos/addon-controller/api/v1beta1/index" - "github.com/projectsveltos/addon-controller/controllers" - "github.com/projectsveltos/addon-controller/controllers/dependencymanager" - "github.com/projectsveltos/addon-controller/internal/telemetry" - libsveltosv1beta1 "github.com/projectsveltos/libsveltos/api/v1beta1" - "github.com/projectsveltos/libsveltos/lib/crd" - "github.com/projectsveltos/libsveltos/lib/deployer" - logs "github.com/projectsveltos/libsveltos/lib/logsettings" - libsveltosset "github.com/projectsveltos/libsveltos/lib/set" - //+kubebuilder:scaffold:imports -) - -var ( - setupLog = ctrl.Log.WithName("setup") - diagnosticsAddress string - insecureDiagnostics bool - shardKey string - workers int - concurrentReconciles int - agentInMgmtCluster bool - reportMode controllers.ReportMode - tmpReportMode int - restConfigQPS float32 - restConfigBurst int - webhookPort int - syncPeriod time.Duration - conflictRetryTime time.Duration - healthErrorRetryTime time.Duration - version string - healthAddr string - profilerAddress string - driftDetectionConfigMap string - luaConfigMap string - capiOnboardAnnotation string - disableCaching bool - disableTelemetry bool - autoDeployDependencies bool - registry string - luaCallStackSize int - luaRegistrySize int - helmChartUpdateCheckInterval time.Duration + "github.com/projectsveltos/addon-controller/pkg/app" ) -const ( - defaultReconcilers = 10 - defaultWorkers = 20 - defaulReportMode = int(controllers.CollectFromManagementCluster) - mebibytes_bytes = 1 << 20 - gibibytes_per_bytes = 1 << 30 -) - -// Add RBAC for the authorized diagnostics endpoint. -// +kubebuilder:rbac:groups=authentication.k8s.io,resources=tokenreviews,verbs=create -// +kubebuilder:rbac:groups=authorization.k8s.io,resources=subjectaccessreviews,verbs=create - func main() { - scheme, err := controllers.InitScheme() - if err != nil { - os.Exit(1) - } - - setupLogging() - - sveltosNamespace := os.Getenv("NAMESPACE") - if sveltosNamespace == "" { - setupLog.V(logs.LogInfo).Error(nil, "Missing required environment variables NAMESPACE") - os.Exit(1) - } - - reportMode = controllers.ReportMode(tmpReportMode) - ctrl.SetLogger(klog.Background()) - ctrlOptions := getCtrlOptions(scheme) - - restConfig := getRestConfig() - - ctx := ctrl.SetupSignalHandler() - - controllers.SetDriftdetectionConfigMap(driftDetectionConfigMap) - controllers.SetLuaConfigMap(luaConfigMap) - controllers.SetLuaCallStackSize(luaCallStackSize) - controllers.SetLuaRegistrySize(luaRegistrySize) - controllers.SetCAPIOnboardAnnotation(capiOnboardAnnotation) - controllers.SetDriftDetectionRegistry(registry) - controllers.SetAgentInMgmtCluster(agentInMgmtCluster) - controllers.SetSveltosNamespace(sveltosNamespace) - - if isInitContainer() { - runInitContainerWork(ctx, restConfig, scheme) - os.Exit(0) - } - - mgr, err := ctrl.NewManager(restConfig, ctrlOptions) - if err != nil { - setupLog.Error(err, "unable to start manager") - os.Exit(1) - } - - dc, err := discovery.NewDiscoveryClientForConfig(mgr.GetConfig()) - if err != nil { - setupLog.Error(err, "unable to get discovery client") - os.Exit(1) - } - // Setup the context that's going to be used in controllers and for the manager. - controllers.SetManagementClusterAccess(mgr.GetClient(), mgr.GetConfig(), dc) - - // Start dependency manager - dependencymanager.InitializeManagerInstance(ctx, mgr.GetClient(), autoDeployDependencies, ctrl.Log.WithName("dependency_manager")) - - logs.RegisterForLogSettings(ctx, - libsveltosv1beta1.ComponentAddonManager, ctrl.Log.WithName("log-setter"), - ctrl.GetConfigOrDie()) - - debug.SetMemoryLimit(gibibytes_per_bytes) - go printMemUsage(ctx, ctrl.Log.WithName("memory-usage")) - controllers.NewLicenseManager() - - if shardKey == "" && !disableTelemetry { - err = telemetry.StartCollecting(ctx, mgr.GetConfig(), mgr.GetClient(), sveltosNamespace, version) - if err != nil { - setupLog.Error(err, "failed starting telemetry client") - } - } - - startControllersAndWatchers(ctx, mgr) - - setupChecks(mgr) - - setupIndexes(ctx, mgr) - - setupLog.Info("starting manager") - if err := mgr.Start(ctx); err != nil { - setupLog.Error(err, "problem running manager") - os.Exit(1) - } -} - -func getCacheConfig() (disableFor []client.Object, byObject map[client.Object]cache.ByObject) { - disableFor = []client.Object{} - byObject = map[client.Object]cache.ByObject{} - if disableCaching { - // Note: Only Secrets with type addons.projectsveltos.io/cluster-profile are cached - // The default client of the manager won't use the cache for secrets at all. - disableFor = []client.Object{ - &corev1.Secret{}, - &corev1.ConfigMap{}, - } - - fieldSelector := fields.OneTermEqualSelector("type", string(libsveltosv1beta1.ClusterProfileSecretType)) - - byObject[&corev1.Secret{}] = cache.ByObject{ - Field: fieldSelector, - } - } - return -} - -func initFlags(fs *pflag.FlagSet) { - fs.IntVar(&tmpReportMode, "report-mode", defaulReportMode, - "Indicates how ReportSummaries need to be collected") - - fs.BoolVar(&agentInMgmtCluster, "agent-in-mgmt-cluster", false, - "When set, indicates drift-detection-manager needs to be started in the management cluster") - - fs.BoolVar(&disableCaching, "disable-secret-caching", false, - "When set, disable caching secrets and configmaps") - - fs.BoolVar(&disableTelemetry, "disable-telemetry", false, - "When set, disable telemetry reporting") - - fs.StringVar(&diagnosticsAddress, "diagnostics-address", ":8443", - "The address the diagnostics endpoint binds to. Per default metrics are served via https and with"+ - "authentication/authorization. To serve via http and without authentication/authorization set --insecure-diagnostics."+ - "If --insecure-diagnostics is not set the diagnostics endpoint also serves pprof endpoints") - - fs.BoolVar(&insecureDiagnostics, "insecure-diagnostics", false, - "Enable insecure diagnostics serving. For more details see the description of --diagnostics-address.") - - fs.StringVar(&shardKey, "shard-key", "", - "If set, only clusters will annotation matching this shard key will be reconciled by this deployment.") - - fs.StringVar(&capiOnboardAnnotation, "capi-onboard-annotation", "", - "If provided, Sveltos will only manage CAPI clusters that have this exact annotation.") - - fs.IntVar(&workers, "worker-number", defaultWorkers, - "Number of worker. Workers are used to deploy features in CAPI clusters") - - fs.IntVar(&concurrentReconciles, "concurrent-reconciles", defaultReconcilers, - "concurrent reconciles is the maximum number of concurrent Reconciles which can be run. Defaults to 10") - - fs.StringVar(&version, "version", "", "current sveltos version") - - fs.StringVar(&healthAddr, "health-addr", ":9440", - "The address the health endpoint binds to.") - - fs.StringVar(&profilerAddress, "profiler-address", "", - "Bind address to expose the pprof profiler (e.g. localhost:6060)") - - fs.StringVar(&driftDetectionConfigMap, "drift-detection-config", "", - "The name of the ConfigMap in the namespace where projectsveltos is deployed containing the drift-detection-manager configuration") - - fs.StringVar(&luaConfigMap, "lua-methods", "", - "The name of the ConfigMap in the namespace where projectsveltos is deployed containing lua utilities to be loaded."+ - "Changing the content of the ConfigMap does not cause Sveltos to redeploy.") - - fs.StringVar(®istry, "registry", "", - "Container registry for drift-detection images. Defaults to docker.io/ if empty.") - - const defautlRestConfigQPS = 20 - fs.Float32Var(&restConfigQPS, "kube-api-qps", defautlRestConfigQPS, - fmt.Sprintf("Maximum queries per second from the controller client to the Kubernetes API server. Defaults to %d", - defautlRestConfigQPS)) - - const defaultRestConfigBurst = 60 - fs.IntVar(&restConfigBurst, "kube-api-burst", defaultRestConfigBurst, - fmt.Sprintf("Maximum number of queries that should be allowed in one burst from the controller client to the Kubernetes API server. Default %d", - defaultRestConfigBurst)) - - const defaultWebhookPort = 9443 - fs.IntVar(&webhookPort, "webhook-port", defaultWebhookPort, - "Webhook Server port") - - const defaultSyncPeriod = 10 - fs.DurationVar(&syncPeriod, "sync-period", defaultSyncPeriod*time.Minute, - fmt.Sprintf("The minimum interval at which watched resources are reconciled (e.g. 15m). Default: %d minutes", - defaultSyncPeriod)) - - const defaultConflictRetryTime = 60 - fs.DurationVar(&conflictRetryTime, "conflict-retry-time", defaultConflictRetryTime*time.Second, - fmt.Sprintf("The minimum interval at which watched ClusterProfile with conflicts are retried. Defaul: %d seconds", - defaultConflictRetryTime)) - - const defaultHealthErrorRetryTime = 60 - fs.DurationVar(&healthErrorRetryTime, "health-error-retry-time", defaultHealthErrorRetryTime*time.Second, - fmt.Sprintf("The minimum interval at which health check failures are retried. Default: %d seconds", - defaultHealthErrorRetryTime)) - - fs.DurationVar(&helmChartUpdateCheckInterval, "helm-chart-update-check-interval", time.Hour, - "Interval at which Sveltos checks whether newer versions of deployed Helm charts have been "+ - "published upstream (HTTP repositories and OCI registries). Set to 0 to disable.") - - // AutoDeployDependencies enables automatic deployment of prerequisite profiles. - // - // Profile instances can specify dependencies on other profiles using the - // DependsOn field, forming a directed acyclic graph (DAG) of dependencies. - // - // When AutoDeployDependencies is set to true, Sveltos automatically resolves and deploys - // the prerequisite profiles listed in the DependsOn field. This automation - // ensures that all required dependencies are deployed to the same managed clusters - // as the dependent profile. Sveltos analyzes the dependency graph to identify - // and deploy prerequisites in the correct order. - // - // By default, AutoDeployDependencies is enabled (true). When disabled, administrators - // are responsible for ensuring that both the dependent and prerequisite profiles - // target the same set of managed clusters through matching cluster selectors. - // - // Enabling AutoDeployDependencies simplifies multi-cluster management by automating - // dependency resolution, reducing manual effort, and minimizing the risk of - // configuration inconsistencies. - fs.BoolVar(&autoDeployDependencies, "auto-deploy-dependencies", true, - " When AutoDeployDependencies is set to true, Sveltos will automatically resolve and deploy the prerequisite profiles specified in the DependsOn field") - - fs.IntVar(&luaCallStackSize, "lua-call-stack-size", lua.CallStackSize, "Call stack size. This defaults to lua.CallStackSize") - - fs.IntVar(&luaRegistrySize, "lua-registry-size", lua.RegistrySize, "Call stack size. This defaults to lua.RegistrySize") -} - -func setupIndexes(ctx context.Context, mgr ctrl.Manager) { - if err := index.AddDefaultIndexes(ctx, mgr); err != nil { - setupLog.Error(err, "unable to setup indexes") - os.Exit(1) - } -} - -func setupChecks(mgr ctrl.Manager) { - if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { - setupLog.Error(err, "unable to set up health check") - os.Exit(1) - } - if err := mgr.AddReadyzCheck("readyz", healthz.Ping); err != nil { - setupLog.Error(err, "unable to set up ready check") - os.Exit(1) - } -} - -// capiCRDHandler restarts process if a CAPI CRD is updated -func capiCRDHandler(gvk *schema.GroupVersionKind, action crd.ChangeType) { - if action == crd.Modify { - return - } - if gvk.Group == clusterv1.GroupVersion.Group && gvk.Version == clusterv1.GroupVersion.Version { - setupLog.V(logs.LogInfo).Info("Initiating graceful restart due to CAPI CRD update", - "GVK", gvk.String(), "Action", string(action)) - - if killErr := syscall.Kill(syscall.Getpid(), syscall.SIGTERM); killErr != nil { - panic("kill -TERM failed") - } - } -} - -// isCAPIInstalled returns true if CAPI is installed with v1beta2 served, false otherwise -func isCAPIInstalled(ctx context.Context, c client.Client, logger logr.Logger) (bool, error) { - clusterCRD := &apiextensionsv1.CustomResourceDefinition{} - - err := c.Get(ctx, types.NamespacedName{Name: "clusters.cluster.x-k8s.io"}, clusterCRD) - if err != nil { - if apierrors.IsNotFound(err) { - return false, nil - } - return false, err - } - - for _, version := range clusterCRD.Spec.Versions { - if version.Name == clusterv1.GroupVersion.Version && version.Served { - return true, nil - } - } - - logger.V(logs.LogInfo).Info("clusterCRD CRD present but v1beta2 not served") - return false, nil -} - -// fluxCRDHandler restarts process if a Flux CRD is updated -func fluxCRDHandler(gvk *schema.GroupVersionKind, action crd.ChangeType) { - if action == crd.Modify { - return - } - - if gvk.Group == sourcev1.GroupVersion.Group && gvk.Kind == sourcev1.GitRepositoryKind { - setupLog.V(logs.LogInfo).Info("Initiating graceful restart due to Flux CRD update", - "GVK", gvk.String(), "Action", string(action)) - - if killErr := syscall.Kill(syscall.Getpid(), syscall.SIGTERM); killErr != nil { - panic("kill -TERM failed") - } - } -} - -// isFluxInstalled returns true if Flux is installed, false otherwise -func isFluxInstalled(ctx context.Context, c client.Client) (bool, error) { - gitRepositoryCRD := &apiextensionsv1.CustomResourceDefinition{} - - err := c.Get(ctx, types.NamespacedName{Name: "gitrepositories.source.toolkit.fluxcd.io"}, - gitRepositoryCRD) - if err != nil { - if apierrors.IsNotFound(err) { - return false, nil - } - return false, err - } - - return true, nil -} - -func capiWatchers(ctx context.Context, mgr ctrl.Manager, watchersForCAPI []watcherForCAPI, logger logr.Logger) { - const maxRetries = 20 - retries := 0 - for { - capiPresent, err := isCAPIInstalled(ctx, mgr.GetClient(), logger) - if err != nil { - if retries < maxRetries { - logger.Info(fmt.Sprintf("failed to verify if CAPI is present: %v", err)) - time.Sleep(time.Second) - } - retries++ - } else { - if !capiPresent { - setupLog.V(logs.LogInfo).Info("CAPI currently not present. Starting CRD watcher") - go crd.WatchCustomResourceDefinition(ctx, mgr.GetConfig(), capiCRDHandler, setupLog) - } else { - setupLog.V(logs.LogInfo).Info("CAPI present. Start CAPI watchers") - for i := range watchersForCAPI { - watcher := watchersForCAPI[i] - err = watcher.WatchForCAPI(mgr, watcher.GetController()) - if err != nil { - setupLog.V(logs.LogInfo).Info( - fmt.Sprintf("failed to start CAPI watcher: %v", err)) - continue - } - } - } - return - } - } -} - -func fluxWatchers(ctx context.Context, mgr ctrl.Manager, watchersForFlux []watcherForFlux, logger logr.Logger) { - const maxRetries = 20 - retries := 0 - for { - fluxPresent, err := isFluxInstalled(ctx, mgr.GetClient()) - if err != nil { - if retries < maxRetries { - logger.Info(fmt.Sprintf("failed to verify if Flux is present: %v", err)) - time.Sleep(time.Second) - } - retries++ - } else { - if !fluxPresent { - setupLog.V(logs.LogInfo).Info("Flux currently not present. Starting CRD watcher") - go crd.WatchCustomResourceDefinition(ctx, mgr.GetConfig(), fluxCRDHandler, setupLog) - } else { - setupLog.V(logs.LogInfo).Info("Flux present. Start Flux watchers") - for i := range watchersForFlux { - watcher := watchersForFlux[i] - err = watcher.WatchForFlux(mgr, watcher.GetController()) - if err != nil { - continue - } - } - } - return - } - } -} - -type watcherForCAPI interface { - WatchForCAPI(mgr manager.Manager, c controller.Controller) error - GetController() controller.Controller -} - -type watcherForFlux interface { - WatchForFlux(mgr manager.Manager, c controller.Controller) error - GetController() controller.Controller -} - -func startWatchers(ctx context.Context, mgr manager.Manager, - watchersForCAPI []watcherForCAPI, watchersForFlux []watcherForFlux) { - - go capiWatchers(ctx, mgr, watchersForCAPI, setupLog) - - go fluxWatchers(ctx, mgr, watchersForFlux, setupLog) -} - -func getProfileReconciler(mgr manager.Manager) *controllers.ProfileReconciler { - return &controllers.ProfileReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), - SetMap: make(map[corev1.ObjectReference]*libsveltosset.Set), - ClusterMap: make(map[corev1.ObjectReference]*libsveltosset.Set), - Profiles: make(map[corev1.ObjectReference]libsveltosv1beta1.Selector), - ClusterLabels: make(map[corev1.ObjectReference]map[string]string), - Mux: sync.Mutex{}, - ConcurrentReconciles: concurrentReconciles, - Logger: ctrl.Log.WithName("profilereconciler"), - } -} - -func getClusterProfileReconciler(mgr manager.Manager) *controllers.ClusterProfileReconciler { - return &controllers.ClusterProfileReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), - ClusterSetMap: make(map[corev1.ObjectReference]*libsveltosset.Set), - ClusterMap: make(map[corev1.ObjectReference]*libsveltosset.Set), - ClusterProfiles: make(map[corev1.ObjectReference]libsveltosv1beta1.Selector), - ClusterLabels: make(map[corev1.ObjectReference]map[string]string), - Mux: sync.Mutex{}, - ConcurrentReconciles: concurrentReconciles, - Logger: ctrl.Log.WithName("clusterprofilereconciler"), - } -} - -func getClusterSummaryReconciler(ctx context.Context, mgr manager.Manager) *controllers.ClusterSummaryReconciler { - d := deployer.GetClient(ctx, ctrl.Log.WithName("deployer"), mgr.GetClient(), workers) - controllers.RegisterFeatures(d, setupLog) - - return &controllers.ClusterSummaryReconciler{ - Config: mgr.GetConfig(), - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), - ShardKey: shardKey, - Version: version, - ReportMode: reportMode, - Deployer: d, - ClusterMap: make(map[corev1.ObjectReference]*libsveltosset.Set), - ReferenceMap: make(map[corev1.ObjectReference]*libsveltosset.Set), - PolicyMux: sync.Mutex{}, - ConcurrentReconciles: concurrentReconciles, - ConflictRetryTime: conflictRetryTime, - HealthErrorRetryTime: healthErrorRetryTime, - Logger: ctrl.Log.WithName("clustersummaryreconciler"), - } -} - -func getSetReconciler(mgr manager.Manager) *controllers.SetReconciler { - return &controllers.SetReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), - ConcurrentReconciles: concurrentReconciles, - Mux: sync.Mutex{}, - ClusterMap: make(map[corev1.ObjectReference]*libsveltosset.Set), - SetMap: make(map[corev1.ObjectReference]*libsveltosset.Set), - Sets: make(map[corev1.ObjectReference]libsveltosv1beta1.Selector), - ClusterLabels: make(map[corev1.ObjectReference]map[string]string), - Logger: ctrl.Log.WithName("setreconciler"), - } -} - -func getClusterSetReconciler(mgr manager.Manager) *controllers.ClusterSetReconciler { - return &controllers.ClusterSetReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), - ConcurrentReconciles: concurrentReconciles, - Mux: sync.Mutex{}, - ClusterMap: make(map[corev1.ObjectReference]*libsveltosset.Set), - ClusterSetMap: make(map[corev1.ObjectReference]*libsveltosset.Set), - ClusterSets: make(map[corev1.ObjectReference]libsveltosv1beta1.Selector), - ClusterLabels: make(map[corev1.ObjectReference]map[string]string), - Logger: ctrl.Log.WithName("clustersetreconciler"), - } -} - -// getDiagnosticsOptions returns metrics options which can be used to configure a Manager. -func getDiagnosticsOptions() metricsserver.Options { - // If "--insecure-diagnostics" is set, serve metrics via http - // and without authentication/authorization. - if insecureDiagnostics { - return metricsserver.Options{ - BindAddress: diagnosticsAddress, - SecureServing: false, - } - } - - // If "--insecure-diagnostics" is not set, serve metrics via https - // and with authentication/authorization. As the endpoint is protected, - // we also serve pprof endpoints and an endpoint to change the log level. - return metricsserver.Options{ - BindAddress: diagnosticsAddress, - SecureServing: true, - FilterProvider: filters.WithAuthenticationAndAuthorization, - ExtraHandlers: map[string]http.Handler{ - // Add pprof handler. - "/debug/pprof/": http.HandlerFunc(pprof.Index), - "/debug/pprof/cmdline": http.HandlerFunc(pprof.Cmdline), - "/debug/pprof/profile": http.HandlerFunc(pprof.Profile), - "/debug/pprof/symbol": http.HandlerFunc(pprof.Symbol), - "/debug/pprof/trace": http.HandlerFunc(pprof.Trace), - "/debug/pprof/heap": pprof.Handler("heap"), - }, - } -} - -// startControllers starts all reconcilers: -// - ClusterProfile/Profile -// - clusterSummary -// - ClusterSet/Set -// -// It also starts needed watchers: -// - cluster API watchers for ClusterProfile/Profile, ClusterSet/Set -// - Flux watcher for ClusterSummary -func startControllersAndWatchers(ctx context.Context, mgr manager.Manager) { - var clusterProfileReconciler *controllers.ClusterProfileReconciler - var profileReconciler *controllers.ProfileReconciler - var clusterSetReconciler *controllers.ClusterSetReconciler - var setReconciler *controllers.SetReconciler - - var err error - - //+kubebuilder:scaffold:builder - if err = (&controllers.SveltosClusterReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), - }).SetupWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create controller", "controller", "SveltosCluster") - os.Exit(1) - } - - watchersForCAPI := make([]watcherForCAPI, 0) - watchersForFlux := make([]watcherForFlux, 0) - - if shardKey == "" { - // Only if shardKey is not set, start ClusterProfile/Profile and ClusterSet/Set reconcilers. - // When shardKey is set, only ClusterSummary reconciler will be started and only - // cluster matching the shardkey will be managed - clusterProfileReconciler = getClusterProfileReconciler(mgr) - err = clusterProfileReconciler.SetupWithManager(mgr) - if err != nil { - setupLog.Error(err, "unable to create controller", "controller", configv1beta1.ClusterProfileKind) - os.Exit(1) - } - watchersForCAPI = append(watchersForCAPI, clusterProfileReconciler) - - profileReconciler = getProfileReconciler(mgr) - err = profileReconciler.SetupWithManager(mgr) - if err != nil { - setupLog.Error(err, "unable to create controller", "controller", configv1beta1.ProfileKind) - os.Exit(1) - } - watchersForCAPI = append(watchersForCAPI, profileReconciler) - - clusterSetReconciler = getClusterSetReconciler(mgr) - err = clusterSetReconciler.SetupWithManager(mgr) - if err != nil { - setupLog.Error(err, "unable to create controller", "controller", libsveltosv1beta1.ClusterSetKind) - os.Exit(1) - } - watchersForCAPI = append(watchersForCAPI, clusterSetReconciler) - - setReconciler = getSetReconciler(mgr) - err = setReconciler.SetupWithManager(mgr) - if err != nil { - setupLog.Error(err, "unable to create controller", "controller", libsveltosv1beta1.SetKind) - os.Exit(1) - } - watchersForCAPI = append(watchersForCAPI, setReconciler) - - if err := (&controllers.ClusterPromotionReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), - Config: mgr.GetConfig(), - ConcurrentReconciles: concurrentReconciles, - }).SetupWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create controller", "controller", "ClusterPromotion") - os.Exit(1) - } - - // Needs a fleet-wide view of every ClusterSummary to dedup chart keys correctly, so - // this only ever runs on the default (unsharded) deployment, same as the reconcilers - // started above. Running it per-shard would give no benefit (chart-key dedup is - // inherently fleet-wide) and would reintroduce cross-shard inconsistency. - if helmChartUpdateCheckInterval > 0 { - go controllers.RunHelmChartUpdateChecker(ctx, mgr.GetClient(), helmChartUpdateCheckInterval, - ctrl.Log.WithName("helm-chart-update-checker")) - } - } - - clusterSummaryReconciler := getClusterSummaryReconciler(ctx, mgr) - err = clusterSummaryReconciler.SetupWithManager(ctx, mgr) - if err != nil { - setupLog.Error(err, "unable to create controller", "controller", configv1beta1.ClusterSummaryKind) - os.Exit(1) - } - watchersForCAPI = append(watchersForCAPI, clusterSummaryReconciler) - watchersForFlux = append(watchersForFlux, clusterSummaryReconciler) - - startWatchers(ctx, mgr, watchersForCAPI, watchersForFlux) -} - -// printMemUsage memory stats. Call GC -func printMemUsage(ctx context.Context, logger logr.Logger) { - const five = 5 - ticker := time.NewTicker(five * time.Minute) - defer ticker.Stop() - - for { - select { - case <-ctx.Done(): - logger.Info("stopping memory usage printer") - return - case <-ticker.C: - time.Sleep(time.Minute) - var m runtime.MemStats - runtime.ReadMemStats(&m) - // For info on each, see: /pkg/runtime/#MemStats - l := logger.WithValues("Alloc (MiB)", bToMb(m.Alloc)). - WithValues("TotalAlloc (MiB)", bToMb(m.TotalAlloc)). - WithValues("Sys (MiB)", bToMb(m.Sys)). - WithValues("NumGC", m.NumGC) - l.V(1).Info("memory stats") - } - } -} - -func bToMb(b uint64) uint64 { - return b / mebibytes_bytes -} - -// reduceMemoryFootprint removes large, rarely-used fields from all cached objects -func reduceMemoryFootprint(obj interface{}) (interface{}, error) { - accessor, err := meta.Accessor(obj) - if err != nil { - return obj, nil - } - - // Remove managedFields (typically the largest field) - accessor.SetManagedFields(nil) - - // Clean up annotations - annotations := accessor.GetAnnotations() - if len(annotations) > 0 { - // Remove large annotations - delete(annotations, "kubectl.kubernetes.io/last-applied-configuration") - delete(annotations, "deployment.kubernetes.io/revision") - - // If annotations is now empty, set to nil - if len(annotations) == 0 { - accessor.SetAnnotations(nil) - } else { - accessor.SetAnnotations(annotations) - } - } - - return obj, nil -} - -func getCtrlOptions(scheme *apimachineryruntime.Scheme) ctrl.Options { - disableFor, byObject := getCacheConfig() - - return ctrl.Options{ - Scheme: scheme, - Metrics: getDiagnosticsOptions(), - HealthProbeBindAddress: healthAddr, - WebhookServer: webhook.NewServer( - webhook.Options{ - Port: webhookPort, - }), - Cache: cache.Options{ - SyncPeriod: &syncPeriod, - ByObject: byObject, - DefaultTransform: reduceMemoryFootprint, - }, - Client: client.Options{ - Cache: &client.CacheOptions{ - DisableFor: disableFor, - }, - }, - PprofBindAddress: profilerAddress, - } -} - -func isInitContainer() bool { - return os.Getenv("IS_INITIALIZATION") == "true" -} - -func runInitContainerWork(ctx context.Context, config *rest.Config, - scheme *apimachineryruntime.Scheme) { - - directClient, err := client.New(config, client.Options{Scheme: scheme}) - if err != nil { - return - } - - dc, err := discovery.NewDiscoveryClientForConfig(config) - if err != nil { - setupLog.Error(err, "unable to get discovery client") - os.Exit(1) - } - - controllers.SetManagementClusterAccess(directClient, config, dc) - controllers.Initialization(ctx, config, scheme, shardKey, - ctrl.Log.WithName("initialization")) -} - -func setupLogging() { - klog.InitFlags(nil) - - initFlags(pflag.CommandLine) - pflag.CommandLine.SetNormalizeFunc(cliflag.WordSepNormalizeFunc) - pflag.CommandLine.AddGoFlagSet(flag.CommandLine) - pflag.Parse() - - ctrl.SetLogger(klog.Background()) -} - -func getRestConfig() *rest.Config { - restConfig := ctrl.GetConfigOrDie() - restConfig.QPS = restConfigQPS - restConfig.Burst = restConfigBurst - restConfig.UserAgent = remote.DefaultClusterAPIUserAgent("addon-controller") - restConfig.WarningHandler = apiwarnings.DefaultHandler(klog.Background().WithName("API Server Warning")) - return restConfig + app.Run() } diff --git a/config/crd/bases/config.projectsveltos.io_clusterprofiles.yaml b/config/crd/bases/config.projectsveltos.io_clusterprofiles.yaml index 9d549c14..56561760 100644 --- a/config/crd/bases/config.projectsveltos.io_clusterprofiles.yaml +++ b/config/crd/bases/config.projectsveltos.io_clusterprofiles.yaml @@ -1289,7 +1289,7 @@ spec: type: array featureID: description: |- - FeatureID is an indentifier of the feature (Helm/Kustomize/Resources) + FeatureID is an identifier of the feature (Helm/Kustomize/Resources) This field indicates when to run this check. For instance: - if set to Helm this check will be run after all helm @@ -1307,6 +1307,46 @@ spec: Group of the resource to fetch in the managed Cluster. Required when Kind is set. Leave empty for metric-only checks. type: string + jobCheck: + description: |- + JobCheck runs a Job in the managed cluster and uses its Complete/Failed + outcome as the check result. Mutually exclusive with Script and EvaluateCEL. + properties: + jobRef: + description: |- + JobRef references the Secret/ConfigMap containing the Job manifest to + deploy in the managed Cluster as this check. + properties: + kind: + description: 'Kind of the resource. Supported kinds + are: Secrets and ConfigMaps.' + enum: + - Secret + - ConfigMap + type: string + name: + description: Name of the referenced resource. + minLength: 1 + type: string + namespace: + description: |- + Namespace of the referenced resource. + Namespace can be left empty. In such a case, namespace will + be implicit set to cluster's namespace. + type: string + required: + - kind + - name + - namespace + type: object + timeout: + description: |- + Timeout is how long to wait for the Job to reach Complete or Failed + before treating the check as failed. + type: string + required: + - jobRef + type: object kind: description: |- Kind of the resource to fetch in the managed Cluster. @@ -1421,6 +1461,9 @@ spec: - featureID - name type: object + x-kubernetes-validations: + - message: jobCheck cannot be set together with script or evaluateCEL + rule: '!has(self.jobCheck) || (!has(self.script) && !has(self.evaluateCEL))' type: array x-kubernetes-list-type: atomic preDeleteChecks: @@ -1459,7 +1502,7 @@ spec: type: array featureID: description: |- - FeatureID is an indentifier of the feature (Helm/Kustomize/Resources) + FeatureID is an identifier of the feature (Helm/Kustomize/Resources) This field indicates when to run this check. For instance: - if set to Helm this check will be run after all helm @@ -1477,6 +1520,46 @@ spec: Group of the resource to fetch in the managed Cluster. Required when Kind is set. Leave empty for metric-only checks. type: string + jobCheck: + description: |- + JobCheck runs a Job in the managed cluster and uses its Complete/Failed + outcome as the check result. Mutually exclusive with Script and EvaluateCEL. + properties: + jobRef: + description: |- + JobRef references the Secret/ConfigMap containing the Job manifest to + deploy in the managed Cluster as this check. + properties: + kind: + description: 'Kind of the resource. Supported kinds + are: Secrets and ConfigMaps.' + enum: + - Secret + - ConfigMap + type: string + name: + description: Name of the referenced resource. + minLength: 1 + type: string + namespace: + description: |- + Namespace of the referenced resource. + Namespace can be left empty. In such a case, namespace will + be implicit set to cluster's namespace. + type: string + required: + - kind + - name + - namespace + type: object + timeout: + description: |- + Timeout is how long to wait for the Job to reach Complete or Failed + before treating the check as failed. + type: string + required: + - jobRef + type: object kind: description: |- Kind of the resource to fetch in the managed Cluster. @@ -1591,6 +1674,9 @@ spec: - featureID - name type: object + x-kubernetes-validations: + - message: jobCheck cannot be set together with script or evaluateCEL + rule: '!has(self.jobCheck) || (!has(self.script) && !has(self.evaluateCEL))' type: array x-kubernetes-list-type: atomic preDeployChecks: @@ -1629,7 +1715,7 @@ spec: type: array featureID: description: |- - FeatureID is an indentifier of the feature (Helm/Kustomize/Resources) + FeatureID is an identifier of the feature (Helm/Kustomize/Resources) This field indicates when to run this check. For instance: - if set to Helm this check will be run after all helm @@ -1647,6 +1733,46 @@ spec: Group of the resource to fetch in the managed Cluster. Required when Kind is set. Leave empty for metric-only checks. type: string + jobCheck: + description: |- + JobCheck runs a Job in the managed cluster and uses its Complete/Failed + outcome as the check result. Mutually exclusive with Script and EvaluateCEL. + properties: + jobRef: + description: |- + JobRef references the Secret/ConfigMap containing the Job manifest to + deploy in the managed Cluster as this check. + properties: + kind: + description: 'Kind of the resource. Supported kinds + are: Secrets and ConfigMaps.' + enum: + - Secret + - ConfigMap + type: string + name: + description: Name of the referenced resource. + minLength: 1 + type: string + namespace: + description: |- + Namespace of the referenced resource. + Namespace can be left empty. In such a case, namespace will + be implicit set to cluster's namespace. + type: string + required: + - kind + - name + - namespace + type: object + timeout: + description: |- + Timeout is how long to wait for the Job to reach Complete or Failed + before treating the check as failed. + type: string + required: + - jobRef + type: object kind: description: |- Kind of the resource to fetch in the managed Cluster. @@ -1761,6 +1887,9 @@ spec: - featureID - name type: object + x-kubernetes-validations: + - message: jobCheck cannot be set together with script or evaluateCEL + rule: '!has(self.jobCheck) || (!has(self.script) && !has(self.evaluateCEL))' type: array x-kubernetes-list-type: atomic reloader: @@ -1955,7 +2084,7 @@ spec: type: array featureID: description: |- - FeatureID is an indentifier of the feature (Helm/Kustomize/Resources) + FeatureID is an identifier of the feature (Helm/Kustomize/Resources) This field indicates when to run this check. For instance: - if set to Helm this check will be run after all helm @@ -1973,6 +2102,46 @@ spec: Group of the resource to fetch in the managed Cluster. Required when Kind is set. Leave empty for metric-only checks. type: string + jobCheck: + description: |- + JobCheck runs a Job in the managed cluster and uses its Complete/Failed + outcome as the check result. Mutually exclusive with Script and EvaluateCEL. + properties: + jobRef: + description: |- + JobRef references the Secret/ConfigMap containing the Job manifest to + deploy in the managed Cluster as this check. + properties: + kind: + description: 'Kind of the resource. Supported kinds + are: Secrets and ConfigMaps.' + enum: + - Secret + - ConfigMap + type: string + name: + description: Name of the referenced resource. + minLength: 1 + type: string + namespace: + description: |- + Namespace of the referenced resource. + Namespace can be left empty. In such a case, namespace will + be implicit set to cluster's namespace. + type: string + required: + - kind + - name + - namespace + type: object + timeout: + description: |- + Timeout is how long to wait for the Job to reach Complete or Failed + before treating the check as failed. + type: string + required: + - jobRef + type: object kind: description: |- Kind of the resource to fetch in the managed Cluster. @@ -2087,6 +2256,9 @@ spec: - featureID - name type: object + x-kubernetes-validations: + - message: jobCheck cannot be set together with script or evaluateCEL + rule: '!has(self.jobCheck) || (!has(self.script) && !has(self.evaluateCEL))' type: array x-kubernetes-list-type: atomic type: object diff --git a/config/crd/bases/config.projectsveltos.io_clusterpromotions.yaml b/config/crd/bases/config.projectsveltos.io_clusterpromotions.yaml index 7b1f7b41..e3eea918 100644 --- a/config/crd/bases/config.projectsveltos.io_clusterpromotions.yaml +++ b/config/crd/bases/config.projectsveltos.io_clusterpromotions.yaml @@ -1192,7 +1192,7 @@ spec: type: array featureID: description: |- - FeatureID is an indentifier of the feature (Helm/Kustomize/Resources) + FeatureID is an identifier of the feature (Helm/Kustomize/Resources) This field indicates when to run this check. For instance: - if set to Helm this check will be run after all helm @@ -1210,6 +1210,46 @@ spec: Group of the resource to fetch in the managed Cluster. Required when Kind is set. Leave empty for metric-only checks. type: string + jobCheck: + description: |- + JobCheck runs a Job in the managed cluster and uses its Complete/Failed + outcome as the check result. Mutually exclusive with Script and EvaluateCEL. + properties: + jobRef: + description: |- + JobRef references the Secret/ConfigMap containing the Job manifest to + deploy in the managed Cluster as this check. + properties: + kind: + description: 'Kind of the resource. Supported kinds + are: Secrets and ConfigMaps.' + enum: + - Secret + - ConfigMap + type: string + name: + description: Name of the referenced resource. + minLength: 1 + type: string + namespace: + description: |- + Namespace of the referenced resource. + Namespace can be left empty. In such a case, namespace will + be implicit set to cluster's namespace. + type: string + required: + - kind + - name + - namespace + type: object + timeout: + description: |- + Timeout is how long to wait for the Job to reach Complete or Failed + before treating the check as failed. + type: string + required: + - jobRef + type: object kind: description: |- Kind of the resource to fetch in the managed Cluster. @@ -1324,6 +1364,9 @@ spec: - featureID - name type: object + x-kubernetes-validations: + - message: jobCheck cannot be set together with script or evaluateCEL + rule: '!has(self.jobCheck) || (!has(self.script) && !has(self.evaluateCEL))' type: array x-kubernetes-list-type: atomic preDeleteChecks: @@ -1363,7 +1406,7 @@ spec: type: array featureID: description: |- - FeatureID is an indentifier of the feature (Helm/Kustomize/Resources) + FeatureID is an identifier of the feature (Helm/Kustomize/Resources) This field indicates when to run this check. For instance: - if set to Helm this check will be run after all helm @@ -1381,6 +1424,46 @@ spec: Group of the resource to fetch in the managed Cluster. Required when Kind is set. Leave empty for metric-only checks. type: string + jobCheck: + description: |- + JobCheck runs a Job in the managed cluster and uses its Complete/Failed + outcome as the check result. Mutually exclusive with Script and EvaluateCEL. + properties: + jobRef: + description: |- + JobRef references the Secret/ConfigMap containing the Job manifest to + deploy in the managed Cluster as this check. + properties: + kind: + description: 'Kind of the resource. Supported kinds + are: Secrets and ConfigMaps.' + enum: + - Secret + - ConfigMap + type: string + name: + description: Name of the referenced resource. + minLength: 1 + type: string + namespace: + description: |- + Namespace of the referenced resource. + Namespace can be left empty. In such a case, namespace will + be implicit set to cluster's namespace. + type: string + required: + - kind + - name + - namespace + type: object + timeout: + description: |- + Timeout is how long to wait for the Job to reach Complete or Failed + before treating the check as failed. + type: string + required: + - jobRef + type: object kind: description: |- Kind of the resource to fetch in the managed Cluster. @@ -1495,6 +1578,9 @@ spec: - featureID - name type: object + x-kubernetes-validations: + - message: jobCheck cannot be set together with script or evaluateCEL + rule: '!has(self.jobCheck) || (!has(self.script) && !has(self.evaluateCEL))' type: array x-kubernetes-list-type: atomic preDeployChecks: @@ -1534,7 +1620,7 @@ spec: type: array featureID: description: |- - FeatureID is an indentifier of the feature (Helm/Kustomize/Resources) + FeatureID is an identifier of the feature (Helm/Kustomize/Resources) This field indicates when to run this check. For instance: - if set to Helm this check will be run after all helm @@ -1552,6 +1638,46 @@ spec: Group of the resource to fetch in the managed Cluster. Required when Kind is set. Leave empty for metric-only checks. type: string + jobCheck: + description: |- + JobCheck runs a Job in the managed cluster and uses its Complete/Failed + outcome as the check result. Mutually exclusive with Script and EvaluateCEL. + properties: + jobRef: + description: |- + JobRef references the Secret/ConfigMap containing the Job manifest to + deploy in the managed Cluster as this check. + properties: + kind: + description: 'Kind of the resource. Supported kinds + are: Secrets and ConfigMaps.' + enum: + - Secret + - ConfigMap + type: string + name: + description: Name of the referenced resource. + minLength: 1 + type: string + namespace: + description: |- + Namespace of the referenced resource. + Namespace can be left empty. In such a case, namespace will + be implicit set to cluster's namespace. + type: string + required: + - kind + - name + - namespace + type: object + timeout: + description: |- + Timeout is how long to wait for the Job to reach Complete or Failed + before treating the check as failed. + type: string + required: + - jobRef + type: object kind: description: |- Kind of the resource to fetch in the managed Cluster. @@ -1666,6 +1792,9 @@ spec: - featureID - name type: object + x-kubernetes-validations: + - message: jobCheck cannot be set together with script or evaluateCEL + rule: '!has(self.jobCheck) || (!has(self.script) && !has(self.evaluateCEL))' type: array x-kubernetes-list-type: atomic reloader: @@ -1853,7 +1982,7 @@ spec: type: array featureID: description: |- - FeatureID is an indentifier of the feature (Helm/Kustomize/Resources) + FeatureID is an identifier of the feature (Helm/Kustomize/Resources) This field indicates when to run this check. For instance: - if set to Helm this check will be run after all helm @@ -1871,6 +2000,46 @@ spec: Group of the resource to fetch in the managed Cluster. Required when Kind is set. Leave empty for metric-only checks. type: string + jobCheck: + description: |- + JobCheck runs a Job in the managed cluster and uses its Complete/Failed + outcome as the check result. Mutually exclusive with Script and EvaluateCEL. + properties: + jobRef: + description: |- + JobRef references the Secret/ConfigMap containing the Job manifest to + deploy in the managed Cluster as this check. + properties: + kind: + description: 'Kind of the resource. Supported kinds + are: Secrets and ConfigMaps.' + enum: + - Secret + - ConfigMap + type: string + name: + description: Name of the referenced resource. + minLength: 1 + type: string + namespace: + description: |- + Namespace of the referenced resource. + Namespace can be left empty. In such a case, namespace will + be implicit set to cluster's namespace. + type: string + required: + - kind + - name + - namespace + type: object + timeout: + description: |- + Timeout is how long to wait for the Job to reach Complete or Failed + before treating the check as failed. + type: string + required: + - jobRef + type: object kind: description: |- Kind of the resource to fetch in the managed Cluster. @@ -1985,6 +2154,9 @@ spec: - featureID - name type: object + x-kubernetes-validations: + - message: jobCheck cannot be set together with script or evaluateCEL + rule: '!has(self.jobCheck) || (!has(self.script) && !has(self.evaluateCEL))' type: array x-kubernetes-list-type: atomic type: object @@ -2095,7 +2267,7 @@ spec: type: array featureID: description: |- - FeatureID is an indentifier of the feature (Helm/Kustomize/Resources) + FeatureID is an identifier of the feature (Helm/Kustomize/Resources) This field indicates when to run this check. For instance: - if set to Helm this check will be run after all helm @@ -2113,6 +2285,46 @@ spec: Group of the resource to fetch in the managed Cluster. Required when Kind is set. Leave empty for metric-only checks. type: string + jobCheck: + description: |- + JobCheck runs a Job in the managed cluster and uses its Complete/Failed + outcome as the check result. Mutually exclusive with Script and EvaluateCEL. + properties: + jobRef: + description: |- + JobRef references the Secret/ConfigMap containing the Job manifest to + deploy in the managed Cluster as this check. + properties: + kind: + description: 'Kind of the resource. Supported + kinds are: Secrets and ConfigMaps.' + enum: + - Secret + - ConfigMap + type: string + name: + description: Name of the referenced resource. + minLength: 1 + type: string + namespace: + description: |- + Namespace of the referenced resource. + Namespace can be left empty. In such a case, namespace will + be implicit set to cluster's namespace. + type: string + required: + - kind + - name + - namespace + type: object + timeout: + description: |- + Timeout is how long to wait for the Job to reach Complete or Failed + before treating the check as failed. + type: string + required: + - jobRef + type: object kind: description: |- Kind of the resource to fetch in the managed Cluster. @@ -2230,6 +2442,11 @@ spec: - featureID - name type: object + x-kubernetes-validations: + - message: jobCheck cannot be set together with script + or evaluateCEL + rule: '!has(self.jobCheck) || (!has(self.script) + && !has(self.evaluateCEL))' type: array preHealthCheckDeployment: description: |- @@ -2465,7 +2682,7 @@ spec: type: array featureID: description: |- - FeatureID is an indentifier of the feature (Helm/Kustomize/Resources) + FeatureID is an identifier of the feature (Helm/Kustomize/Resources) This field indicates when to run this check. For instance: - if set to Helm this check will be run after all helm @@ -2483,6 +2700,46 @@ spec: Group of the resource to fetch in the managed Cluster. Required when Kind is set. Leave empty for metric-only checks. type: string + jobCheck: + description: |- + JobCheck runs a Job in the managed cluster and uses its Complete/Failed + outcome as the check result. Mutually exclusive with Script and EvaluateCEL. + properties: + jobRef: + description: |- + JobRef references the Secret/ConfigMap containing the Job manifest to + deploy in the managed Cluster as this check. + properties: + kind: + description: 'Kind of the resource. Supported + kinds are: Secrets and ConfigMaps.' + enum: + - Secret + - ConfigMap + type: string + name: + description: Name of the referenced resource. + minLength: 1 + type: string + namespace: + description: |- + Namespace of the referenced resource. + Namespace can be left empty. In such a case, namespace will + be implicit set to cluster's namespace. + type: string + required: + - kind + - name + - namespace + type: object + timeout: + description: |- + Timeout is how long to wait for the Job to reach Complete or Failed + before treating the check as failed. + type: string + required: + - jobRef + type: object kind: description: |- Kind of the resource to fetch in the managed Cluster. @@ -2600,6 +2857,11 @@ spec: - featureID - name type: object + x-kubernetes-validations: + - message: jobCheck cannot be set together with script + or evaluateCEL + rule: '!has(self.jobCheck) || (!has(self.script) + && !has(self.evaluateCEL))' type: array preHealthCheckDeployment: description: |- diff --git a/config/crd/bases/config.projectsveltos.io_clustersummaries.yaml b/config/crd/bases/config.projectsveltos.io_clustersummaries.yaml index 495c5e95..59b4b00e 100644 --- a/config/crd/bases/config.projectsveltos.io_clustersummaries.yaml +++ b/config/crd/bases/config.projectsveltos.io_clustersummaries.yaml @@ -1329,7 +1329,7 @@ spec: type: array featureID: description: |- - FeatureID is an indentifier of the feature (Helm/Kustomize/Resources) + FeatureID is an identifier of the feature (Helm/Kustomize/Resources) This field indicates when to run this check. For instance: - if set to Helm this check will be run after all helm @@ -1347,6 +1347,46 @@ spec: Group of the resource to fetch in the managed Cluster. Required when Kind is set. Leave empty for metric-only checks. type: string + jobCheck: + description: |- + JobCheck runs a Job in the managed cluster and uses its Complete/Failed + outcome as the check result. Mutually exclusive with Script and EvaluateCEL. + properties: + jobRef: + description: |- + JobRef references the Secret/ConfigMap containing the Job manifest to + deploy in the managed Cluster as this check. + properties: + kind: + description: 'Kind of the resource. Supported kinds + are: Secrets and ConfigMaps.' + enum: + - Secret + - ConfigMap + type: string + name: + description: Name of the referenced resource. + minLength: 1 + type: string + namespace: + description: |- + Namespace of the referenced resource. + Namespace can be left empty. In such a case, namespace will + be implicit set to cluster's namespace. + type: string + required: + - kind + - name + - namespace + type: object + timeout: + description: |- + Timeout is how long to wait for the Job to reach Complete or Failed + before treating the check as failed. + type: string + required: + - jobRef + type: object kind: description: |- Kind of the resource to fetch in the managed Cluster. @@ -1461,6 +1501,9 @@ spec: - featureID - name type: object + x-kubernetes-validations: + - message: jobCheck cannot be set together with script or evaluateCEL + rule: '!has(self.jobCheck) || (!has(self.script) && !has(self.evaluateCEL))' type: array x-kubernetes-list-type: atomic preDeleteChecks: @@ -1500,7 +1543,7 @@ spec: type: array featureID: description: |- - FeatureID is an indentifier of the feature (Helm/Kustomize/Resources) + FeatureID is an identifier of the feature (Helm/Kustomize/Resources) This field indicates when to run this check. For instance: - if set to Helm this check will be run after all helm @@ -1518,6 +1561,46 @@ spec: Group of the resource to fetch in the managed Cluster. Required when Kind is set. Leave empty for metric-only checks. type: string + jobCheck: + description: |- + JobCheck runs a Job in the managed cluster and uses its Complete/Failed + outcome as the check result. Mutually exclusive with Script and EvaluateCEL. + properties: + jobRef: + description: |- + JobRef references the Secret/ConfigMap containing the Job manifest to + deploy in the managed Cluster as this check. + properties: + kind: + description: 'Kind of the resource. Supported kinds + are: Secrets and ConfigMaps.' + enum: + - Secret + - ConfigMap + type: string + name: + description: Name of the referenced resource. + minLength: 1 + type: string + namespace: + description: |- + Namespace of the referenced resource. + Namespace can be left empty. In such a case, namespace will + be implicit set to cluster's namespace. + type: string + required: + - kind + - name + - namespace + type: object + timeout: + description: |- + Timeout is how long to wait for the Job to reach Complete or Failed + before treating the check as failed. + type: string + required: + - jobRef + type: object kind: description: |- Kind of the resource to fetch in the managed Cluster. @@ -1632,6 +1715,9 @@ spec: - featureID - name type: object + x-kubernetes-validations: + - message: jobCheck cannot be set together with script or evaluateCEL + rule: '!has(self.jobCheck) || (!has(self.script) && !has(self.evaluateCEL))' type: array x-kubernetes-list-type: atomic preDeployChecks: @@ -1671,7 +1757,7 @@ spec: type: array featureID: description: |- - FeatureID is an indentifier of the feature (Helm/Kustomize/Resources) + FeatureID is an identifier of the feature (Helm/Kustomize/Resources) This field indicates when to run this check. For instance: - if set to Helm this check will be run after all helm @@ -1689,6 +1775,46 @@ spec: Group of the resource to fetch in the managed Cluster. Required when Kind is set. Leave empty for metric-only checks. type: string + jobCheck: + description: |- + JobCheck runs a Job in the managed cluster and uses its Complete/Failed + outcome as the check result. Mutually exclusive with Script and EvaluateCEL. + properties: + jobRef: + description: |- + JobRef references the Secret/ConfigMap containing the Job manifest to + deploy in the managed Cluster as this check. + properties: + kind: + description: 'Kind of the resource. Supported kinds + are: Secrets and ConfigMaps.' + enum: + - Secret + - ConfigMap + type: string + name: + description: Name of the referenced resource. + minLength: 1 + type: string + namespace: + description: |- + Namespace of the referenced resource. + Namespace can be left empty. In such a case, namespace will + be implicit set to cluster's namespace. + type: string + required: + - kind + - name + - namespace + type: object + timeout: + description: |- + Timeout is how long to wait for the Job to reach Complete or Failed + before treating the check as failed. + type: string + required: + - jobRef + type: object kind: description: |- Kind of the resource to fetch in the managed Cluster. @@ -1803,6 +1929,9 @@ spec: - featureID - name type: object + x-kubernetes-validations: + - message: jobCheck cannot be set together with script or evaluateCEL + rule: '!has(self.jobCheck) || (!has(self.script) && !has(self.evaluateCEL))' type: array x-kubernetes-list-type: atomic reloader: @@ -1998,7 +2127,7 @@ spec: type: array featureID: description: |- - FeatureID is an indentifier of the feature (Helm/Kustomize/Resources) + FeatureID is an identifier of the feature (Helm/Kustomize/Resources) This field indicates when to run this check. For instance: - if set to Helm this check will be run after all helm @@ -2016,6 +2145,46 @@ spec: Group of the resource to fetch in the managed Cluster. Required when Kind is set. Leave empty for metric-only checks. type: string + jobCheck: + description: |- + JobCheck runs a Job in the managed cluster and uses its Complete/Failed + outcome as the check result. Mutually exclusive with Script and EvaluateCEL. + properties: + jobRef: + description: |- + JobRef references the Secret/ConfigMap containing the Job manifest to + deploy in the managed Cluster as this check. + properties: + kind: + description: 'Kind of the resource. Supported kinds + are: Secrets and ConfigMaps.' + enum: + - Secret + - ConfigMap + type: string + name: + description: Name of the referenced resource. + minLength: 1 + type: string + namespace: + description: |- + Namespace of the referenced resource. + Namespace can be left empty. In such a case, namespace will + be implicit set to cluster's namespace. + type: string + required: + - kind + - name + - namespace + type: object + timeout: + description: |- + Timeout is how long to wait for the Job to reach Complete or Failed + before treating the check as failed. + type: string + required: + - jobRef + type: object kind: description: |- Kind of the resource to fetch in the managed Cluster. @@ -2130,6 +2299,9 @@ spec: - featureID - name type: object + x-kubernetes-validations: + - message: jobCheck cannot be set together with script or evaluateCEL + rule: '!has(self.jobCheck) || (!has(self.script) && !has(self.evaluateCEL))' type: array x-kubernetes-list-type: atomic type: object @@ -2164,7 +2336,7 @@ spec: type: string type: array featureID: - description: FeatureID is an indentifier of the feature whose + description: FeatureID is an identifier of the feature whose status is reported enum: - Resources diff --git a/config/crd/bases/config.projectsveltos.io_profiles.yaml b/config/crd/bases/config.projectsveltos.io_profiles.yaml index 21b45e0a..87338483 100644 --- a/config/crd/bases/config.projectsveltos.io_profiles.yaml +++ b/config/crd/bases/config.projectsveltos.io_profiles.yaml @@ -1289,7 +1289,7 @@ spec: type: array featureID: description: |- - FeatureID is an indentifier of the feature (Helm/Kustomize/Resources) + FeatureID is an identifier of the feature (Helm/Kustomize/Resources) This field indicates when to run this check. For instance: - if set to Helm this check will be run after all helm @@ -1307,6 +1307,46 @@ spec: Group of the resource to fetch in the managed Cluster. Required when Kind is set. Leave empty for metric-only checks. type: string + jobCheck: + description: |- + JobCheck runs a Job in the managed cluster and uses its Complete/Failed + outcome as the check result. Mutually exclusive with Script and EvaluateCEL. + properties: + jobRef: + description: |- + JobRef references the Secret/ConfigMap containing the Job manifest to + deploy in the managed Cluster as this check. + properties: + kind: + description: 'Kind of the resource. Supported kinds + are: Secrets and ConfigMaps.' + enum: + - Secret + - ConfigMap + type: string + name: + description: Name of the referenced resource. + minLength: 1 + type: string + namespace: + description: |- + Namespace of the referenced resource. + Namespace can be left empty. In such a case, namespace will + be implicit set to cluster's namespace. + type: string + required: + - kind + - name + - namespace + type: object + timeout: + description: |- + Timeout is how long to wait for the Job to reach Complete or Failed + before treating the check as failed. + type: string + required: + - jobRef + type: object kind: description: |- Kind of the resource to fetch in the managed Cluster. @@ -1421,6 +1461,9 @@ spec: - featureID - name type: object + x-kubernetes-validations: + - message: jobCheck cannot be set together with script or evaluateCEL + rule: '!has(self.jobCheck) || (!has(self.script) && !has(self.evaluateCEL))' type: array x-kubernetes-list-type: atomic preDeleteChecks: @@ -1459,7 +1502,7 @@ spec: type: array featureID: description: |- - FeatureID is an indentifier of the feature (Helm/Kustomize/Resources) + FeatureID is an identifier of the feature (Helm/Kustomize/Resources) This field indicates when to run this check. For instance: - if set to Helm this check will be run after all helm @@ -1477,6 +1520,46 @@ spec: Group of the resource to fetch in the managed Cluster. Required when Kind is set. Leave empty for metric-only checks. type: string + jobCheck: + description: |- + JobCheck runs a Job in the managed cluster and uses its Complete/Failed + outcome as the check result. Mutually exclusive with Script and EvaluateCEL. + properties: + jobRef: + description: |- + JobRef references the Secret/ConfigMap containing the Job manifest to + deploy in the managed Cluster as this check. + properties: + kind: + description: 'Kind of the resource. Supported kinds + are: Secrets and ConfigMaps.' + enum: + - Secret + - ConfigMap + type: string + name: + description: Name of the referenced resource. + minLength: 1 + type: string + namespace: + description: |- + Namespace of the referenced resource. + Namespace can be left empty. In such a case, namespace will + be implicit set to cluster's namespace. + type: string + required: + - kind + - name + - namespace + type: object + timeout: + description: |- + Timeout is how long to wait for the Job to reach Complete or Failed + before treating the check as failed. + type: string + required: + - jobRef + type: object kind: description: |- Kind of the resource to fetch in the managed Cluster. @@ -1591,6 +1674,9 @@ spec: - featureID - name type: object + x-kubernetes-validations: + - message: jobCheck cannot be set together with script or evaluateCEL + rule: '!has(self.jobCheck) || (!has(self.script) && !has(self.evaluateCEL))' type: array x-kubernetes-list-type: atomic preDeployChecks: @@ -1629,7 +1715,7 @@ spec: type: array featureID: description: |- - FeatureID is an indentifier of the feature (Helm/Kustomize/Resources) + FeatureID is an identifier of the feature (Helm/Kustomize/Resources) This field indicates when to run this check. For instance: - if set to Helm this check will be run after all helm @@ -1647,6 +1733,46 @@ spec: Group of the resource to fetch in the managed Cluster. Required when Kind is set. Leave empty for metric-only checks. type: string + jobCheck: + description: |- + JobCheck runs a Job in the managed cluster and uses its Complete/Failed + outcome as the check result. Mutually exclusive with Script and EvaluateCEL. + properties: + jobRef: + description: |- + JobRef references the Secret/ConfigMap containing the Job manifest to + deploy in the managed Cluster as this check. + properties: + kind: + description: 'Kind of the resource. Supported kinds + are: Secrets and ConfigMaps.' + enum: + - Secret + - ConfigMap + type: string + name: + description: Name of the referenced resource. + minLength: 1 + type: string + namespace: + description: |- + Namespace of the referenced resource. + Namespace can be left empty. In such a case, namespace will + be implicit set to cluster's namespace. + type: string + required: + - kind + - name + - namespace + type: object + timeout: + description: |- + Timeout is how long to wait for the Job to reach Complete or Failed + before treating the check as failed. + type: string + required: + - jobRef + type: object kind: description: |- Kind of the resource to fetch in the managed Cluster. @@ -1761,6 +1887,9 @@ spec: - featureID - name type: object + x-kubernetes-validations: + - message: jobCheck cannot be set together with script or evaluateCEL + rule: '!has(self.jobCheck) || (!has(self.script) && !has(self.evaluateCEL))' type: array x-kubernetes-list-type: atomic reloader: @@ -1955,7 +2084,7 @@ spec: type: array featureID: description: |- - FeatureID is an indentifier of the feature (Helm/Kustomize/Resources) + FeatureID is an identifier of the feature (Helm/Kustomize/Resources) This field indicates when to run this check. For instance: - if set to Helm this check will be run after all helm @@ -1973,6 +2102,46 @@ spec: Group of the resource to fetch in the managed Cluster. Required when Kind is set. Leave empty for metric-only checks. type: string + jobCheck: + description: |- + JobCheck runs a Job in the managed cluster and uses its Complete/Failed + outcome as the check result. Mutually exclusive with Script and EvaluateCEL. + properties: + jobRef: + description: |- + JobRef references the Secret/ConfigMap containing the Job manifest to + deploy in the managed Cluster as this check. + properties: + kind: + description: 'Kind of the resource. Supported kinds + are: Secrets and ConfigMaps.' + enum: + - Secret + - ConfigMap + type: string + name: + description: Name of the referenced resource. + minLength: 1 + type: string + namespace: + description: |- + Namespace of the referenced resource. + Namespace can be left empty. In such a case, namespace will + be implicit set to cluster's namespace. + type: string + required: + - kind + - name + - namespace + type: object + timeout: + description: |- + Timeout is how long to wait for the Job to reach Complete or Failed + before treating the check as failed. + type: string + required: + - jobRef + type: object kind: description: |- Kind of the resource to fetch in the managed Cluster. @@ -2087,6 +2256,9 @@ spec: - featureID - name type: object + x-kubernetes-validations: + - message: jobCheck cannot be set together with script or evaluateCEL + rule: '!has(self.jobCheck) || (!has(self.script) && !has(self.evaluateCEL))' type: array x-kubernetes-list-type: atomic type: object diff --git a/controllers/clusterpromotion_controller.go b/controllers/clusterpromotion_controller.go index a0167efe..c846e8c8 100644 --- a/controllers/clusterpromotion_controller.go +++ b/controllers/clusterpromotion_controller.go @@ -45,10 +45,9 @@ const ( maxFreeClusterPromotionStages = 2 ) -// clusterPromotionEnterpriseDeps bundles what the Sveltos Enterprise ClusterPromotion -// implementation needs. Defined here, not in sveltos-enterprise, so this (open source) -// package can declare the plugin seam below without importing anything private. -type clusterPromotionEnterpriseDeps struct { +// ClusterPromotionEnterpriseDeps bundles what the Sveltos Enterprise ClusterPromotion +// implementation needs. +type ClusterPromotionEnterpriseDeps struct { Client client.Client Config *rest.Config Scheme *runtime.Scheme @@ -56,18 +55,24 @@ type clusterPromotionEnterpriseDeps struct { SveltosNamespace string } +// reconcileClusterPromotionNormal implements ClusterPromotion's stage-advancement business +// logic (a Sveltos Enterprise feature). The default (clusterpromotion_default.go) is a stub +// that reports the feature is unavailable. A Sveltos Enterprise build wires in the real +// implementation via SetClusterPromotionReconciler before starting the manager; this package +// never imports anything private itself. var ( - // reconcileClusterPromotionNormal implements ClusterPromotion's stage-advancement business - // logic (a Sveltos Enterprise feature). A default (non-"enterprise" tagged) build of this - // package cannot import the private sveltos-enterprise module at all, so it wires this to - // a stub that reports the feature is unavailable (see clusterpromotion_oss.go). Official - // Sveltos images are built with `-tags enterprise` and a checkout of sveltos-enterprise - // available, which wires this to the real implementation instead (see - // clusterpromotion_plugin.go). Either way, this file never imports sveltos-enterprise. - reconcileClusterPromotionNormal func(ctx context.Context, deps clusterPromotionEnterpriseDeps, + reconcileClusterPromotionNormal func(ctx context.Context, deps ClusterPromotionEnterpriseDeps, promotionScope *scope.ClusterPromotionScope, isInFreeTopX bool, logger logr.Logger) reconcile.Result ) +// SetClusterPromotionReconciler overrides the ClusterPromotion stage-advancement implementation. +// Called by a Sveltos Enterprise build's composition root before starting the manager. +func SetClusterPromotionReconciler(fn func(ctx context.Context, deps ClusterPromotionEnterpriseDeps, + promotionScope *scope.ClusterPromotionScope, isInFreeTopX bool, logger logr.Logger) reconcile.Result) { + + reconcileClusterPromotionNormal = fn +} + var ( clusterPromotionNameLabel = "config.projectsveltos.io/promotionname" ) @@ -185,7 +190,7 @@ func (r *ClusterPromotionReconciler) reconcileNormal( isInFreeTopX := GetLicenseManager().IsClusterPromotionInTopX("", promotionScope.ClusterPromotion.Name, maxFreeClusterPromotionStages) - deps := clusterPromotionEnterpriseDeps{ + deps := ClusterPromotionEnterpriseDeps{ Client: r.Client, Config: r.Config, Scheme: r.Scheme, diff --git a/controllers/clusterpromotion_oss.go b/controllers/clusterpromotion_default.go similarity index 72% rename from controllers/clusterpromotion_oss.go rename to controllers/clusterpromotion_default.go index 9c36cdb0..02c8991c 100644 --- a/controllers/clusterpromotion_oss.go +++ b/controllers/clusterpromotion_default.go @@ -1,5 +1,3 @@ -//go:build !enterprise - /* Copyright 2026. projectsveltos.io. All rights reserved. @@ -28,12 +26,10 @@ import ( logs "github.com/projectsveltos/libsveltos/lib/logsettings" ) -// Default (non-"enterprise") build: ClusterPromotion is a Sveltos Enterprise feature and -// its implementation lives in the private sveltos-enterprise module, which this build -// does not import. Building official images requires `-tags enterprise` with a checkout -// of sveltos-enterprise available; see clusterpromotion_plugin.go. +// Default ClusterPromotion implementation: reports the feature is unavailable. A Sveltos +// Enterprise build overrides this via SetClusterPromotionReconciler before starting the manager. func init() { - reconcileClusterPromotionNormal = func(_ context.Context, _ clusterPromotionEnterpriseDeps, + reconcileClusterPromotionNormal = func(_ context.Context, _ ClusterPromotionEnterpriseDeps, promotionScope *scope.ClusterPromotionScope, _ bool, logger logr.Logger) reconcile.Result { logger.V(logs.LogInfo).Info("ClusterPromotion requires a Sveltos Enterprise build") diff --git a/controllers/clusterpromotion_plugin.go b/controllers/clusterpromotion_plugin.go deleted file mode 100644 index fb5138b4..00000000 --- a/controllers/clusterpromotion_plugin.go +++ /dev/null @@ -1,49 +0,0 @@ -//go:build enterprise - -/* -Copyright 2026. projectsveltos.io. All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package controllers - -import ( - "context" - - "github.com/go-logr/logr" - "sigs.k8s.io/controller-runtime/pkg/reconcile" - - "github.com/projectsveltos/addon-controller/pkg/scope" - "github.com/projectsveltos/sveltos-enterprise/clusterpromotion" -) - -// Official Sveltos images are built with `-tags enterprise` and a checkout of the private -// sveltos-enterprise module available (CI only). This file is the only place in this -// package that imports it; a default build excludes this file entirely (see -// clusterpromotion_oss.go), so `go build` from a public clone never needs to resolve it. -func init() { - reconcileClusterPromotionNormal = func(ctx context.Context, deps clusterPromotionEnterpriseDeps, - promotionScope *scope.ClusterPromotionScope, isInFreeTopX bool, logger logr.Logger) reconcile.Result { - - enterpriseReconciler := clusterpromotion.Reconciler{ - Client: deps.Client, - Config: deps.Config, - Scheme: deps.Scheme, - EventRecorder: deps.EventRecorder, - SveltosNamespace: deps.SveltosNamespace, - } - - return enterpriseReconciler.ReconcileNormal(ctx, promotionScope, isInFreeTopX, logger) - } -} diff --git a/controllers/delete_checks.go b/controllers/delete_checks.go index bb700715..ac811938 100644 --- a/controllers/delete_checks.go +++ b/controllers/delete_checks.go @@ -75,8 +75,8 @@ func validateDeleteChecks(ctx context.Context, clusterSummary *configv1beta1.Clu } logger.V(logs.LogDebug).Info("validate delete checks") - err = clusterops.ValidateHealthPolicies(ctx, remoteRestConfig, - clusterSummary.Spec.ClusterProfileSpec.PreDeleteChecks, featureID, true, logger) + err = clusterops.ValidateHealthPolicies(ctx, getManagementClusterClient(), clusterSummary, getSveltosNamespace(), + remoteRestConfig, clusterSummary.Spec.ClusterProfileSpec.PreDeleteChecks, featureID, true, logger) if err != nil { logger.V(logs.LogDebug).Error(err, "delete check failed") return err diff --git a/controllers/handlers_helm.go b/controllers/handlers_helm.go index 22cda222..77de9f87 100644 --- a/controllers/handlers_helm.go +++ b/controllers/handlers_helm.go @@ -280,8 +280,8 @@ func postProcessDeployedHelmCharts(ctx context.Context, clusterSummary *configv1 if err != nil { return err } - return clusterops.ValidateHealthPolicies(ctx, remoteRestConfig, clusterSummary.Spec.ClusterProfileSpec.ValidateHealths, - libsveltosv1beta1.FeatureHelm, false, logger) + return clusterops.ValidateHealthPolicies(ctx, c, clusterSummary, getSveltosNamespace(), remoteRestConfig, + clusterSummary.Spec.ClusterProfileSpec.ValidateHealths, libsveltosv1beta1.FeatureHelm, false, logger) } func manageDriftDetectionManagerDeploymentForHelm(ctx context.Context, c client.Client, diff --git a/controllers/handlers_kustomize.go b/controllers/handlers_kustomize.go index c2198901..6a9456b8 100644 --- a/controllers/handlers_kustomize.go +++ b/controllers/handlers_kustomize.go @@ -249,8 +249,8 @@ func processKustomizeDeployment(ctx context.Context, remoteRestConfig *rest.Conf return err } - return clusterops.ValidateHealthPolicies(ctx, remoteRestConfig, clusterSummary.Spec.ClusterProfileSpec.ValidateHealths, - libsveltosv1beta1.FeatureKustomize, false, logger) + return clusterops.ValidateHealthPolicies(ctx, c, clusterSummary, getSveltosNamespace(), remoteRestConfig, + clusterSummary.Spec.ClusterProfileSpec.ValidateHealths, libsveltosv1beta1.FeatureKustomize, false, logger) } func cleanStaleKustomizeResources(ctx context.Context, clusterSummary *configv1beta1.ClusterSummary, diff --git a/controllers/handlers_resources.go b/controllers/handlers_resources.go index 43329615..8679af44 100644 --- a/controllers/handlers_resources.go +++ b/controllers/handlers_resources.go @@ -207,8 +207,8 @@ func postProcessDeployedResources(ctx context.Context, remoteRestConfig *rest.Co return err } - return clusterops.ValidateHealthPolicies(ctx, remoteRestConfig, clusterSummary.Spec.ClusterProfileSpec.ValidateHealths, - libsveltosv1beta1.FeatureResources, false, logger) + return clusterops.ValidateHealthPolicies(ctx, c, clusterSummary, getSveltosNamespace(), remoteRestConfig, + clusterSummary.Spec.ClusterProfileSpec.ValidateHealths, libsveltosv1beta1.FeatureResources, false, logger) } func cleanStaleResources(ctx context.Context, clusterSummary *configv1beta1.ClusterSummary, diff --git a/controllers/handlers_utils.go b/controllers/handlers_utils.go index 2e22ba65..f793bc7b 100644 --- a/controllers/handlers_utils.go +++ b/controllers/handlers_utils.go @@ -1916,6 +1916,6 @@ func validatePreDeployChecks(ctx context.Context, c client.Client, clusterSummar return err } - return clusterops.ValidateHealthPolicies(ctx, remoteRestConfig, + return clusterops.ValidateHealthPolicies(ctx, c, clusterSummary, getSveltosNamespace(), remoteRestConfig, clusterSummary.Spec.ClusterProfileSpec.PreDeployChecks, featureID, false, logger) } diff --git a/go.mod b/go.mod index 993d4ea6..d96e84fe 100644 --- a/go.mod +++ b/go.mod @@ -20,8 +20,7 @@ require ( github.com/onsi/gomega v1.42.1 github.com/opencontainers/image-spec v1.1.1 github.com/pkg/errors v0.9.1 - github.com/projectsveltos/libsveltos v1.13.1-0.20260730171509-41b249690865 - github.com/projectsveltos/sveltos-enterprise v0.0.0-20260728165936-3be605d9a912 + github.com/projectsveltos/libsveltos v1.13.1-0.20260804055751-6f6de6aa73cb github.com/prometheus/client_golang v1.24.1 github.com/sigstore/cosign/v3 v3.1.2 github.com/sigstore/sigstore v1.10.8 @@ -226,7 +225,6 @@ require ( github.com/prometheus/client_model v0.6.2 // indirect github.com/prometheus/common v0.70.1 // indirect github.com/prometheus/procfs v0.21.1 // indirect - github.com/robfig/cron v1.2.0 // indirect github.com/rubenv/sql-migrate v1.8.1 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/sagikazarmark/locafero v0.11.0 // indirect diff --git a/go.sum b/go.sum index 933ac675..c3507fe6 100644 --- a/go.sum +++ b/go.sum @@ -642,8 +642,8 @@ github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRI github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/poy/onpar v1.1.2 h1:QaNrNiZx0+Nar5dLgTVp5mXkyoVFIbepjyEoGSnhbAY= github.com/poy/onpar v1.1.2/go.mod h1:6X8FLNoxyr9kkmnlqpK6LSoiOtrO6MICtWwEuWkLjzg= -github.com/projectsveltos/libsveltos v1.13.1-0.20260730171509-41b249690865 h1:DKfSfxhZ9r9EODFeBX2wUyB5ayS9uowPtybaABVJDdY= -github.com/projectsveltos/libsveltos v1.13.1-0.20260730171509-41b249690865/go.mod h1:TWOi1aZgpKGGI8lGdXhNE9Rnf3akS5bEVUuDV0EfmSo= +github.com/projectsveltos/libsveltos v1.13.1-0.20260804055751-6f6de6aa73cb h1:nhZnxIOEzx3kJnuMGPNmSBl/TH4NtMLAy24XfgDCzdE= +github.com/projectsveltos/libsveltos v1.13.1-0.20260804055751-6f6de6aa73cb/go.mod h1:TWOi1aZgpKGGI8lGdXhNE9Rnf3akS5bEVUuDV0EfmSo= github.com/projectsveltos/lua-utils/glua-json v0.0.0-20251212200258-2b3cdcb7c0f5 h1:khnc+994UszxZYu69J+R5FKiLA/Nk1JQj0EYAkwTWz0= github.com/projectsveltos/lua-utils/glua-json v0.0.0-20251212200258-2b3cdcb7c0f5/go.mod h1:yVL8KQFa9tmcxgwl9nwIMtKgtmIVC1zaFRSCfOwYvPY= github.com/projectsveltos/lua-utils/glua-runes v0.0.0-20251212200258-2b3cdcb7c0f5 h1:YbsebwRwTRhV8QacvEAdFqxcxHdeu7JTVtsBovbkgos= @@ -652,8 +652,6 @@ github.com/projectsveltos/lua-utils/glua-sprig v0.0.0-20251212200258-2b3cdcb7c0f github.com/projectsveltos/lua-utils/glua-sprig v0.0.0-20251212200258-2b3cdcb7c0f5/go.mod h1:esi+znTJzieo7M60Ytx56vJZbWJ8WuJfVyRvOCOWs64= github.com/projectsveltos/lua-utils/glua-strings v0.0.0-20251212200258-2b3cdcb7c0f5 h1:ifNj1y4pqhSSDL0B5XfCPTnFy9ZjGTzStuOJu1jE9xs= github.com/projectsveltos/lua-utils/glua-strings v0.0.0-20251212200258-2b3cdcb7c0f5/go.mod h1:P/l817Avvelnzyb9YyMmnDYG3OabOyK8KuWF8s35kj0= -github.com/projectsveltos/sveltos-enterprise v0.0.0-20260728165936-3be605d9a912 h1:o/fB6TrHaJCyqv9odkaDcBVmK/TGPoRfBqb8rtVzqgc= -github.com/projectsveltos/sveltos-enterprise v0.0.0-20260728165936-3be605d9a912/go.mod h1:dThrx/Isg2w+51UFQCOG3x/idcaPDyjtMKa9wY2OIzM= github.com/prometheus/client_golang v1.24.1 h1:JnJkREXzWxUdCuPFpIWZiPispT9xVV59uiuyR2bPlnU= github.com/prometheus/client_golang v1.24.1/go.mod h1:F+oSRECHg4sse5ucfYpYDeIv/hu68Zo0uoHKetWnzcE= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= @@ -671,8 +669,6 @@ github.com/redis/go-redis/extra/redisotel/v9 v9.5.3 h1:kuvuJL/+MZIEdvtb/kTBRiRgY github.com/redis/go-redis/extra/redisotel/v9 v9.5.3/go.mod h1:7f/FMrf5RRRVHXgfk7CzSVzXHiWeuOQUu2bsVqWoa+g= github.com/redis/go-redis/v9 v9.20.1 h1:sfCU6A8P3dXbKyWes02uxA2baehGux9dZHfEKtsTB1w= github.com/redis/go-redis/v9 v9.20.1/go.mod h1:v/M13XI1PVCDcm01VtPFOADfZtHf8YW3baQf57KlIkA= -github.com/robfig/cron v1.2.0 h1:ZjScXvvxeQ63Dbyxy76Fj3AT3Ut0aKsyd2/tl3DTMuQ= -github.com/robfig/cron v1.2.0/go.mod h1:JGuDeoQd7Z6yL4zQhZ3OPEVHB7fL6Ka6skscFHfmt2k= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/rubenv/sql-migrate v1.8.1 h1:EPNwCvjAowHI3TnZ+4fQu3a915OpnQoPAjTXCGOy2U0= diff --git a/lib/clusterops/jobhealthcheck_default.go b/lib/clusterops/jobhealthcheck_default.go new file mode 100644 index 00000000..8f2b389a --- /dev/null +++ b/lib/clusterops/jobhealthcheck_default.go @@ -0,0 +1,36 @@ +/* +Copyright 2026. projectsveltos.io. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package clusterops + +import ( + "context" + "fmt" + + "github.com/go-logr/logr" + + libsveltosv1beta1 "github.com/projectsveltos/libsveltos/api/v1beta1" +) + +// Default JobCheck implementation: reports the feature is unavailable. A Sveltos Enterprise +// build overrides this via SetJobHealthCheckValidator before starting the manager. +func init() { + validateJobHealthCheck = func(_ context.Context, _ JobHealthCheckDeps, + check *libsveltosv1beta1.ValidateHealth, _ logr.Logger) error { + + return fmt.Errorf("JobCheck (%s) requires a Sveltos Enterprise build", check.Name) + } +} diff --git a/lib/clusterops/validate_health.go b/lib/clusterops/validate_health.go index 177f86e4..5234ad9f 100644 --- a/lib/clusterops/validate_health.go +++ b/lib/clusterops/validate_health.go @@ -37,7 +37,9 @@ import ( "k8s.io/client-go/dynamic" "k8s.io/client-go/rest" "k8s.io/client-go/restmapper" + "sigs.k8s.io/controller-runtime/pkg/client" + configv1beta1 "github.com/projectsveltos/addon-controller/api/v1beta1" libsveltosv1beta1 "github.com/projectsveltos/libsveltos/api/v1beta1" "github.com/projectsveltos/libsveltos/lib/cel" logs "github.com/projectsveltos/libsveltos/lib/logsettings" @@ -69,6 +71,31 @@ func (e *HealthCheckError) Unwrap() error { return e.InternalErr } +// JobHealthCheckDeps bundles what the Sveltos Enterprise JobHealthCheck implementation needs. +type JobHealthCheckDeps struct { + MgmtClient client.Client + ClusterSummary *configv1beta1.ClusterSummary + SveltosNamespace string + RemoteConfig *rest.Config +} + +// validateJobHealthCheck runs a JobCheck. JobCheck is a Sveltos Enterprise feature; the default +// (jobhealthcheck_oss.go) is a stub reporting the feature is unavailable. A Sveltos Enterprise +// build wires in the real implementation via SetJobHealthCheckValidator. +var ( + validateJobHealthCheck func(ctx context.Context, deps JobHealthCheckDeps, + check *libsveltosv1beta1.ValidateHealth, logger logr.Logger) error +) + +// SetJobHealthCheckValidator overrides the JobCheck implementation. Called by a Sveltos +// Enterprise build's composition root before starting the manager; this package never imports +// anything private itself. +func SetJobHealthCheckValidator(fn func(ctx context.Context, deps JobHealthCheckDeps, + check *libsveltosv1beta1.ValidateHealth, logger logr.Logger) error) { + + validateJobHealthCheck = fn +} + // prometheusResponse is the top-level Prometheus HTTP API response. type prometheusResponse struct { Status string `json:"status"` @@ -83,7 +110,8 @@ type prometheusData struct { } // ValidateHealthPolicies runs all validateDeployment checks registered for the feature (Helm/Kustomize/Resources) -func ValidateHealthPolicies(ctx context.Context, remoteConfig *rest.Config, validateHealths []libsveltosv1beta1.ValidateHealth, +func ValidateHealthPolicies(ctx context.Context, mgmtClient client.Client, clusterSummary *configv1beta1.ClusterSummary, + sveltosNamespace string, remoteConfig *rest.Config, validateHealths []libsveltosv1beta1.ValidateHealth, featureID libsveltosv1beta1.FeatureID, isDelete bool, logger logr.Logger) error { // If SveltosCluster is in pull mode, this will done by the agent in the managed cluster @@ -98,7 +126,8 @@ func ValidateHealthPolicies(ctx context.Context, remoteConfig *rest.Config, vali continue } - if err := validateHealthPolicy(ctx, remoteConfig, check, isDelete, logger); err != nil { + if err := validateHealthPolicy(ctx, mgmtClient, clusterSummary, sveltosNamespace, remoteConfig, + check, isDelete, logger); err != nil { logger.V(logs.LogInfo).Info(fmt.Sprintf("failed to validate check: %s", err)) return &HealthCheckError{ FeatureID: featureID, @@ -111,12 +140,22 @@ func ValidateHealthPolicies(ctx context.Context, remoteConfig *rest.Config, vali return nil } -func validateHealthPolicy(ctx context.Context, remoteConfig *rest.Config, check *libsveltosv1beta1.ValidateHealth, +func validateHealthPolicy(ctx context.Context, mgmtClient client.Client, clusterSummary *configv1beta1.ClusterSummary, + sveltosNamespace string, remoteConfig *rest.Config, check *libsveltosv1beta1.ValidateHealth, isDelete bool, logger logr.Logger) error { l := logger.WithValues("validation", check.Name) l.V(logs.LogDebug).Info("running health validation") + if check.JobCheck != nil { + return validateJobHealthCheck(ctx, JobHealthCheckDeps{ + MgmtClient: mgmtClient, + ClusterSummary: clusterSummary, + SveltosNamespace: sveltosNamespace, + RemoteConfig: remoteConfig, + }, check, l) + } + metricsData, err := fetchMetrics(ctx, remoteConfig, check, l) if err != nil { return err diff --git a/lib/crd/clusterprofiles.go b/lib/crd/clusterprofiles.go index 8808ae44..d32d9187 100644 --- a/lib/crd/clusterprofiles.go +++ b/lib/crd/clusterprofiles.go @@ -1308,7 +1308,7 @@ spec: type: array featureID: description: |- - FeatureID is an indentifier of the feature (Helm/Kustomize/Resources) + FeatureID is an identifier of the feature (Helm/Kustomize/Resources) This field indicates when to run this check. For instance: - if set to Helm this check will be run after all helm @@ -1326,6 +1326,46 @@ spec: Group of the resource to fetch in the managed Cluster. Required when Kind is set. Leave empty for metric-only checks. type: string + jobCheck: + description: |- + JobCheck runs a Job in the managed cluster and uses its Complete/Failed + outcome as the check result. Mutually exclusive with Script and EvaluateCEL. + properties: + jobRef: + description: |- + JobRef references the Secret/ConfigMap containing the Job manifest to + deploy in the managed Cluster as this check. + properties: + kind: + description: 'Kind of the resource. Supported kinds + are: Secrets and ConfigMaps.' + enum: + - Secret + - ConfigMap + type: string + name: + description: Name of the referenced resource. + minLength: 1 + type: string + namespace: + description: |- + Namespace of the referenced resource. + Namespace can be left empty. In such a case, namespace will + be implicit set to cluster's namespace. + type: string + required: + - kind + - name + - namespace + type: object + timeout: + description: |- + Timeout is how long to wait for the Job to reach Complete or Failed + before treating the check as failed. + type: string + required: + - jobRef + type: object kind: description: |- Kind of the resource to fetch in the managed Cluster. @@ -1440,6 +1480,9 @@ spec: - featureID - name type: object + x-kubernetes-validations: + - message: jobCheck cannot be set together with script or evaluateCEL + rule: '!has(self.jobCheck) || (!has(self.script) && !has(self.evaluateCEL))' type: array x-kubernetes-list-type: atomic preDeleteChecks: @@ -1478,7 +1521,7 @@ spec: type: array featureID: description: |- - FeatureID is an indentifier of the feature (Helm/Kustomize/Resources) + FeatureID is an identifier of the feature (Helm/Kustomize/Resources) This field indicates when to run this check. For instance: - if set to Helm this check will be run after all helm @@ -1496,6 +1539,46 @@ spec: Group of the resource to fetch in the managed Cluster. Required when Kind is set. Leave empty for metric-only checks. type: string + jobCheck: + description: |- + JobCheck runs a Job in the managed cluster and uses its Complete/Failed + outcome as the check result. Mutually exclusive with Script and EvaluateCEL. + properties: + jobRef: + description: |- + JobRef references the Secret/ConfigMap containing the Job manifest to + deploy in the managed Cluster as this check. + properties: + kind: + description: 'Kind of the resource. Supported kinds + are: Secrets and ConfigMaps.' + enum: + - Secret + - ConfigMap + type: string + name: + description: Name of the referenced resource. + minLength: 1 + type: string + namespace: + description: |- + Namespace of the referenced resource. + Namespace can be left empty. In such a case, namespace will + be implicit set to cluster's namespace. + type: string + required: + - kind + - name + - namespace + type: object + timeout: + description: |- + Timeout is how long to wait for the Job to reach Complete or Failed + before treating the check as failed. + type: string + required: + - jobRef + type: object kind: description: |- Kind of the resource to fetch in the managed Cluster. @@ -1610,6 +1693,9 @@ spec: - featureID - name type: object + x-kubernetes-validations: + - message: jobCheck cannot be set together with script or evaluateCEL + rule: '!has(self.jobCheck) || (!has(self.script) && !has(self.evaluateCEL))' type: array x-kubernetes-list-type: atomic preDeployChecks: @@ -1648,7 +1734,7 @@ spec: type: array featureID: description: |- - FeatureID is an indentifier of the feature (Helm/Kustomize/Resources) + FeatureID is an identifier of the feature (Helm/Kustomize/Resources) This field indicates when to run this check. For instance: - if set to Helm this check will be run after all helm @@ -1666,6 +1752,46 @@ spec: Group of the resource to fetch in the managed Cluster. Required when Kind is set. Leave empty for metric-only checks. type: string + jobCheck: + description: |- + JobCheck runs a Job in the managed cluster and uses its Complete/Failed + outcome as the check result. Mutually exclusive with Script and EvaluateCEL. + properties: + jobRef: + description: |- + JobRef references the Secret/ConfigMap containing the Job manifest to + deploy in the managed Cluster as this check. + properties: + kind: + description: 'Kind of the resource. Supported kinds + are: Secrets and ConfigMaps.' + enum: + - Secret + - ConfigMap + type: string + name: + description: Name of the referenced resource. + minLength: 1 + type: string + namespace: + description: |- + Namespace of the referenced resource. + Namespace can be left empty. In such a case, namespace will + be implicit set to cluster's namespace. + type: string + required: + - kind + - name + - namespace + type: object + timeout: + description: |- + Timeout is how long to wait for the Job to reach Complete or Failed + before treating the check as failed. + type: string + required: + - jobRef + type: object kind: description: |- Kind of the resource to fetch in the managed Cluster. @@ -1780,6 +1906,9 @@ spec: - featureID - name type: object + x-kubernetes-validations: + - message: jobCheck cannot be set together with script or evaluateCEL + rule: '!has(self.jobCheck) || (!has(self.script) && !has(self.evaluateCEL))' type: array x-kubernetes-list-type: atomic reloader: @@ -1974,7 +2103,7 @@ spec: type: array featureID: description: |- - FeatureID is an indentifier of the feature (Helm/Kustomize/Resources) + FeatureID is an identifier of the feature (Helm/Kustomize/Resources) This field indicates when to run this check. For instance: - if set to Helm this check will be run after all helm @@ -1992,6 +2121,46 @@ spec: Group of the resource to fetch in the managed Cluster. Required when Kind is set. Leave empty for metric-only checks. type: string + jobCheck: + description: |- + JobCheck runs a Job in the managed cluster and uses its Complete/Failed + outcome as the check result. Mutually exclusive with Script and EvaluateCEL. + properties: + jobRef: + description: |- + JobRef references the Secret/ConfigMap containing the Job manifest to + deploy in the managed Cluster as this check. + properties: + kind: + description: 'Kind of the resource. Supported kinds + are: Secrets and ConfigMaps.' + enum: + - Secret + - ConfigMap + type: string + name: + description: Name of the referenced resource. + minLength: 1 + type: string + namespace: + description: |- + Namespace of the referenced resource. + Namespace can be left empty. In such a case, namespace will + be implicit set to cluster's namespace. + type: string + required: + - kind + - name + - namespace + type: object + timeout: + description: |- + Timeout is how long to wait for the Job to reach Complete or Failed + before treating the check as failed. + type: string + required: + - jobRef + type: object kind: description: |- Kind of the resource to fetch in the managed Cluster. @@ -2106,6 +2275,9 @@ spec: - featureID - name type: object + x-kubernetes-validations: + - message: jobCheck cannot be set together with script or evaluateCEL + rule: '!has(self.jobCheck) || (!has(self.script) && !has(self.evaluateCEL))' type: array x-kubernetes-list-type: atomic type: object diff --git a/lib/crd/clusterpromotions.go b/lib/crd/clusterpromotions.go index aee1b603..1f0dde5b 100644 --- a/lib/crd/clusterpromotions.go +++ b/lib/crd/clusterpromotions.go @@ -1211,7 +1211,7 @@ spec: type: array featureID: description: |- - FeatureID is an indentifier of the feature (Helm/Kustomize/Resources) + FeatureID is an identifier of the feature (Helm/Kustomize/Resources) This field indicates when to run this check. For instance: - if set to Helm this check will be run after all helm @@ -1229,6 +1229,46 @@ spec: Group of the resource to fetch in the managed Cluster. Required when Kind is set. Leave empty for metric-only checks. type: string + jobCheck: + description: |- + JobCheck runs a Job in the managed cluster and uses its Complete/Failed + outcome as the check result. Mutually exclusive with Script and EvaluateCEL. + properties: + jobRef: + description: |- + JobRef references the Secret/ConfigMap containing the Job manifest to + deploy in the managed Cluster as this check. + properties: + kind: + description: 'Kind of the resource. Supported kinds + are: Secrets and ConfigMaps.' + enum: + - Secret + - ConfigMap + type: string + name: + description: Name of the referenced resource. + minLength: 1 + type: string + namespace: + description: |- + Namespace of the referenced resource. + Namespace can be left empty. In such a case, namespace will + be implicit set to cluster's namespace. + type: string + required: + - kind + - name + - namespace + type: object + timeout: + description: |- + Timeout is how long to wait for the Job to reach Complete or Failed + before treating the check as failed. + type: string + required: + - jobRef + type: object kind: description: |- Kind of the resource to fetch in the managed Cluster. @@ -1343,6 +1383,9 @@ spec: - featureID - name type: object + x-kubernetes-validations: + - message: jobCheck cannot be set together with script or evaluateCEL + rule: '!has(self.jobCheck) || (!has(self.script) && !has(self.evaluateCEL))' type: array x-kubernetes-list-type: atomic preDeleteChecks: @@ -1382,7 +1425,7 @@ spec: type: array featureID: description: |- - FeatureID is an indentifier of the feature (Helm/Kustomize/Resources) + FeatureID is an identifier of the feature (Helm/Kustomize/Resources) This field indicates when to run this check. For instance: - if set to Helm this check will be run after all helm @@ -1400,6 +1443,46 @@ spec: Group of the resource to fetch in the managed Cluster. Required when Kind is set. Leave empty for metric-only checks. type: string + jobCheck: + description: |- + JobCheck runs a Job in the managed cluster and uses its Complete/Failed + outcome as the check result. Mutually exclusive with Script and EvaluateCEL. + properties: + jobRef: + description: |- + JobRef references the Secret/ConfigMap containing the Job manifest to + deploy in the managed Cluster as this check. + properties: + kind: + description: 'Kind of the resource. Supported kinds + are: Secrets and ConfigMaps.' + enum: + - Secret + - ConfigMap + type: string + name: + description: Name of the referenced resource. + minLength: 1 + type: string + namespace: + description: |- + Namespace of the referenced resource. + Namespace can be left empty. In such a case, namespace will + be implicit set to cluster's namespace. + type: string + required: + - kind + - name + - namespace + type: object + timeout: + description: |- + Timeout is how long to wait for the Job to reach Complete or Failed + before treating the check as failed. + type: string + required: + - jobRef + type: object kind: description: |- Kind of the resource to fetch in the managed Cluster. @@ -1514,6 +1597,9 @@ spec: - featureID - name type: object + x-kubernetes-validations: + - message: jobCheck cannot be set together with script or evaluateCEL + rule: '!has(self.jobCheck) || (!has(self.script) && !has(self.evaluateCEL))' type: array x-kubernetes-list-type: atomic preDeployChecks: @@ -1553,7 +1639,7 @@ spec: type: array featureID: description: |- - FeatureID is an indentifier of the feature (Helm/Kustomize/Resources) + FeatureID is an identifier of the feature (Helm/Kustomize/Resources) This field indicates when to run this check. For instance: - if set to Helm this check will be run after all helm @@ -1571,6 +1657,46 @@ spec: Group of the resource to fetch in the managed Cluster. Required when Kind is set. Leave empty for metric-only checks. type: string + jobCheck: + description: |- + JobCheck runs a Job in the managed cluster and uses its Complete/Failed + outcome as the check result. Mutually exclusive with Script and EvaluateCEL. + properties: + jobRef: + description: |- + JobRef references the Secret/ConfigMap containing the Job manifest to + deploy in the managed Cluster as this check. + properties: + kind: + description: 'Kind of the resource. Supported kinds + are: Secrets and ConfigMaps.' + enum: + - Secret + - ConfigMap + type: string + name: + description: Name of the referenced resource. + minLength: 1 + type: string + namespace: + description: |- + Namespace of the referenced resource. + Namespace can be left empty. In such a case, namespace will + be implicit set to cluster's namespace. + type: string + required: + - kind + - name + - namespace + type: object + timeout: + description: |- + Timeout is how long to wait for the Job to reach Complete or Failed + before treating the check as failed. + type: string + required: + - jobRef + type: object kind: description: |- Kind of the resource to fetch in the managed Cluster. @@ -1685,6 +1811,9 @@ spec: - featureID - name type: object + x-kubernetes-validations: + - message: jobCheck cannot be set together with script or evaluateCEL + rule: '!has(self.jobCheck) || (!has(self.script) && !has(self.evaluateCEL))' type: array x-kubernetes-list-type: atomic reloader: @@ -1872,7 +2001,7 @@ spec: type: array featureID: description: |- - FeatureID is an indentifier of the feature (Helm/Kustomize/Resources) + FeatureID is an identifier of the feature (Helm/Kustomize/Resources) This field indicates when to run this check. For instance: - if set to Helm this check will be run after all helm @@ -1890,6 +2019,46 @@ spec: Group of the resource to fetch in the managed Cluster. Required when Kind is set. Leave empty for metric-only checks. type: string + jobCheck: + description: |- + JobCheck runs a Job in the managed cluster and uses its Complete/Failed + outcome as the check result. Mutually exclusive with Script and EvaluateCEL. + properties: + jobRef: + description: |- + JobRef references the Secret/ConfigMap containing the Job manifest to + deploy in the managed Cluster as this check. + properties: + kind: + description: 'Kind of the resource. Supported kinds + are: Secrets and ConfigMaps.' + enum: + - Secret + - ConfigMap + type: string + name: + description: Name of the referenced resource. + minLength: 1 + type: string + namespace: + description: |- + Namespace of the referenced resource. + Namespace can be left empty. In such a case, namespace will + be implicit set to cluster's namespace. + type: string + required: + - kind + - name + - namespace + type: object + timeout: + description: |- + Timeout is how long to wait for the Job to reach Complete or Failed + before treating the check as failed. + type: string + required: + - jobRef + type: object kind: description: |- Kind of the resource to fetch in the managed Cluster. @@ -2004,6 +2173,9 @@ spec: - featureID - name type: object + x-kubernetes-validations: + - message: jobCheck cannot be set together with script or evaluateCEL + rule: '!has(self.jobCheck) || (!has(self.script) && !has(self.evaluateCEL))' type: array x-kubernetes-list-type: atomic type: object @@ -2114,7 +2286,7 @@ spec: type: array featureID: description: |- - FeatureID is an indentifier of the feature (Helm/Kustomize/Resources) + FeatureID is an identifier of the feature (Helm/Kustomize/Resources) This field indicates when to run this check. For instance: - if set to Helm this check will be run after all helm @@ -2132,6 +2304,46 @@ spec: Group of the resource to fetch in the managed Cluster. Required when Kind is set. Leave empty for metric-only checks. type: string + jobCheck: + description: |- + JobCheck runs a Job in the managed cluster and uses its Complete/Failed + outcome as the check result. Mutually exclusive with Script and EvaluateCEL. + properties: + jobRef: + description: |- + JobRef references the Secret/ConfigMap containing the Job manifest to + deploy in the managed Cluster as this check. + properties: + kind: + description: 'Kind of the resource. Supported + kinds are: Secrets and ConfigMaps.' + enum: + - Secret + - ConfigMap + type: string + name: + description: Name of the referenced resource. + minLength: 1 + type: string + namespace: + description: |- + Namespace of the referenced resource. + Namespace can be left empty. In such a case, namespace will + be implicit set to cluster's namespace. + type: string + required: + - kind + - name + - namespace + type: object + timeout: + description: |- + Timeout is how long to wait for the Job to reach Complete or Failed + before treating the check as failed. + type: string + required: + - jobRef + type: object kind: description: |- Kind of the resource to fetch in the managed Cluster. @@ -2249,6 +2461,11 @@ spec: - featureID - name type: object + x-kubernetes-validations: + - message: jobCheck cannot be set together with script + or evaluateCEL + rule: '!has(self.jobCheck) || (!has(self.script) + && !has(self.evaluateCEL))' type: array preHealthCheckDeployment: description: |- @@ -2484,7 +2701,7 @@ spec: type: array featureID: description: |- - FeatureID is an indentifier of the feature (Helm/Kustomize/Resources) + FeatureID is an identifier of the feature (Helm/Kustomize/Resources) This field indicates when to run this check. For instance: - if set to Helm this check will be run after all helm @@ -2502,6 +2719,46 @@ spec: Group of the resource to fetch in the managed Cluster. Required when Kind is set. Leave empty for metric-only checks. type: string + jobCheck: + description: |- + JobCheck runs a Job in the managed cluster and uses its Complete/Failed + outcome as the check result. Mutually exclusive with Script and EvaluateCEL. + properties: + jobRef: + description: |- + JobRef references the Secret/ConfigMap containing the Job manifest to + deploy in the managed Cluster as this check. + properties: + kind: + description: 'Kind of the resource. Supported + kinds are: Secrets and ConfigMaps.' + enum: + - Secret + - ConfigMap + type: string + name: + description: Name of the referenced resource. + minLength: 1 + type: string + namespace: + description: |- + Namespace of the referenced resource. + Namespace can be left empty. In such a case, namespace will + be implicit set to cluster's namespace. + type: string + required: + - kind + - name + - namespace + type: object + timeout: + description: |- + Timeout is how long to wait for the Job to reach Complete or Failed + before treating the check as failed. + type: string + required: + - jobRef + type: object kind: description: |- Kind of the resource to fetch in the managed Cluster. @@ -2619,6 +2876,11 @@ spec: - featureID - name type: object + x-kubernetes-validations: + - message: jobCheck cannot be set together with script + or evaluateCEL + rule: '!has(self.jobCheck) || (!has(self.script) + && !has(self.evaluateCEL))' type: array preHealthCheckDeployment: description: |- diff --git a/lib/crd/clustersummaries.go b/lib/crd/clustersummaries.go index 454892f8..4d23014b 100644 --- a/lib/crd/clustersummaries.go +++ b/lib/crd/clustersummaries.go @@ -1348,7 +1348,7 @@ spec: type: array featureID: description: |- - FeatureID is an indentifier of the feature (Helm/Kustomize/Resources) + FeatureID is an identifier of the feature (Helm/Kustomize/Resources) This field indicates when to run this check. For instance: - if set to Helm this check will be run after all helm @@ -1366,6 +1366,46 @@ spec: Group of the resource to fetch in the managed Cluster. Required when Kind is set. Leave empty for metric-only checks. type: string + jobCheck: + description: |- + JobCheck runs a Job in the managed cluster and uses its Complete/Failed + outcome as the check result. Mutually exclusive with Script and EvaluateCEL. + properties: + jobRef: + description: |- + JobRef references the Secret/ConfigMap containing the Job manifest to + deploy in the managed Cluster as this check. + properties: + kind: + description: 'Kind of the resource. Supported kinds + are: Secrets and ConfigMaps.' + enum: + - Secret + - ConfigMap + type: string + name: + description: Name of the referenced resource. + minLength: 1 + type: string + namespace: + description: |- + Namespace of the referenced resource. + Namespace can be left empty. In such a case, namespace will + be implicit set to cluster's namespace. + type: string + required: + - kind + - name + - namespace + type: object + timeout: + description: |- + Timeout is how long to wait for the Job to reach Complete or Failed + before treating the check as failed. + type: string + required: + - jobRef + type: object kind: description: |- Kind of the resource to fetch in the managed Cluster. @@ -1480,6 +1520,9 @@ spec: - featureID - name type: object + x-kubernetes-validations: + - message: jobCheck cannot be set together with script or evaluateCEL + rule: '!has(self.jobCheck) || (!has(self.script) && !has(self.evaluateCEL))' type: array x-kubernetes-list-type: atomic preDeleteChecks: @@ -1519,7 +1562,7 @@ spec: type: array featureID: description: |- - FeatureID is an indentifier of the feature (Helm/Kustomize/Resources) + FeatureID is an identifier of the feature (Helm/Kustomize/Resources) This field indicates when to run this check. For instance: - if set to Helm this check will be run after all helm @@ -1537,6 +1580,46 @@ spec: Group of the resource to fetch in the managed Cluster. Required when Kind is set. Leave empty for metric-only checks. type: string + jobCheck: + description: |- + JobCheck runs a Job in the managed cluster and uses its Complete/Failed + outcome as the check result. Mutually exclusive with Script and EvaluateCEL. + properties: + jobRef: + description: |- + JobRef references the Secret/ConfigMap containing the Job manifest to + deploy in the managed Cluster as this check. + properties: + kind: + description: 'Kind of the resource. Supported kinds + are: Secrets and ConfigMaps.' + enum: + - Secret + - ConfigMap + type: string + name: + description: Name of the referenced resource. + minLength: 1 + type: string + namespace: + description: |- + Namespace of the referenced resource. + Namespace can be left empty. In such a case, namespace will + be implicit set to cluster's namespace. + type: string + required: + - kind + - name + - namespace + type: object + timeout: + description: |- + Timeout is how long to wait for the Job to reach Complete or Failed + before treating the check as failed. + type: string + required: + - jobRef + type: object kind: description: |- Kind of the resource to fetch in the managed Cluster. @@ -1651,6 +1734,9 @@ spec: - featureID - name type: object + x-kubernetes-validations: + - message: jobCheck cannot be set together with script or evaluateCEL + rule: '!has(self.jobCheck) || (!has(self.script) && !has(self.evaluateCEL))' type: array x-kubernetes-list-type: atomic preDeployChecks: @@ -1690,7 +1776,7 @@ spec: type: array featureID: description: |- - FeatureID is an indentifier of the feature (Helm/Kustomize/Resources) + FeatureID is an identifier of the feature (Helm/Kustomize/Resources) This field indicates when to run this check. For instance: - if set to Helm this check will be run after all helm @@ -1708,6 +1794,46 @@ spec: Group of the resource to fetch in the managed Cluster. Required when Kind is set. Leave empty for metric-only checks. type: string + jobCheck: + description: |- + JobCheck runs a Job in the managed cluster and uses its Complete/Failed + outcome as the check result. Mutually exclusive with Script and EvaluateCEL. + properties: + jobRef: + description: |- + JobRef references the Secret/ConfigMap containing the Job manifest to + deploy in the managed Cluster as this check. + properties: + kind: + description: 'Kind of the resource. Supported kinds + are: Secrets and ConfigMaps.' + enum: + - Secret + - ConfigMap + type: string + name: + description: Name of the referenced resource. + minLength: 1 + type: string + namespace: + description: |- + Namespace of the referenced resource. + Namespace can be left empty. In such a case, namespace will + be implicit set to cluster's namespace. + type: string + required: + - kind + - name + - namespace + type: object + timeout: + description: |- + Timeout is how long to wait for the Job to reach Complete or Failed + before treating the check as failed. + type: string + required: + - jobRef + type: object kind: description: |- Kind of the resource to fetch in the managed Cluster. @@ -1822,6 +1948,9 @@ spec: - featureID - name type: object + x-kubernetes-validations: + - message: jobCheck cannot be set together with script or evaluateCEL + rule: '!has(self.jobCheck) || (!has(self.script) && !has(self.evaluateCEL))' type: array x-kubernetes-list-type: atomic reloader: @@ -2017,7 +2146,7 @@ spec: type: array featureID: description: |- - FeatureID is an indentifier of the feature (Helm/Kustomize/Resources) + FeatureID is an identifier of the feature (Helm/Kustomize/Resources) This field indicates when to run this check. For instance: - if set to Helm this check will be run after all helm @@ -2035,6 +2164,46 @@ spec: Group of the resource to fetch in the managed Cluster. Required when Kind is set. Leave empty for metric-only checks. type: string + jobCheck: + description: |- + JobCheck runs a Job in the managed cluster and uses its Complete/Failed + outcome as the check result. Mutually exclusive with Script and EvaluateCEL. + properties: + jobRef: + description: |- + JobRef references the Secret/ConfigMap containing the Job manifest to + deploy in the managed Cluster as this check. + properties: + kind: + description: 'Kind of the resource. Supported kinds + are: Secrets and ConfigMaps.' + enum: + - Secret + - ConfigMap + type: string + name: + description: Name of the referenced resource. + minLength: 1 + type: string + namespace: + description: |- + Namespace of the referenced resource. + Namespace can be left empty. In such a case, namespace will + be implicit set to cluster's namespace. + type: string + required: + - kind + - name + - namespace + type: object + timeout: + description: |- + Timeout is how long to wait for the Job to reach Complete or Failed + before treating the check as failed. + type: string + required: + - jobRef + type: object kind: description: |- Kind of the resource to fetch in the managed Cluster. @@ -2149,6 +2318,9 @@ spec: - featureID - name type: object + x-kubernetes-validations: + - message: jobCheck cannot be set together with script or evaluateCEL + rule: '!has(self.jobCheck) || (!has(self.script) && !has(self.evaluateCEL))' type: array x-kubernetes-list-type: atomic type: object @@ -2183,7 +2355,7 @@ spec: type: string type: array featureID: - description: FeatureID is an indentifier of the feature whose + description: FeatureID is an identifier of the feature whose status is reported enum: - Resources diff --git a/lib/crd/profiles.go b/lib/crd/profiles.go index 20e3676d..9d9eb2f9 100644 --- a/lib/crd/profiles.go +++ b/lib/crd/profiles.go @@ -1308,7 +1308,7 @@ spec: type: array featureID: description: |- - FeatureID is an indentifier of the feature (Helm/Kustomize/Resources) + FeatureID is an identifier of the feature (Helm/Kustomize/Resources) This field indicates when to run this check. For instance: - if set to Helm this check will be run after all helm @@ -1326,6 +1326,46 @@ spec: Group of the resource to fetch in the managed Cluster. Required when Kind is set. Leave empty for metric-only checks. type: string + jobCheck: + description: |- + JobCheck runs a Job in the managed cluster and uses its Complete/Failed + outcome as the check result. Mutually exclusive with Script and EvaluateCEL. + properties: + jobRef: + description: |- + JobRef references the Secret/ConfigMap containing the Job manifest to + deploy in the managed Cluster as this check. + properties: + kind: + description: 'Kind of the resource. Supported kinds + are: Secrets and ConfigMaps.' + enum: + - Secret + - ConfigMap + type: string + name: + description: Name of the referenced resource. + minLength: 1 + type: string + namespace: + description: |- + Namespace of the referenced resource. + Namespace can be left empty. In such a case, namespace will + be implicit set to cluster's namespace. + type: string + required: + - kind + - name + - namespace + type: object + timeout: + description: |- + Timeout is how long to wait for the Job to reach Complete or Failed + before treating the check as failed. + type: string + required: + - jobRef + type: object kind: description: |- Kind of the resource to fetch in the managed Cluster. @@ -1440,6 +1480,9 @@ spec: - featureID - name type: object + x-kubernetes-validations: + - message: jobCheck cannot be set together with script or evaluateCEL + rule: '!has(self.jobCheck) || (!has(self.script) && !has(self.evaluateCEL))' type: array x-kubernetes-list-type: atomic preDeleteChecks: @@ -1478,7 +1521,7 @@ spec: type: array featureID: description: |- - FeatureID is an indentifier of the feature (Helm/Kustomize/Resources) + FeatureID is an identifier of the feature (Helm/Kustomize/Resources) This field indicates when to run this check. For instance: - if set to Helm this check will be run after all helm @@ -1496,6 +1539,46 @@ spec: Group of the resource to fetch in the managed Cluster. Required when Kind is set. Leave empty for metric-only checks. type: string + jobCheck: + description: |- + JobCheck runs a Job in the managed cluster and uses its Complete/Failed + outcome as the check result. Mutually exclusive with Script and EvaluateCEL. + properties: + jobRef: + description: |- + JobRef references the Secret/ConfigMap containing the Job manifest to + deploy in the managed Cluster as this check. + properties: + kind: + description: 'Kind of the resource. Supported kinds + are: Secrets and ConfigMaps.' + enum: + - Secret + - ConfigMap + type: string + name: + description: Name of the referenced resource. + minLength: 1 + type: string + namespace: + description: |- + Namespace of the referenced resource. + Namespace can be left empty. In such a case, namespace will + be implicit set to cluster's namespace. + type: string + required: + - kind + - name + - namespace + type: object + timeout: + description: |- + Timeout is how long to wait for the Job to reach Complete or Failed + before treating the check as failed. + type: string + required: + - jobRef + type: object kind: description: |- Kind of the resource to fetch in the managed Cluster. @@ -1610,6 +1693,9 @@ spec: - featureID - name type: object + x-kubernetes-validations: + - message: jobCheck cannot be set together with script or evaluateCEL + rule: '!has(self.jobCheck) || (!has(self.script) && !has(self.evaluateCEL))' type: array x-kubernetes-list-type: atomic preDeployChecks: @@ -1648,7 +1734,7 @@ spec: type: array featureID: description: |- - FeatureID is an indentifier of the feature (Helm/Kustomize/Resources) + FeatureID is an identifier of the feature (Helm/Kustomize/Resources) This field indicates when to run this check. For instance: - if set to Helm this check will be run after all helm @@ -1666,6 +1752,46 @@ spec: Group of the resource to fetch in the managed Cluster. Required when Kind is set. Leave empty for metric-only checks. type: string + jobCheck: + description: |- + JobCheck runs a Job in the managed cluster and uses its Complete/Failed + outcome as the check result. Mutually exclusive with Script and EvaluateCEL. + properties: + jobRef: + description: |- + JobRef references the Secret/ConfigMap containing the Job manifest to + deploy in the managed Cluster as this check. + properties: + kind: + description: 'Kind of the resource. Supported kinds + are: Secrets and ConfigMaps.' + enum: + - Secret + - ConfigMap + type: string + name: + description: Name of the referenced resource. + minLength: 1 + type: string + namespace: + description: |- + Namespace of the referenced resource. + Namespace can be left empty. In such a case, namespace will + be implicit set to cluster's namespace. + type: string + required: + - kind + - name + - namespace + type: object + timeout: + description: |- + Timeout is how long to wait for the Job to reach Complete or Failed + before treating the check as failed. + type: string + required: + - jobRef + type: object kind: description: |- Kind of the resource to fetch in the managed Cluster. @@ -1780,6 +1906,9 @@ spec: - featureID - name type: object + x-kubernetes-validations: + - message: jobCheck cannot be set together with script or evaluateCEL + rule: '!has(self.jobCheck) || (!has(self.script) && !has(self.evaluateCEL))' type: array x-kubernetes-list-type: atomic reloader: @@ -1974,7 +2103,7 @@ spec: type: array featureID: description: |- - FeatureID is an indentifier of the feature (Helm/Kustomize/Resources) + FeatureID is an identifier of the feature (Helm/Kustomize/Resources) This field indicates when to run this check. For instance: - if set to Helm this check will be run after all helm @@ -1992,6 +2121,46 @@ spec: Group of the resource to fetch in the managed Cluster. Required when Kind is set. Leave empty for metric-only checks. type: string + jobCheck: + description: |- + JobCheck runs a Job in the managed cluster and uses its Complete/Failed + outcome as the check result. Mutually exclusive with Script and EvaluateCEL. + properties: + jobRef: + description: |- + JobRef references the Secret/ConfigMap containing the Job manifest to + deploy in the managed Cluster as this check. + properties: + kind: + description: 'Kind of the resource. Supported kinds + are: Secrets and ConfigMaps.' + enum: + - Secret + - ConfigMap + type: string + name: + description: Name of the referenced resource. + minLength: 1 + type: string + namespace: + description: |- + Namespace of the referenced resource. + Namespace can be left empty. In such a case, namespace will + be implicit set to cluster's namespace. + type: string + required: + - kind + - name + - namespace + type: object + timeout: + description: |- + Timeout is how long to wait for the Job to reach Complete or Failed + before treating the check as failed. + type: string + required: + - jobRef + type: object kind: description: |- Kind of the resource to fetch in the managed Cluster. @@ -2106,6 +2275,9 @@ spec: - featureID - name type: object + x-kubernetes-validations: + - message: jobCheck cannot be set together with script or evaluateCEL + rule: '!has(self.jobCheck) || (!has(self.script) && !has(self.evaluateCEL))' type: array x-kubernetes-list-type: atomic type: object diff --git a/manifest/manifest.yaml b/manifest/manifest.yaml index 4aa37798..eb5c4d2e 100644 --- a/manifest/manifest.yaml +++ b/manifest/manifest.yaml @@ -1598,7 +1598,7 @@ spec: type: array featureID: description: |- - FeatureID is an indentifier of the feature (Helm/Kustomize/Resources) + FeatureID is an identifier of the feature (Helm/Kustomize/Resources) This field indicates when to run this check. For instance: - if set to Helm this check will be run after all helm @@ -1616,6 +1616,46 @@ spec: Group of the resource to fetch in the managed Cluster. Required when Kind is set. Leave empty for metric-only checks. type: string + jobCheck: + description: |- + JobCheck runs a Job in the managed cluster and uses its Complete/Failed + outcome as the check result. Mutually exclusive with Script and EvaluateCEL. + properties: + jobRef: + description: |- + JobRef references the Secret/ConfigMap containing the Job manifest to + deploy in the managed Cluster as this check. + properties: + kind: + description: 'Kind of the resource. Supported kinds + are: Secrets and ConfigMaps.' + enum: + - Secret + - ConfigMap + type: string + name: + description: Name of the referenced resource. + minLength: 1 + type: string + namespace: + description: |- + Namespace of the referenced resource. + Namespace can be left empty. In such a case, namespace will + be implicit set to cluster's namespace. + type: string + required: + - kind + - name + - namespace + type: object + timeout: + description: |- + Timeout is how long to wait for the Job to reach Complete or Failed + before treating the check as failed. + type: string + required: + - jobRef + type: object kind: description: |- Kind of the resource to fetch in the managed Cluster. @@ -1730,6 +1770,9 @@ spec: - featureID - name type: object + x-kubernetes-validations: + - message: jobCheck cannot be set together with script or evaluateCEL + rule: '!has(self.jobCheck) || (!has(self.script) && !has(self.evaluateCEL))' type: array x-kubernetes-list-type: atomic preDeleteChecks: @@ -1768,7 +1811,7 @@ spec: type: array featureID: description: |- - FeatureID is an indentifier of the feature (Helm/Kustomize/Resources) + FeatureID is an identifier of the feature (Helm/Kustomize/Resources) This field indicates when to run this check. For instance: - if set to Helm this check will be run after all helm @@ -1786,6 +1829,46 @@ spec: Group of the resource to fetch in the managed Cluster. Required when Kind is set. Leave empty for metric-only checks. type: string + jobCheck: + description: |- + JobCheck runs a Job in the managed cluster and uses its Complete/Failed + outcome as the check result. Mutually exclusive with Script and EvaluateCEL. + properties: + jobRef: + description: |- + JobRef references the Secret/ConfigMap containing the Job manifest to + deploy in the managed Cluster as this check. + properties: + kind: + description: 'Kind of the resource. Supported kinds + are: Secrets and ConfigMaps.' + enum: + - Secret + - ConfigMap + type: string + name: + description: Name of the referenced resource. + minLength: 1 + type: string + namespace: + description: |- + Namespace of the referenced resource. + Namespace can be left empty. In such a case, namespace will + be implicit set to cluster's namespace. + type: string + required: + - kind + - name + - namespace + type: object + timeout: + description: |- + Timeout is how long to wait for the Job to reach Complete or Failed + before treating the check as failed. + type: string + required: + - jobRef + type: object kind: description: |- Kind of the resource to fetch in the managed Cluster. @@ -1900,6 +1983,9 @@ spec: - featureID - name type: object + x-kubernetes-validations: + - message: jobCheck cannot be set together with script or evaluateCEL + rule: '!has(self.jobCheck) || (!has(self.script) && !has(self.evaluateCEL))' type: array x-kubernetes-list-type: atomic preDeployChecks: @@ -1938,7 +2024,7 @@ spec: type: array featureID: description: |- - FeatureID is an indentifier of the feature (Helm/Kustomize/Resources) + FeatureID is an identifier of the feature (Helm/Kustomize/Resources) This field indicates when to run this check. For instance: - if set to Helm this check will be run after all helm @@ -1956,6 +2042,46 @@ spec: Group of the resource to fetch in the managed Cluster. Required when Kind is set. Leave empty for metric-only checks. type: string + jobCheck: + description: |- + JobCheck runs a Job in the managed cluster and uses its Complete/Failed + outcome as the check result. Mutually exclusive with Script and EvaluateCEL. + properties: + jobRef: + description: |- + JobRef references the Secret/ConfigMap containing the Job manifest to + deploy in the managed Cluster as this check. + properties: + kind: + description: 'Kind of the resource. Supported kinds + are: Secrets and ConfigMaps.' + enum: + - Secret + - ConfigMap + type: string + name: + description: Name of the referenced resource. + minLength: 1 + type: string + namespace: + description: |- + Namespace of the referenced resource. + Namespace can be left empty. In such a case, namespace will + be implicit set to cluster's namespace. + type: string + required: + - kind + - name + - namespace + type: object + timeout: + description: |- + Timeout is how long to wait for the Job to reach Complete or Failed + before treating the check as failed. + type: string + required: + - jobRef + type: object kind: description: |- Kind of the resource to fetch in the managed Cluster. @@ -2070,6 +2196,9 @@ spec: - featureID - name type: object + x-kubernetes-validations: + - message: jobCheck cannot be set together with script or evaluateCEL + rule: '!has(self.jobCheck) || (!has(self.script) && !has(self.evaluateCEL))' type: array x-kubernetes-list-type: atomic reloader: @@ -2264,7 +2393,7 @@ spec: type: array featureID: description: |- - FeatureID is an indentifier of the feature (Helm/Kustomize/Resources) + FeatureID is an identifier of the feature (Helm/Kustomize/Resources) This field indicates when to run this check. For instance: - if set to Helm this check will be run after all helm @@ -2282,6 +2411,46 @@ spec: Group of the resource to fetch in the managed Cluster. Required when Kind is set. Leave empty for metric-only checks. type: string + jobCheck: + description: |- + JobCheck runs a Job in the managed cluster and uses its Complete/Failed + outcome as the check result. Mutually exclusive with Script and EvaluateCEL. + properties: + jobRef: + description: |- + JobRef references the Secret/ConfigMap containing the Job manifest to + deploy in the managed Cluster as this check. + properties: + kind: + description: 'Kind of the resource. Supported kinds + are: Secrets and ConfigMaps.' + enum: + - Secret + - ConfigMap + type: string + name: + description: Name of the referenced resource. + minLength: 1 + type: string + namespace: + description: |- + Namespace of the referenced resource. + Namespace can be left empty. In such a case, namespace will + be implicit set to cluster's namespace. + type: string + required: + - kind + - name + - namespace + type: object + timeout: + description: |- + Timeout is how long to wait for the Job to reach Complete or Failed + before treating the check as failed. + type: string + required: + - jobRef + type: object kind: description: |- Kind of the resource to fetch in the managed Cluster. @@ -2396,6 +2565,9 @@ spec: - featureID - name type: object + x-kubernetes-validations: + - message: jobCheck cannot be set together with script or evaluateCEL + rule: '!has(self.jobCheck) || (!has(self.script) && !has(self.evaluateCEL))' type: array x-kubernetes-list-type: atomic type: object @@ -3859,7 +4031,7 @@ spec: type: array featureID: description: |- - FeatureID is an indentifier of the feature (Helm/Kustomize/Resources) + FeatureID is an identifier of the feature (Helm/Kustomize/Resources) This field indicates when to run this check. For instance: - if set to Helm this check will be run after all helm @@ -3877,6 +4049,46 @@ spec: Group of the resource to fetch in the managed Cluster. Required when Kind is set. Leave empty for metric-only checks. type: string + jobCheck: + description: |- + JobCheck runs a Job in the managed cluster and uses its Complete/Failed + outcome as the check result. Mutually exclusive with Script and EvaluateCEL. + properties: + jobRef: + description: |- + JobRef references the Secret/ConfigMap containing the Job manifest to + deploy in the managed Cluster as this check. + properties: + kind: + description: 'Kind of the resource. Supported kinds + are: Secrets and ConfigMaps.' + enum: + - Secret + - ConfigMap + type: string + name: + description: Name of the referenced resource. + minLength: 1 + type: string + namespace: + description: |- + Namespace of the referenced resource. + Namespace can be left empty. In such a case, namespace will + be implicit set to cluster's namespace. + type: string + required: + - kind + - name + - namespace + type: object + timeout: + description: |- + Timeout is how long to wait for the Job to reach Complete or Failed + before treating the check as failed. + type: string + required: + - jobRef + type: object kind: description: |- Kind of the resource to fetch in the managed Cluster. @@ -3991,6 +4203,9 @@ spec: - featureID - name type: object + x-kubernetes-validations: + - message: jobCheck cannot be set together with script or evaluateCEL + rule: '!has(self.jobCheck) || (!has(self.script) && !has(self.evaluateCEL))' type: array x-kubernetes-list-type: atomic preDeleteChecks: @@ -4030,7 +4245,7 @@ spec: type: array featureID: description: |- - FeatureID is an indentifier of the feature (Helm/Kustomize/Resources) + FeatureID is an identifier of the feature (Helm/Kustomize/Resources) This field indicates when to run this check. For instance: - if set to Helm this check will be run after all helm @@ -4048,6 +4263,46 @@ spec: Group of the resource to fetch in the managed Cluster. Required when Kind is set. Leave empty for metric-only checks. type: string + jobCheck: + description: |- + JobCheck runs a Job in the managed cluster and uses its Complete/Failed + outcome as the check result. Mutually exclusive with Script and EvaluateCEL. + properties: + jobRef: + description: |- + JobRef references the Secret/ConfigMap containing the Job manifest to + deploy in the managed Cluster as this check. + properties: + kind: + description: 'Kind of the resource. Supported kinds + are: Secrets and ConfigMaps.' + enum: + - Secret + - ConfigMap + type: string + name: + description: Name of the referenced resource. + minLength: 1 + type: string + namespace: + description: |- + Namespace of the referenced resource. + Namespace can be left empty. In such a case, namespace will + be implicit set to cluster's namespace. + type: string + required: + - kind + - name + - namespace + type: object + timeout: + description: |- + Timeout is how long to wait for the Job to reach Complete or Failed + before treating the check as failed. + type: string + required: + - jobRef + type: object kind: description: |- Kind of the resource to fetch in the managed Cluster. @@ -4162,6 +4417,9 @@ spec: - featureID - name type: object + x-kubernetes-validations: + - message: jobCheck cannot be set together with script or evaluateCEL + rule: '!has(self.jobCheck) || (!has(self.script) && !has(self.evaluateCEL))' type: array x-kubernetes-list-type: atomic preDeployChecks: @@ -4201,7 +4459,7 @@ spec: type: array featureID: description: |- - FeatureID is an indentifier of the feature (Helm/Kustomize/Resources) + FeatureID is an identifier of the feature (Helm/Kustomize/Resources) This field indicates when to run this check. For instance: - if set to Helm this check will be run after all helm @@ -4219,6 +4477,46 @@ spec: Group of the resource to fetch in the managed Cluster. Required when Kind is set. Leave empty for metric-only checks. type: string + jobCheck: + description: |- + JobCheck runs a Job in the managed cluster and uses its Complete/Failed + outcome as the check result. Mutually exclusive with Script and EvaluateCEL. + properties: + jobRef: + description: |- + JobRef references the Secret/ConfigMap containing the Job manifest to + deploy in the managed Cluster as this check. + properties: + kind: + description: 'Kind of the resource. Supported kinds + are: Secrets and ConfigMaps.' + enum: + - Secret + - ConfigMap + type: string + name: + description: Name of the referenced resource. + minLength: 1 + type: string + namespace: + description: |- + Namespace of the referenced resource. + Namespace can be left empty. In such a case, namespace will + be implicit set to cluster's namespace. + type: string + required: + - kind + - name + - namespace + type: object + timeout: + description: |- + Timeout is how long to wait for the Job to reach Complete or Failed + before treating the check as failed. + type: string + required: + - jobRef + type: object kind: description: |- Kind of the resource to fetch in the managed Cluster. @@ -4333,6 +4631,9 @@ spec: - featureID - name type: object + x-kubernetes-validations: + - message: jobCheck cannot be set together with script or evaluateCEL + rule: '!has(self.jobCheck) || (!has(self.script) && !has(self.evaluateCEL))' type: array x-kubernetes-list-type: atomic reloader: @@ -4520,7 +4821,7 @@ spec: type: array featureID: description: |- - FeatureID is an indentifier of the feature (Helm/Kustomize/Resources) + FeatureID is an identifier of the feature (Helm/Kustomize/Resources) This field indicates when to run this check. For instance: - if set to Helm this check will be run after all helm @@ -4538,6 +4839,46 @@ spec: Group of the resource to fetch in the managed Cluster. Required when Kind is set. Leave empty for metric-only checks. type: string + jobCheck: + description: |- + JobCheck runs a Job in the managed cluster and uses its Complete/Failed + outcome as the check result. Mutually exclusive with Script and EvaluateCEL. + properties: + jobRef: + description: |- + JobRef references the Secret/ConfigMap containing the Job manifest to + deploy in the managed Cluster as this check. + properties: + kind: + description: 'Kind of the resource. Supported kinds + are: Secrets and ConfigMaps.' + enum: + - Secret + - ConfigMap + type: string + name: + description: Name of the referenced resource. + minLength: 1 + type: string + namespace: + description: |- + Namespace of the referenced resource. + Namespace can be left empty. In such a case, namespace will + be implicit set to cluster's namespace. + type: string + required: + - kind + - name + - namespace + type: object + timeout: + description: |- + Timeout is how long to wait for the Job to reach Complete or Failed + before treating the check as failed. + type: string + required: + - jobRef + type: object kind: description: |- Kind of the resource to fetch in the managed Cluster. @@ -4652,6 +4993,9 @@ spec: - featureID - name type: object + x-kubernetes-validations: + - message: jobCheck cannot be set together with script or evaluateCEL + rule: '!has(self.jobCheck) || (!has(self.script) && !has(self.evaluateCEL))' type: array x-kubernetes-list-type: atomic type: object @@ -4762,7 +5106,7 @@ spec: type: array featureID: description: |- - FeatureID is an indentifier of the feature (Helm/Kustomize/Resources) + FeatureID is an identifier of the feature (Helm/Kustomize/Resources) This field indicates when to run this check. For instance: - if set to Helm this check will be run after all helm @@ -4780,6 +5124,46 @@ spec: Group of the resource to fetch in the managed Cluster. Required when Kind is set. Leave empty for metric-only checks. type: string + jobCheck: + description: |- + JobCheck runs a Job in the managed cluster and uses its Complete/Failed + outcome as the check result. Mutually exclusive with Script and EvaluateCEL. + properties: + jobRef: + description: |- + JobRef references the Secret/ConfigMap containing the Job manifest to + deploy in the managed Cluster as this check. + properties: + kind: + description: 'Kind of the resource. Supported + kinds are: Secrets and ConfigMaps.' + enum: + - Secret + - ConfigMap + type: string + name: + description: Name of the referenced resource. + minLength: 1 + type: string + namespace: + description: |- + Namespace of the referenced resource. + Namespace can be left empty. In such a case, namespace will + be implicit set to cluster's namespace. + type: string + required: + - kind + - name + - namespace + type: object + timeout: + description: |- + Timeout is how long to wait for the Job to reach Complete or Failed + before treating the check as failed. + type: string + required: + - jobRef + type: object kind: description: |- Kind of the resource to fetch in the managed Cluster. @@ -4897,6 +5281,11 @@ spec: - featureID - name type: object + x-kubernetes-validations: + - message: jobCheck cannot be set together with script + or evaluateCEL + rule: '!has(self.jobCheck) || (!has(self.script) + && !has(self.evaluateCEL))' type: array preHealthCheckDeployment: description: |- @@ -5132,7 +5521,7 @@ spec: type: array featureID: description: |- - FeatureID is an indentifier of the feature (Helm/Kustomize/Resources) + FeatureID is an identifier of the feature (Helm/Kustomize/Resources) This field indicates when to run this check. For instance: - if set to Helm this check will be run after all helm @@ -5150,6 +5539,46 @@ spec: Group of the resource to fetch in the managed Cluster. Required when Kind is set. Leave empty for metric-only checks. type: string + jobCheck: + description: |- + JobCheck runs a Job in the managed cluster and uses its Complete/Failed + outcome as the check result. Mutually exclusive with Script and EvaluateCEL. + properties: + jobRef: + description: |- + JobRef references the Secret/ConfigMap containing the Job manifest to + deploy in the managed Cluster as this check. + properties: + kind: + description: 'Kind of the resource. Supported + kinds are: Secrets and ConfigMaps.' + enum: + - Secret + - ConfigMap + type: string + name: + description: Name of the referenced resource. + minLength: 1 + type: string + namespace: + description: |- + Namespace of the referenced resource. + Namespace can be left empty. In such a case, namespace will + be implicit set to cluster's namespace. + type: string + required: + - kind + - name + - namespace + type: object + timeout: + description: |- + Timeout is how long to wait for the Job to reach Complete or Failed + before treating the check as failed. + type: string + required: + - jobRef + type: object kind: description: |- Kind of the resource to fetch in the managed Cluster. @@ -5267,6 +5696,11 @@ spec: - featureID - name type: object + x-kubernetes-validations: + - message: jobCheck cannot be set together with script + or evaluateCEL + rule: '!has(self.jobCheck) || (!has(self.script) + && !has(self.evaluateCEL))' type: array preHealthCheckDeployment: description: |- @@ -7156,7 +7590,7 @@ spec: type: array featureID: description: |- - FeatureID is an indentifier of the feature (Helm/Kustomize/Resources) + FeatureID is an identifier of the feature (Helm/Kustomize/Resources) This field indicates when to run this check. For instance: - if set to Helm this check will be run after all helm @@ -7174,6 +7608,46 @@ spec: Group of the resource to fetch in the managed Cluster. Required when Kind is set. Leave empty for metric-only checks. type: string + jobCheck: + description: |- + JobCheck runs a Job in the managed cluster and uses its Complete/Failed + outcome as the check result. Mutually exclusive with Script and EvaluateCEL. + properties: + jobRef: + description: |- + JobRef references the Secret/ConfigMap containing the Job manifest to + deploy in the managed Cluster as this check. + properties: + kind: + description: 'Kind of the resource. Supported kinds + are: Secrets and ConfigMaps.' + enum: + - Secret + - ConfigMap + type: string + name: + description: Name of the referenced resource. + minLength: 1 + type: string + namespace: + description: |- + Namespace of the referenced resource. + Namespace can be left empty. In such a case, namespace will + be implicit set to cluster's namespace. + type: string + required: + - kind + - name + - namespace + type: object + timeout: + description: |- + Timeout is how long to wait for the Job to reach Complete or Failed + before treating the check as failed. + type: string + required: + - jobRef + type: object kind: description: |- Kind of the resource to fetch in the managed Cluster. @@ -7288,6 +7762,9 @@ spec: - featureID - name type: object + x-kubernetes-validations: + - message: jobCheck cannot be set together with script or evaluateCEL + rule: '!has(self.jobCheck) || (!has(self.script) && !has(self.evaluateCEL))' type: array x-kubernetes-list-type: atomic preDeleteChecks: @@ -7327,7 +7804,7 @@ spec: type: array featureID: description: |- - FeatureID is an indentifier of the feature (Helm/Kustomize/Resources) + FeatureID is an identifier of the feature (Helm/Kustomize/Resources) This field indicates when to run this check. For instance: - if set to Helm this check will be run after all helm @@ -7345,6 +7822,46 @@ spec: Group of the resource to fetch in the managed Cluster. Required when Kind is set. Leave empty for metric-only checks. type: string + jobCheck: + description: |- + JobCheck runs a Job in the managed cluster and uses its Complete/Failed + outcome as the check result. Mutually exclusive with Script and EvaluateCEL. + properties: + jobRef: + description: |- + JobRef references the Secret/ConfigMap containing the Job manifest to + deploy in the managed Cluster as this check. + properties: + kind: + description: 'Kind of the resource. Supported kinds + are: Secrets and ConfigMaps.' + enum: + - Secret + - ConfigMap + type: string + name: + description: Name of the referenced resource. + minLength: 1 + type: string + namespace: + description: |- + Namespace of the referenced resource. + Namespace can be left empty. In such a case, namespace will + be implicit set to cluster's namespace. + type: string + required: + - kind + - name + - namespace + type: object + timeout: + description: |- + Timeout is how long to wait for the Job to reach Complete or Failed + before treating the check as failed. + type: string + required: + - jobRef + type: object kind: description: |- Kind of the resource to fetch in the managed Cluster. @@ -7459,6 +7976,9 @@ spec: - featureID - name type: object + x-kubernetes-validations: + - message: jobCheck cannot be set together with script or evaluateCEL + rule: '!has(self.jobCheck) || (!has(self.script) && !has(self.evaluateCEL))' type: array x-kubernetes-list-type: atomic preDeployChecks: @@ -7498,7 +8018,7 @@ spec: type: array featureID: description: |- - FeatureID is an indentifier of the feature (Helm/Kustomize/Resources) + FeatureID is an identifier of the feature (Helm/Kustomize/Resources) This field indicates when to run this check. For instance: - if set to Helm this check will be run after all helm @@ -7516,6 +8036,46 @@ spec: Group of the resource to fetch in the managed Cluster. Required when Kind is set. Leave empty for metric-only checks. type: string + jobCheck: + description: |- + JobCheck runs a Job in the managed cluster and uses its Complete/Failed + outcome as the check result. Mutually exclusive with Script and EvaluateCEL. + properties: + jobRef: + description: |- + JobRef references the Secret/ConfigMap containing the Job manifest to + deploy in the managed Cluster as this check. + properties: + kind: + description: 'Kind of the resource. Supported kinds + are: Secrets and ConfigMaps.' + enum: + - Secret + - ConfigMap + type: string + name: + description: Name of the referenced resource. + minLength: 1 + type: string + namespace: + description: |- + Namespace of the referenced resource. + Namespace can be left empty. In such a case, namespace will + be implicit set to cluster's namespace. + type: string + required: + - kind + - name + - namespace + type: object + timeout: + description: |- + Timeout is how long to wait for the Job to reach Complete or Failed + before treating the check as failed. + type: string + required: + - jobRef + type: object kind: description: |- Kind of the resource to fetch in the managed Cluster. @@ -7630,6 +8190,9 @@ spec: - featureID - name type: object + x-kubernetes-validations: + - message: jobCheck cannot be set together with script or evaluateCEL + rule: '!has(self.jobCheck) || (!has(self.script) && !has(self.evaluateCEL))' type: array x-kubernetes-list-type: atomic reloader: @@ -7825,7 +8388,7 @@ spec: type: array featureID: description: |- - FeatureID is an indentifier of the feature (Helm/Kustomize/Resources) + FeatureID is an identifier of the feature (Helm/Kustomize/Resources) This field indicates when to run this check. For instance: - if set to Helm this check will be run after all helm @@ -7843,6 +8406,46 @@ spec: Group of the resource to fetch in the managed Cluster. Required when Kind is set. Leave empty for metric-only checks. type: string + jobCheck: + description: |- + JobCheck runs a Job in the managed cluster and uses its Complete/Failed + outcome as the check result. Mutually exclusive with Script and EvaluateCEL. + properties: + jobRef: + description: |- + JobRef references the Secret/ConfigMap containing the Job manifest to + deploy in the managed Cluster as this check. + properties: + kind: + description: 'Kind of the resource. Supported kinds + are: Secrets and ConfigMaps.' + enum: + - Secret + - ConfigMap + type: string + name: + description: Name of the referenced resource. + minLength: 1 + type: string + namespace: + description: |- + Namespace of the referenced resource. + Namespace can be left empty. In such a case, namespace will + be implicit set to cluster's namespace. + type: string + required: + - kind + - name + - namespace + type: object + timeout: + description: |- + Timeout is how long to wait for the Job to reach Complete or Failed + before treating the check as failed. + type: string + required: + - jobRef + type: object kind: description: |- Kind of the resource to fetch in the managed Cluster. @@ -7957,6 +8560,9 @@ spec: - featureID - name type: object + x-kubernetes-validations: + - message: jobCheck cannot be set together with script or evaluateCEL + rule: '!has(self.jobCheck) || (!has(self.script) && !has(self.evaluateCEL))' type: array x-kubernetes-list-type: atomic type: object @@ -7991,7 +8597,7 @@ spec: type: string type: array featureID: - description: FeatureID is an indentifier of the feature whose + description: FeatureID is an identifier of the feature whose status is reported enum: - Resources @@ -9491,7 +10097,7 @@ spec: type: array featureID: description: |- - FeatureID is an indentifier of the feature (Helm/Kustomize/Resources) + FeatureID is an identifier of the feature (Helm/Kustomize/Resources) This field indicates when to run this check. For instance: - if set to Helm this check will be run after all helm @@ -9509,6 +10115,46 @@ spec: Group of the resource to fetch in the managed Cluster. Required when Kind is set. Leave empty for metric-only checks. type: string + jobCheck: + description: |- + JobCheck runs a Job in the managed cluster and uses its Complete/Failed + outcome as the check result. Mutually exclusive with Script and EvaluateCEL. + properties: + jobRef: + description: |- + JobRef references the Secret/ConfigMap containing the Job manifest to + deploy in the managed Cluster as this check. + properties: + kind: + description: 'Kind of the resource. Supported kinds + are: Secrets and ConfigMaps.' + enum: + - Secret + - ConfigMap + type: string + name: + description: Name of the referenced resource. + minLength: 1 + type: string + namespace: + description: |- + Namespace of the referenced resource. + Namespace can be left empty. In such a case, namespace will + be implicit set to cluster's namespace. + type: string + required: + - kind + - name + - namespace + type: object + timeout: + description: |- + Timeout is how long to wait for the Job to reach Complete or Failed + before treating the check as failed. + type: string + required: + - jobRef + type: object kind: description: |- Kind of the resource to fetch in the managed Cluster. @@ -9623,6 +10269,9 @@ spec: - featureID - name type: object + x-kubernetes-validations: + - message: jobCheck cannot be set together with script or evaluateCEL + rule: '!has(self.jobCheck) || (!has(self.script) && !has(self.evaluateCEL))' type: array x-kubernetes-list-type: atomic preDeleteChecks: @@ -9661,7 +10310,7 @@ spec: type: array featureID: description: |- - FeatureID is an indentifier of the feature (Helm/Kustomize/Resources) + FeatureID is an identifier of the feature (Helm/Kustomize/Resources) This field indicates when to run this check. For instance: - if set to Helm this check will be run after all helm @@ -9679,6 +10328,46 @@ spec: Group of the resource to fetch in the managed Cluster. Required when Kind is set. Leave empty for metric-only checks. type: string + jobCheck: + description: |- + JobCheck runs a Job in the managed cluster and uses its Complete/Failed + outcome as the check result. Mutually exclusive with Script and EvaluateCEL. + properties: + jobRef: + description: |- + JobRef references the Secret/ConfigMap containing the Job manifest to + deploy in the managed Cluster as this check. + properties: + kind: + description: 'Kind of the resource. Supported kinds + are: Secrets and ConfigMaps.' + enum: + - Secret + - ConfigMap + type: string + name: + description: Name of the referenced resource. + minLength: 1 + type: string + namespace: + description: |- + Namespace of the referenced resource. + Namespace can be left empty. In such a case, namespace will + be implicit set to cluster's namespace. + type: string + required: + - kind + - name + - namespace + type: object + timeout: + description: |- + Timeout is how long to wait for the Job to reach Complete or Failed + before treating the check as failed. + type: string + required: + - jobRef + type: object kind: description: |- Kind of the resource to fetch in the managed Cluster. @@ -9793,6 +10482,9 @@ spec: - featureID - name type: object + x-kubernetes-validations: + - message: jobCheck cannot be set together with script or evaluateCEL + rule: '!has(self.jobCheck) || (!has(self.script) && !has(self.evaluateCEL))' type: array x-kubernetes-list-type: atomic preDeployChecks: @@ -9831,7 +10523,7 @@ spec: type: array featureID: description: |- - FeatureID is an indentifier of the feature (Helm/Kustomize/Resources) + FeatureID is an identifier of the feature (Helm/Kustomize/Resources) This field indicates when to run this check. For instance: - if set to Helm this check will be run after all helm @@ -9849,6 +10541,46 @@ spec: Group of the resource to fetch in the managed Cluster. Required when Kind is set. Leave empty for metric-only checks. type: string + jobCheck: + description: |- + JobCheck runs a Job in the managed cluster and uses its Complete/Failed + outcome as the check result. Mutually exclusive with Script and EvaluateCEL. + properties: + jobRef: + description: |- + JobRef references the Secret/ConfigMap containing the Job manifest to + deploy in the managed Cluster as this check. + properties: + kind: + description: 'Kind of the resource. Supported kinds + are: Secrets and ConfigMaps.' + enum: + - Secret + - ConfigMap + type: string + name: + description: Name of the referenced resource. + minLength: 1 + type: string + namespace: + description: |- + Namespace of the referenced resource. + Namespace can be left empty. In such a case, namespace will + be implicit set to cluster's namespace. + type: string + required: + - kind + - name + - namespace + type: object + timeout: + description: |- + Timeout is how long to wait for the Job to reach Complete or Failed + before treating the check as failed. + type: string + required: + - jobRef + type: object kind: description: |- Kind of the resource to fetch in the managed Cluster. @@ -9963,6 +10695,9 @@ spec: - featureID - name type: object + x-kubernetes-validations: + - message: jobCheck cannot be set together with script or evaluateCEL + rule: '!has(self.jobCheck) || (!has(self.script) && !has(self.evaluateCEL))' type: array x-kubernetes-list-type: atomic reloader: @@ -10157,7 +10892,7 @@ spec: type: array featureID: description: |- - FeatureID is an indentifier of the feature (Helm/Kustomize/Resources) + FeatureID is an identifier of the feature (Helm/Kustomize/Resources) This field indicates when to run this check. For instance: - if set to Helm this check will be run after all helm @@ -10175,6 +10910,46 @@ spec: Group of the resource to fetch in the managed Cluster. Required when Kind is set. Leave empty for metric-only checks. type: string + jobCheck: + description: |- + JobCheck runs a Job in the managed cluster and uses its Complete/Failed + outcome as the check result. Mutually exclusive with Script and EvaluateCEL. + properties: + jobRef: + description: |- + JobRef references the Secret/ConfigMap containing the Job manifest to + deploy in the managed Cluster as this check. + properties: + kind: + description: 'Kind of the resource. Supported kinds + are: Secrets and ConfigMaps.' + enum: + - Secret + - ConfigMap + type: string + name: + description: Name of the referenced resource. + minLength: 1 + type: string + namespace: + description: |- + Namespace of the referenced resource. + Namespace can be left empty. In such a case, namespace will + be implicit set to cluster's namespace. + type: string + required: + - kind + - name + - namespace + type: object + timeout: + description: |- + Timeout is how long to wait for the Job to reach Complete or Failed + before treating the check as failed. + type: string + required: + - jobRef + type: object kind: description: |- Kind of the resource to fetch in the managed Cluster. @@ -10289,6 +11064,9 @@ spec: - featureID - name type: object + x-kubernetes-validations: + - message: jobCheck cannot be set together with script or evaluateCEL + rule: '!has(self.jobCheck) || (!has(self.script) && !has(self.evaluateCEL))' type: array x-kubernetes-list-type: atomic type: object diff --git a/pkg/app/app.go b/pkg/app/app.go new file mode 100644 index 00000000..2a55e15a --- /dev/null +++ b/pkg/app/app.go @@ -0,0 +1,826 @@ +/* +Copyright 2022-23 projectsveltos.io. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package app + +import ( + "context" + "flag" + "fmt" + "net/http" + "net/http/pprof" + "os" + "runtime" + "runtime/debug" + "sync" + "syscall" + "time" + + _ "embed" + + sourcev1 "github.com/fluxcd/source-controller/api/v1" + "github.com/go-logr/logr" + "github.com/spf13/pflag" + lua "github.com/yuin/gopher-lua" + corev1 "k8s.io/api/core/v1" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/fields" + apimachineryruntime "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/discovery" + _ "k8s.io/client-go/plugin/pkg/client/auth" + "k8s.io/client-go/rest" + cliflag "k8s.io/component-base/cli/flag" + "k8s.io/klog/v2" + clusterv1 "sigs.k8s.io/cluster-api/api/core/v1beta2" + "sigs.k8s.io/cluster-api/controllers/remote" + "sigs.k8s.io/cluster-api/util/apiwarnings" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/cache" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/healthz" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/metrics/filters" + metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" + "sigs.k8s.io/controller-runtime/pkg/webhook" + + configv1beta1 "github.com/projectsveltos/addon-controller/api/v1beta1" + "github.com/projectsveltos/addon-controller/api/v1beta1/index" + "github.com/projectsveltos/addon-controller/controllers" + "github.com/projectsveltos/addon-controller/controllers/dependencymanager" + "github.com/projectsveltos/addon-controller/internal/telemetry" + libsveltosv1beta1 "github.com/projectsveltos/libsveltos/api/v1beta1" + "github.com/projectsveltos/libsveltos/lib/crd" + "github.com/projectsveltos/libsveltos/lib/deployer" + logs "github.com/projectsveltos/libsveltos/lib/logsettings" + libsveltosset "github.com/projectsveltos/libsveltos/lib/set" + //+kubebuilder:scaffold:imports +) + +var ( + setupLog = ctrl.Log.WithName("setup") + diagnosticsAddress string + insecureDiagnostics bool + shardKey string + workers int + concurrentReconciles int + agentInMgmtCluster bool + reportMode controllers.ReportMode + tmpReportMode int + restConfigQPS float32 + restConfigBurst int + webhookPort int + syncPeriod time.Duration + conflictRetryTime time.Duration + healthErrorRetryTime time.Duration + version string + healthAddr string + profilerAddress string + driftDetectionConfigMap string + luaConfigMap string + capiOnboardAnnotation string + disableCaching bool + disableTelemetry bool + autoDeployDependencies bool + registry string + luaCallStackSize int + luaRegistrySize int + helmChartUpdateCheckInterval time.Duration +) + +const ( + defaultReconcilers = 10 + defaultWorkers = 20 + defaulReportMode = int(controllers.CollectFromManagementCluster) + mebibytes_bytes = 1 << 20 + gibibytes_per_bytes = 1 << 30 +) + +// Add RBAC for the authorized diagnostics endpoint. +// +kubebuilder:rbac:groups=authentication.k8s.io,resources=tokenreviews,verbs=create +// +kubebuilder:rbac:groups=authorization.k8s.io,resources=subjectaccessreviews,verbs=create + +// Run starts the addon-controller manager. It blocks until the manager stops. +func Run() { + scheme, err := controllers.InitScheme() + if err != nil { + os.Exit(1) + } + + setupLogging() + + sveltosNamespace := os.Getenv("NAMESPACE") + if sveltosNamespace == "" { + setupLog.V(logs.LogInfo).Error(nil, "Missing required environment variables NAMESPACE") + os.Exit(1) + } + + reportMode = controllers.ReportMode(tmpReportMode) + ctrl.SetLogger(klog.Background()) + ctrlOptions := getCtrlOptions(scheme) + + restConfig := getRestConfig() + + ctx := ctrl.SetupSignalHandler() + + controllers.SetDriftdetectionConfigMap(driftDetectionConfigMap) + controllers.SetLuaConfigMap(luaConfigMap) + controllers.SetLuaCallStackSize(luaCallStackSize) + controllers.SetLuaRegistrySize(luaRegistrySize) + controllers.SetCAPIOnboardAnnotation(capiOnboardAnnotation) + controllers.SetDriftDetectionRegistry(registry) + controllers.SetAgentInMgmtCluster(agentInMgmtCluster) + controllers.SetSveltosNamespace(sveltosNamespace) + + if isInitContainer() { + runInitContainerWork(ctx, restConfig, scheme) + os.Exit(0) + } + + mgr, err := ctrl.NewManager(restConfig, ctrlOptions) + if err != nil { + setupLog.Error(err, "unable to start manager") + os.Exit(1) + } + + dc, err := discovery.NewDiscoveryClientForConfig(mgr.GetConfig()) + if err != nil { + setupLog.Error(err, "unable to get discovery client") + os.Exit(1) + } + // Setup the context that's going to be used in controllers and for the manager. + controllers.SetManagementClusterAccess(mgr.GetClient(), mgr.GetConfig(), dc) + + // Start dependency manager + dependencymanager.InitializeManagerInstance(ctx, mgr.GetClient(), autoDeployDependencies, ctrl.Log.WithName("dependency_manager")) + + logs.RegisterForLogSettings(ctx, + libsveltosv1beta1.ComponentAddonManager, ctrl.Log.WithName("log-setter"), + ctrl.GetConfigOrDie()) + + debug.SetMemoryLimit(gibibytes_per_bytes) + go printMemUsage(ctx, ctrl.Log.WithName("memory-usage")) + controllers.NewLicenseManager() + + if shardKey == "" && !disableTelemetry { + err = telemetry.StartCollecting(ctx, mgr.GetConfig(), mgr.GetClient(), sveltosNamespace, version) + if err != nil { + setupLog.Error(err, "failed starting telemetry client") + } + } + + startControllersAndWatchers(ctx, mgr) + + setupChecks(mgr) + + setupIndexes(ctx, mgr) + + setupLog.Info("starting manager") + if err := mgr.Start(ctx); err != nil { + setupLog.Error(err, "problem running manager") + os.Exit(1) + } +} + +func getCacheConfig() (disableFor []client.Object, byObject map[client.Object]cache.ByObject) { + disableFor = []client.Object{} + byObject = map[client.Object]cache.ByObject{} + if disableCaching { + // Note: Only Secrets with type addons.projectsveltos.io/cluster-profile are cached + // The default client of the manager won't use the cache for secrets at all. + disableFor = []client.Object{ + &corev1.Secret{}, + &corev1.ConfigMap{}, + } + + fieldSelector := fields.OneTermEqualSelector("type", string(libsveltosv1beta1.ClusterProfileSecretType)) + + byObject[&corev1.Secret{}] = cache.ByObject{ + Field: fieldSelector, + } + } + return +} + +func initFlags(fs *pflag.FlagSet) { + fs.IntVar(&tmpReportMode, "report-mode", defaulReportMode, + "Indicates how ReportSummaries need to be collected") + + fs.BoolVar(&agentInMgmtCluster, "agent-in-mgmt-cluster", false, + "When set, indicates drift-detection-manager needs to be started in the management cluster") + + fs.BoolVar(&disableCaching, "disable-secret-caching", false, + "When set, disable caching secrets and configmaps") + + fs.BoolVar(&disableTelemetry, "disable-telemetry", false, + "When set, disable telemetry reporting") + + fs.StringVar(&diagnosticsAddress, "diagnostics-address", ":8443", + "The address the diagnostics endpoint binds to. Per default metrics are served via https and with"+ + "authentication/authorization. To serve via http and without authentication/authorization set --insecure-diagnostics."+ + "If --insecure-diagnostics is not set the diagnostics endpoint also serves pprof endpoints") + + fs.BoolVar(&insecureDiagnostics, "insecure-diagnostics", false, + "Enable insecure diagnostics serving. For more details see the description of --diagnostics-address.") + + fs.StringVar(&shardKey, "shard-key", "", + "If set, only clusters will annotation matching this shard key will be reconciled by this deployment.") + + fs.StringVar(&capiOnboardAnnotation, "capi-onboard-annotation", "", + "If provided, Sveltos will only manage CAPI clusters that have this exact annotation.") + + fs.IntVar(&workers, "worker-number", defaultWorkers, + "Number of worker. Workers are used to deploy features in CAPI clusters") + + fs.IntVar(&concurrentReconciles, "concurrent-reconciles", defaultReconcilers, + "concurrent reconciles is the maximum number of concurrent Reconciles which can be run. Defaults to 10") + + fs.StringVar(&version, "version", "", "current sveltos version") + + fs.StringVar(&healthAddr, "health-addr", ":9440", + "The address the health endpoint binds to.") + + fs.StringVar(&profilerAddress, "profiler-address", "", + "Bind address to expose the pprof profiler (e.g. localhost:6060)") + + fs.StringVar(&driftDetectionConfigMap, "drift-detection-config", "", + "The name of the ConfigMap in the namespace where projectsveltos is deployed containing the drift-detection-manager configuration") + + fs.StringVar(&luaConfigMap, "lua-methods", "", + "The name of the ConfigMap in the namespace where projectsveltos is deployed containing lua utilities to be loaded."+ + "Changing the content of the ConfigMap does not cause Sveltos to redeploy.") + + fs.StringVar(®istry, "registry", "", + "Container registry for drift-detection images. Defaults to docker.io/ if empty.") + + const defautlRestConfigQPS = 20 + fs.Float32Var(&restConfigQPS, "kube-api-qps", defautlRestConfigQPS, + fmt.Sprintf("Maximum queries per second from the controller client to the Kubernetes API server. Defaults to %d", + defautlRestConfigQPS)) + + const defaultRestConfigBurst = 60 + fs.IntVar(&restConfigBurst, "kube-api-burst", defaultRestConfigBurst, + fmt.Sprintf("Maximum number of queries that should be allowed in one burst from the controller client to the Kubernetes API server. Default %d", + defaultRestConfigBurst)) + + const defaultWebhookPort = 9443 + fs.IntVar(&webhookPort, "webhook-port", defaultWebhookPort, + "Webhook Server port") + + const defaultSyncPeriod = 10 + fs.DurationVar(&syncPeriod, "sync-period", defaultSyncPeriod*time.Minute, + fmt.Sprintf("The minimum interval at which watched resources are reconciled (e.g. 15m). Default: %d minutes", + defaultSyncPeriod)) + + const defaultConflictRetryTime = 60 + fs.DurationVar(&conflictRetryTime, "conflict-retry-time", defaultConflictRetryTime*time.Second, + fmt.Sprintf("The minimum interval at which watched ClusterProfile with conflicts are retried. Defaul: %d seconds", + defaultConflictRetryTime)) + + const defaultHealthErrorRetryTime = 60 + fs.DurationVar(&healthErrorRetryTime, "health-error-retry-time", defaultHealthErrorRetryTime*time.Second, + fmt.Sprintf("The minimum interval at which health check failures are retried. Default: %d seconds", + defaultHealthErrorRetryTime)) + + fs.DurationVar(&helmChartUpdateCheckInterval, "helm-chart-update-check-interval", time.Hour, + "Interval at which Sveltos checks whether newer versions of deployed Helm charts have been "+ + "published upstream (HTTP repositories and OCI registries). Set to 0 to disable.") + + // AutoDeployDependencies enables automatic deployment of prerequisite profiles. + // + // Profile instances can specify dependencies on other profiles using the + // DependsOn field, forming a directed acyclic graph (DAG) of dependencies. + // + // When AutoDeployDependencies is set to true, Sveltos automatically resolves and deploys + // the prerequisite profiles listed in the DependsOn field. This automation + // ensures that all required dependencies are deployed to the same managed clusters + // as the dependent profile. Sveltos analyzes the dependency graph to identify + // and deploy prerequisites in the correct order. + // + // By default, AutoDeployDependencies is enabled (true). When disabled, administrators + // are responsible for ensuring that both the dependent and prerequisite profiles + // target the same set of managed clusters through matching cluster selectors. + // + // Enabling AutoDeployDependencies simplifies multi-cluster management by automating + // dependency resolution, reducing manual effort, and minimizing the risk of + // configuration inconsistencies. + fs.BoolVar(&autoDeployDependencies, "auto-deploy-dependencies", true, + " When AutoDeployDependencies is set to true, Sveltos will automatically resolve and deploy the prerequisite profiles specified in the DependsOn field") + + fs.IntVar(&luaCallStackSize, "lua-call-stack-size", lua.CallStackSize, "Call stack size. This defaults to lua.CallStackSize") + + fs.IntVar(&luaRegistrySize, "lua-registry-size", lua.RegistrySize, "Call stack size. This defaults to lua.RegistrySize") +} + +func setupIndexes(ctx context.Context, mgr ctrl.Manager) { + if err := index.AddDefaultIndexes(ctx, mgr); err != nil { + setupLog.Error(err, "unable to setup indexes") + os.Exit(1) + } +} + +func setupChecks(mgr ctrl.Manager) { + if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { + setupLog.Error(err, "unable to set up health check") + os.Exit(1) + } + if err := mgr.AddReadyzCheck("readyz", healthz.Ping); err != nil { + setupLog.Error(err, "unable to set up ready check") + os.Exit(1) + } +} + +// capiCRDHandler restarts process if a CAPI CRD is updated +func capiCRDHandler(gvk *schema.GroupVersionKind, action crd.ChangeType) { + if action == crd.Modify { + return + } + if gvk.Group == clusterv1.GroupVersion.Group && gvk.Version == clusterv1.GroupVersion.Version { + setupLog.V(logs.LogInfo).Info("Initiating graceful restart due to CAPI CRD update", + "GVK", gvk.String(), "Action", string(action)) + + if killErr := syscall.Kill(syscall.Getpid(), syscall.SIGTERM); killErr != nil { + panic("kill -TERM failed") + } + } +} + +// isCAPIInstalled returns true if CAPI is installed with v1beta2 served, false otherwise +func isCAPIInstalled(ctx context.Context, c client.Client, logger logr.Logger) (bool, error) { + clusterCRD := &apiextensionsv1.CustomResourceDefinition{} + + err := c.Get(ctx, types.NamespacedName{Name: "clusters.cluster.x-k8s.io"}, clusterCRD) + if err != nil { + if apierrors.IsNotFound(err) { + return false, nil + } + return false, err + } + + for _, version := range clusterCRD.Spec.Versions { + if version.Name == clusterv1.GroupVersion.Version && version.Served { + return true, nil + } + } + + logger.V(logs.LogInfo).Info("clusterCRD CRD present but v1beta2 not served") + return false, nil +} + +// fluxCRDHandler restarts process if a Flux CRD is updated +func fluxCRDHandler(gvk *schema.GroupVersionKind, action crd.ChangeType) { + if action == crd.Modify { + return + } + + if gvk.Group == sourcev1.GroupVersion.Group && gvk.Kind == sourcev1.GitRepositoryKind { + setupLog.V(logs.LogInfo).Info("Initiating graceful restart due to Flux CRD update", + "GVK", gvk.String(), "Action", string(action)) + + if killErr := syscall.Kill(syscall.Getpid(), syscall.SIGTERM); killErr != nil { + panic("kill -TERM failed") + } + } +} + +// isFluxInstalled returns true if Flux is installed, false otherwise +func isFluxInstalled(ctx context.Context, c client.Client) (bool, error) { + gitRepositoryCRD := &apiextensionsv1.CustomResourceDefinition{} + + err := c.Get(ctx, types.NamespacedName{Name: "gitrepositories.source.toolkit.fluxcd.io"}, + gitRepositoryCRD) + if err != nil { + if apierrors.IsNotFound(err) { + return false, nil + } + return false, err + } + + return true, nil +} + +func capiWatchers(ctx context.Context, mgr ctrl.Manager, watchersForCAPI []watcherForCAPI, logger logr.Logger) { + const maxRetries = 20 + retries := 0 + for { + capiPresent, err := isCAPIInstalled(ctx, mgr.GetClient(), logger) + if err != nil { + if retries < maxRetries { + logger.Info(fmt.Sprintf("failed to verify if CAPI is present: %v", err)) + time.Sleep(time.Second) + } + retries++ + } else { + if !capiPresent { + setupLog.V(logs.LogInfo).Info("CAPI currently not present. Starting CRD watcher") + go crd.WatchCustomResourceDefinition(ctx, mgr.GetConfig(), capiCRDHandler, setupLog) + } else { + setupLog.V(logs.LogInfo).Info("CAPI present. Start CAPI watchers") + for i := range watchersForCAPI { + watcher := watchersForCAPI[i] + err = watcher.WatchForCAPI(mgr, watcher.GetController()) + if err != nil { + setupLog.V(logs.LogInfo).Info( + fmt.Sprintf("failed to start CAPI watcher: %v", err)) + continue + } + } + } + return + } + } +} + +func fluxWatchers(ctx context.Context, mgr ctrl.Manager, watchersForFlux []watcherForFlux, logger logr.Logger) { + const maxRetries = 20 + retries := 0 + for { + fluxPresent, err := isFluxInstalled(ctx, mgr.GetClient()) + if err != nil { + if retries < maxRetries { + logger.Info(fmt.Sprintf("failed to verify if Flux is present: %v", err)) + time.Sleep(time.Second) + } + retries++ + } else { + if !fluxPresent { + setupLog.V(logs.LogInfo).Info("Flux currently not present. Starting CRD watcher") + go crd.WatchCustomResourceDefinition(ctx, mgr.GetConfig(), fluxCRDHandler, setupLog) + } else { + setupLog.V(logs.LogInfo).Info("Flux present. Start Flux watchers") + for i := range watchersForFlux { + watcher := watchersForFlux[i] + err = watcher.WatchForFlux(mgr, watcher.GetController()) + if err != nil { + continue + } + } + } + return + } + } +} + +type watcherForCAPI interface { + WatchForCAPI(mgr manager.Manager, c controller.Controller) error + GetController() controller.Controller +} + +type watcherForFlux interface { + WatchForFlux(mgr manager.Manager, c controller.Controller) error + GetController() controller.Controller +} + +func startWatchers(ctx context.Context, mgr manager.Manager, + watchersForCAPI []watcherForCAPI, watchersForFlux []watcherForFlux) { + + go capiWatchers(ctx, mgr, watchersForCAPI, setupLog) + + go fluxWatchers(ctx, mgr, watchersForFlux, setupLog) +} + +func getProfileReconciler(mgr manager.Manager) *controllers.ProfileReconciler { + return &controllers.ProfileReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + SetMap: make(map[corev1.ObjectReference]*libsveltosset.Set), + ClusterMap: make(map[corev1.ObjectReference]*libsveltosset.Set), + Profiles: make(map[corev1.ObjectReference]libsveltosv1beta1.Selector), + ClusterLabels: make(map[corev1.ObjectReference]map[string]string), + Mux: sync.Mutex{}, + ConcurrentReconciles: concurrentReconciles, + Logger: ctrl.Log.WithName("profilereconciler"), + } +} + +func getClusterProfileReconciler(mgr manager.Manager) *controllers.ClusterProfileReconciler { + return &controllers.ClusterProfileReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + ClusterSetMap: make(map[corev1.ObjectReference]*libsveltosset.Set), + ClusterMap: make(map[corev1.ObjectReference]*libsveltosset.Set), + ClusterProfiles: make(map[corev1.ObjectReference]libsveltosv1beta1.Selector), + ClusterLabels: make(map[corev1.ObjectReference]map[string]string), + Mux: sync.Mutex{}, + ConcurrentReconciles: concurrentReconciles, + Logger: ctrl.Log.WithName("clusterprofilereconciler"), + } +} + +func getClusterSummaryReconciler(ctx context.Context, mgr manager.Manager) *controllers.ClusterSummaryReconciler { + d := deployer.GetClient(ctx, ctrl.Log.WithName("deployer"), mgr.GetClient(), workers) + controllers.RegisterFeatures(d, setupLog) + + return &controllers.ClusterSummaryReconciler{ + Config: mgr.GetConfig(), + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + ShardKey: shardKey, + Version: version, + ReportMode: reportMode, + Deployer: d, + ClusterMap: make(map[corev1.ObjectReference]*libsveltosset.Set), + ReferenceMap: make(map[corev1.ObjectReference]*libsveltosset.Set), + PolicyMux: sync.Mutex{}, + ConcurrentReconciles: concurrentReconciles, + ConflictRetryTime: conflictRetryTime, + HealthErrorRetryTime: healthErrorRetryTime, + Logger: ctrl.Log.WithName("clustersummaryreconciler"), + } +} + +func getSetReconciler(mgr manager.Manager) *controllers.SetReconciler { + return &controllers.SetReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + ConcurrentReconciles: concurrentReconciles, + Mux: sync.Mutex{}, + ClusterMap: make(map[corev1.ObjectReference]*libsveltosset.Set), + SetMap: make(map[corev1.ObjectReference]*libsveltosset.Set), + Sets: make(map[corev1.ObjectReference]libsveltosv1beta1.Selector), + ClusterLabels: make(map[corev1.ObjectReference]map[string]string), + Logger: ctrl.Log.WithName("setreconciler"), + } +} + +func getClusterSetReconciler(mgr manager.Manager) *controllers.ClusterSetReconciler { + return &controllers.ClusterSetReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + ConcurrentReconciles: concurrentReconciles, + Mux: sync.Mutex{}, + ClusterMap: make(map[corev1.ObjectReference]*libsveltosset.Set), + ClusterSetMap: make(map[corev1.ObjectReference]*libsveltosset.Set), + ClusterSets: make(map[corev1.ObjectReference]libsveltosv1beta1.Selector), + ClusterLabels: make(map[corev1.ObjectReference]map[string]string), + Logger: ctrl.Log.WithName("clustersetreconciler"), + } +} + +// getDiagnosticsOptions returns metrics options which can be used to configure a Manager. +func getDiagnosticsOptions() metricsserver.Options { + // If "--insecure-diagnostics" is set, serve metrics via http + // and without authentication/authorization. + if insecureDiagnostics { + return metricsserver.Options{ + BindAddress: diagnosticsAddress, + SecureServing: false, + } + } + + // If "--insecure-diagnostics" is not set, serve metrics via https + // and with authentication/authorization. As the endpoint is protected, + // we also serve pprof endpoints and an endpoint to change the log level. + return metricsserver.Options{ + BindAddress: diagnosticsAddress, + SecureServing: true, + FilterProvider: filters.WithAuthenticationAndAuthorization, + ExtraHandlers: map[string]http.Handler{ + // Add pprof handler. + "/debug/pprof/": http.HandlerFunc(pprof.Index), + "/debug/pprof/cmdline": http.HandlerFunc(pprof.Cmdline), + "/debug/pprof/profile": http.HandlerFunc(pprof.Profile), + "/debug/pprof/symbol": http.HandlerFunc(pprof.Symbol), + "/debug/pprof/trace": http.HandlerFunc(pprof.Trace), + "/debug/pprof/heap": pprof.Handler("heap"), + }, + } +} + +// startControllers starts all reconcilers: +// - ClusterProfile/Profile +// - clusterSummary +// - ClusterSet/Set +// +// It also starts needed watchers: +// - cluster API watchers for ClusterProfile/Profile, ClusterSet/Set +// - Flux watcher for ClusterSummary +func startControllersAndWatchers(ctx context.Context, mgr manager.Manager) { + var clusterProfileReconciler *controllers.ClusterProfileReconciler + var profileReconciler *controllers.ProfileReconciler + var clusterSetReconciler *controllers.ClusterSetReconciler + var setReconciler *controllers.SetReconciler + + var err error + + //+kubebuilder:scaffold:builder + if err = (&controllers.SveltosClusterReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "SveltosCluster") + os.Exit(1) + } + + watchersForCAPI := make([]watcherForCAPI, 0) + watchersForFlux := make([]watcherForFlux, 0) + + if shardKey == "" { + // Only if shardKey is not set, start ClusterProfile/Profile and ClusterSet/Set reconcilers. + // When shardKey is set, only ClusterSummary reconciler will be started and only + // cluster matching the shardkey will be managed + clusterProfileReconciler = getClusterProfileReconciler(mgr) + err = clusterProfileReconciler.SetupWithManager(mgr) + if err != nil { + setupLog.Error(err, "unable to create controller", "controller", configv1beta1.ClusterProfileKind) + os.Exit(1) + } + watchersForCAPI = append(watchersForCAPI, clusterProfileReconciler) + + profileReconciler = getProfileReconciler(mgr) + err = profileReconciler.SetupWithManager(mgr) + if err != nil { + setupLog.Error(err, "unable to create controller", "controller", configv1beta1.ProfileKind) + os.Exit(1) + } + watchersForCAPI = append(watchersForCAPI, profileReconciler) + + clusterSetReconciler = getClusterSetReconciler(mgr) + err = clusterSetReconciler.SetupWithManager(mgr) + if err != nil { + setupLog.Error(err, "unable to create controller", "controller", libsveltosv1beta1.ClusterSetKind) + os.Exit(1) + } + watchersForCAPI = append(watchersForCAPI, clusterSetReconciler) + + setReconciler = getSetReconciler(mgr) + err = setReconciler.SetupWithManager(mgr) + if err != nil { + setupLog.Error(err, "unable to create controller", "controller", libsveltosv1beta1.SetKind) + os.Exit(1) + } + watchersForCAPI = append(watchersForCAPI, setReconciler) + + if err := (&controllers.ClusterPromotionReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Config: mgr.GetConfig(), + ConcurrentReconciles: concurrentReconciles, + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "ClusterPromotion") + os.Exit(1) + } + + // Needs a fleet-wide view of every ClusterSummary to dedup chart keys correctly, so + // this only ever runs on the default (unsharded) deployment, same as the reconcilers + // started above. Running it per-shard would give no benefit (chart-key dedup is + // inherently fleet-wide) and would reintroduce cross-shard inconsistency. + if helmChartUpdateCheckInterval > 0 { + go controllers.RunHelmChartUpdateChecker(ctx, mgr.GetClient(), helmChartUpdateCheckInterval, + ctrl.Log.WithName("helm-chart-update-checker")) + } + } + + clusterSummaryReconciler := getClusterSummaryReconciler(ctx, mgr) + err = clusterSummaryReconciler.SetupWithManager(ctx, mgr) + if err != nil { + setupLog.Error(err, "unable to create controller", "controller", configv1beta1.ClusterSummaryKind) + os.Exit(1) + } + watchersForCAPI = append(watchersForCAPI, clusterSummaryReconciler) + watchersForFlux = append(watchersForFlux, clusterSummaryReconciler) + + startWatchers(ctx, mgr, watchersForCAPI, watchersForFlux) +} + +// printMemUsage memory stats. Call GC +func printMemUsage(ctx context.Context, logger logr.Logger) { + const five = 5 + ticker := time.NewTicker(five * time.Minute) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + logger.Info("stopping memory usage printer") + return + case <-ticker.C: + time.Sleep(time.Minute) + var m runtime.MemStats + runtime.ReadMemStats(&m) + // For info on each, see: /pkg/runtime/#MemStats + l := logger.WithValues("Alloc (MiB)", bToMb(m.Alloc)). + WithValues("TotalAlloc (MiB)", bToMb(m.TotalAlloc)). + WithValues("Sys (MiB)", bToMb(m.Sys)). + WithValues("NumGC", m.NumGC) + l.V(1).Info("memory stats") + } + } +} + +func bToMb(b uint64) uint64 { + return b / mebibytes_bytes +} + +// reduceMemoryFootprint removes large, rarely-used fields from all cached objects +func reduceMemoryFootprint(obj interface{}) (interface{}, error) { + accessor, err := meta.Accessor(obj) + if err != nil { + return obj, nil + } + + // Remove managedFields (typically the largest field) + accessor.SetManagedFields(nil) + + // Clean up annotations + annotations := accessor.GetAnnotations() + if len(annotations) > 0 { + // Remove large annotations + delete(annotations, "kubectl.kubernetes.io/last-applied-configuration") + delete(annotations, "deployment.kubernetes.io/revision") + + // If annotations is now empty, set to nil + if len(annotations) == 0 { + accessor.SetAnnotations(nil) + } else { + accessor.SetAnnotations(annotations) + } + } + + return obj, nil +} + +func getCtrlOptions(scheme *apimachineryruntime.Scheme) ctrl.Options { + disableFor, byObject := getCacheConfig() + + return ctrl.Options{ + Scheme: scheme, + Metrics: getDiagnosticsOptions(), + HealthProbeBindAddress: healthAddr, + WebhookServer: webhook.NewServer( + webhook.Options{ + Port: webhookPort, + }), + Cache: cache.Options{ + SyncPeriod: &syncPeriod, + ByObject: byObject, + DefaultTransform: reduceMemoryFootprint, + }, + Client: client.Options{ + Cache: &client.CacheOptions{ + DisableFor: disableFor, + }, + }, + PprofBindAddress: profilerAddress, + } +} + +func isInitContainer() bool { + return os.Getenv("IS_INITIALIZATION") == "true" +} + +func runInitContainerWork(ctx context.Context, config *rest.Config, + scheme *apimachineryruntime.Scheme) { + + directClient, err := client.New(config, client.Options{Scheme: scheme}) + if err != nil { + return + } + + dc, err := discovery.NewDiscoveryClientForConfig(config) + if err != nil { + setupLog.Error(err, "unable to get discovery client") + os.Exit(1) + } + + controllers.SetManagementClusterAccess(directClient, config, dc) + controllers.Initialization(ctx, config, scheme, shardKey, + ctrl.Log.WithName("initialization")) +} + +func setupLogging() { + klog.InitFlags(nil) + + initFlags(pflag.CommandLine) + pflag.CommandLine.SetNormalizeFunc(cliflag.WordSepNormalizeFunc) + pflag.CommandLine.AddGoFlagSet(flag.CommandLine) + pflag.Parse() + + ctrl.SetLogger(klog.Background()) +} + +func getRestConfig() *rest.Config { + restConfig := ctrl.GetConfigOrDie() + restConfig.QPS = restConfigQPS + restConfig.Burst = restConfigBurst + restConfig.UserAgent = remote.DefaultClusterAPIUserAgent("addon-controller") + restConfig.WarningHandler = apiwarnings.DefaultHandler(klog.Background().WithName("API Server Warning")) + return restConfig +} diff --git a/test/fv/job_health_check_test.go b/test/fv/job_health_check_test.go new file mode 100644 index 00000000..eb2933fb --- /dev/null +++ b/test/fv/job_health_check_test.go @@ -0,0 +1,202 @@ +/* +Copyright 2026. projectsveltos.io. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package fv_test + +import ( + "context" + "fmt" + "strings" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + + configv1beta1 "github.com/projectsveltos/addon-controller/api/v1beta1" + "github.com/projectsveltos/addon-controller/lib/clusterops" + libsveltosv1beta1 "github.com/projectsveltos/libsveltos/api/v1beta1" +) + +// jobHealthCheckConfigMap is the resource ClusterProfile.PolicyRefs deploys; the ValidateHealth +// JobCheck runs after it, gated on FeatureResources. +const jobHealthCheckConfigMap = `apiVersion: v1 +kind: ConfigMap +metadata: + name: %s + namespace: %s +data: + foo: bar` + +// jobHealthCheckJob is a fast-completing (or fast-failing, via %[3]d exit code) Job manifest +// referenced by ValidateHealth.JobCheck.JobRef. backoffLimit is 0 so a failing Job fails +// immediately instead of retrying. +const jobHealthCheckJob = `apiVersion: batch/v1 +kind: Job +metadata: + name: %s + namespace: %s +spec: + backoffLimit: 0 + template: + spec: + restartPolicy: Never + containers: + - name: probe + image: busybox:1.36 + command: ["sh", "-c", "exit %d"]` + +var _ = Describe("JobCheck", func() { + const ( + namePrefix = "job-check-" + ) + + It("provisions once the referenced Job completes successfully", Label("Enterprise"), func() { + resourceNs := randomString() + resourceName := randomString() + jobName := randomString() + + Byf("Create ConfigMap in the management cluster holding the deployed ConfigMap resource") + resourceConfigMap := createConfigMapWithPolicy(defaultNamespace, namePrefix+randomString(), + fmt.Sprintf(jobHealthCheckConfigMap, resourceName, resourceNs)) + Expect(k8sClient.Create(context.TODO(), resourceConfigMap)).To(Succeed()) + + Byf("Create ConfigMap in the management cluster holding a Job manifest that exits 0") + jobConfigMap := createConfigMapWithPolicy(defaultNamespace, namePrefix+randomString(), + fmt.Sprintf(jobHealthCheckJob, jobName, defaultNamespace, 0)) + Expect(k8sClient.Create(context.TODO(), jobConfigMap)).To(Succeed()) + + Byf("Create ClusterProfile deploying the ConfigMap resource with a JobCheck ValidateHealth") + clusterProfile := getClusterProfile(namePrefix, map[string]string{key: value}) + clusterProfile.Spec.SyncMode = configv1beta1.SyncModeContinuous + clusterProfile.Spec.PolicyRefs = []configv1beta1.PolicyRef{ + { + Kind: string(libsveltosv1beta1.ConfigMapReferencedResourceKind), + Namespace: resourceConfigMap.Namespace, + Name: resourceConfigMap.Name, + }, + } + clusterProfile.Spec.ValidateHealths = []libsveltosv1beta1.ValidateHealth{ + { + Name: "job-completes", + FeatureID: libsveltosv1beta1.FeatureResources, + JobCheck: &libsveltosv1beta1.JobHealthCheck{ + JobRef: libsveltosv1beta1.PolicyRef{ + Kind: string(libsveltosv1beta1.ConfigMapReferencedResourceKind), + Namespace: jobConfigMap.Namespace, + Name: jobConfigMap.Name, + }, + Timeout: &metav1.Duration{Duration: time.Minute}, + }, + }, + } + Expect(k8sClient.Create(context.TODO(), clusterProfile)).To(Succeed()) + + verifyClusterProfileMatches(clusterProfile) + + clusterSummary := verifyClusterSummary(clusterops.ClusterProfileLabelName, + clusterProfile.Name, &clusterProfile.Spec, + kindWorkloadCluster.GetNamespace(), kindWorkloadCluster.GetName(), getClusterType()) + + Byf("Verifying ClusterSummary %s becomes Provisioned for Resources once the Job completes", clusterSummary.Name) + verifyFeatureStatusIsProvisioned(kindWorkloadCluster.GetNamespace(), clusterSummary.Name, libsveltosv1beta1.FeatureResources) + + Byf("Deleting ConfigMap %s/%s (deployed resource)", resourceConfigMap.Namespace, resourceConfigMap.Name) + Expect(k8sClient.Delete(context.TODO(), resourceConfigMap)).To(Succeed()) + + Byf("Deleting ConfigMap %s/%s (Job manifest)", jobConfigMap.Namespace, jobConfigMap.Name) + Expect(k8sClient.Delete(context.TODO(), jobConfigMap)).To(Succeed()) + + deleteClusterProfile(clusterProfile) + }) + + It("reports a failure while the referenced Job keeps failing", Label("Enterprise"), func() { + resourceNs := randomString() + resourceName := randomString() + jobName := randomString() + + Byf("Create ConfigMap in the management cluster holding the deployed ConfigMap resource") + resourceConfigMap := createConfigMapWithPolicy(defaultNamespace, namePrefix+randomString(), + fmt.Sprintf(jobHealthCheckConfigMap, resourceName, resourceNs)) + Expect(k8sClient.Create(context.TODO(), resourceConfigMap)).To(Succeed()) + + Byf("Create ConfigMap in the management cluster holding a Job manifest that exits 1") + jobConfigMap := createConfigMapWithPolicy(defaultNamespace, namePrefix+randomString(), + fmt.Sprintf(jobHealthCheckJob, jobName, defaultNamespace, 1)) + Expect(k8sClient.Create(context.TODO(), jobConfigMap)).To(Succeed()) + + Byf("Create ClusterProfile deploying the ConfigMap resource with a JobCheck ValidateHealth") + clusterProfile := getClusterProfile(namePrefix, map[string]string{key: value}) + clusterProfile.Spec.SyncMode = configv1beta1.SyncModeContinuous + clusterProfile.Spec.PolicyRefs = []configv1beta1.PolicyRef{ + { + Kind: string(libsveltosv1beta1.ConfigMapReferencedResourceKind), + Namespace: resourceConfigMap.Namespace, + Name: resourceConfigMap.Name, + }, + } + clusterProfile.Spec.ValidateHealths = []libsveltosv1beta1.ValidateHealth{ + { + Name: "job-fails", + FeatureID: libsveltosv1beta1.FeatureResources, + JobCheck: &libsveltosv1beta1.JobHealthCheck{ + JobRef: libsveltosv1beta1.PolicyRef{ + Kind: string(libsveltosv1beta1.ConfigMapReferencedResourceKind), + Namespace: jobConfigMap.Namespace, + Name: jobConfigMap.Name, + }, + Timeout: &metav1.Duration{Duration: time.Minute}, + }, + }, + } + Expect(k8sClient.Create(context.TODO(), clusterProfile)).To(Succeed()) + + verifyClusterProfileMatches(clusterProfile) + + clusterSummary := verifyClusterSummary(clusterops.ClusterProfileLabelName, + clusterProfile.Name, &clusterProfile.Spec, + kindWorkloadCluster.GetNamespace(), kindWorkloadCluster.GetName(), getClusterType()) + + Byf("Verifying ClusterSummary %s reports a JobCheck failure for Resources", clusterSummary.Name) + Eventually(func() bool { + currentClusterSummary := &configv1beta1.ClusterSummary{} + if err := k8sClient.Get(context.TODO(), + types.NamespacedName{Namespace: clusterSummary.Namespace, Name: clusterSummary.Name}, + currentClusterSummary); err != nil { + return false + } + for i := range currentClusterSummary.Status.FeatureSummaries { + fs := ¤tClusterSummary.Status.FeatureSummaries[i] + if fs.FeatureID == libsveltosv1beta1.FeatureResources && + fs.FailureMessage != nil && + strings.Contains(*fs.FailureMessage, "job-fails") { + return true + } + } + return false + }, timeout, pollingInterval).Should(BeTrue()) + + Byf("Deleting ConfigMap %s/%s (deployed resource)", resourceConfigMap.Namespace, resourceConfigMap.Name) + Expect(k8sClient.Delete(context.TODO(), resourceConfigMap)).To(Succeed()) + + Byf("Deleting ConfigMap %s/%s (Job manifest)", jobConfigMap.Namespace, jobConfigMap.Name) + Expect(k8sClient.Delete(context.TODO(), jobConfigMap)).To(Succeed()) + + deleteClusterProfile(clusterProfile) + }) +}) diff --git a/test/fv/promotion_test.go b/test/fv/promotion_test.go index 99170ed7..74559fe7 100644 --- a/test/fv/promotion_test.go +++ b/test/fv/promotion_test.go @@ -100,7 +100,7 @@ spec: end` ) - It("Deploy ClusterPromotion with multiple stages", Label("ClusterPromotion"), func() { + It("Deploy ClusterPromotion with multiple stages", Label("Enterprise"), func() { configMapNs := defaultNamespace configMap := createConfigMapWithPolicy(configMapNs, namePrefix+randomString(), fmt.Sprintf(counterJob, configMapNs))