From a93e9496151d2db446134359107f0d0554d8b554 Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Fri, 26 Jun 2026 13:34:46 +0200 Subject: [PATCH 1/6] Claude done the thing --- CHANGELOG.md | 9 + .../spark-k8s-operator/templates/roles.yaml | 26 +- .../pages/getting_started/first_steps.adoc | 21 +- .../spark-k8s/pages/usage-guide/examples.adoc | 2 +- extra/crds.yaml | 8486 +++++++++++++---- rust/operator-binary/src/config/jvm.rs | 2 +- rust/operator-binary/src/crd/constants.rs | 15 +- rust/operator-binary/src/crd/mod.rs | 414 +- rust/operator-binary/src/crd/roles.rs | 39 +- .../src/crd/template_merger.rs | 174 +- rust/operator-binary/src/crd/template_spec.rs | 84 +- rust/operator-binary/src/main.rs | 17 +- .../src/pod_driver_controller.rs | 27 +- .../src/spark_k8s_controller.rs | 339 +- .../src/spark_k8s_controller/dereference.rs | 9 +- .../src/spark_k8s_controller/validate.rs | 9 +- .../src/webhooks/conversion.rs | 4 +- .../templates/kuttl/overrides/10-assert.yaml | 4 +- .../templates/kuttl/overrides/11-assert.yaml | 11 - .../kuttl/resources/10-assert.yaml.j2 | 21 +- .../kuttl/resources/12-assert.yaml.j2 | 21 +- tests/templates/kuttl/smoke/50-assert.yaml | 4 +- tests/templates/kuttl/smoke/52-assert.yaml | 45 +- tests/templates/kuttl/smoke/53-assert.yaml.j2 | 19 - tests/test-definition.yaml | 12 +- 25 files changed, 7032 insertions(+), 2782 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fb66fead..9916fb7a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ All notable changes to this project will be documented in this file. ### Added - BREAKING: Add required CLI argument and env var to set the image repository used to construct final product image names: `IMAGE_REPOSITORY` (`--image-repository`), eg. `oci.example.org/my/namespace` ([#684]). +- Add CRD version `v1alpha2` for `SparkApplication` and `SparkApplicationTemplate`. The conversion webhook converts `v1alpha1` objects to `v1alpha2` ([#711]). ### Fixed @@ -15,6 +16,8 @@ All notable changes to this project will be documented in this file. ### Changed +- BREAKING: The operator no longer runs a separate `spark-submit` process for `SparkApplication`s. The driver is launched directly as a Kubernetes `Job` (built from `spec.driver`) running `spark-submit` in client mode; executors are still created by the driver via Spark's Kubernetes backend. A headless driver `Service` is created so executors can reach the driver. This affects both `v1alpha1` (after conversion) and `v1alpha2` objects ([#711]). +- The driver `Job` is no longer retried on failure (`backoffLimit` is `0`); the previous `spec.job.retryOnFailureCount` is deprecated and ignored ([#711]). - Document Helm deployed RBAC permissions and remove unnecessary permissions ([#674]). - BREAKING: Each custom resource accepts now only the known config files in `configOverrides`: - `SparkApplication`: `spark-env.sh` and `security.properties` @@ -27,6 +30,11 @@ All notable changes to this project will be documented in this file. - Fix the `SparkApplication` CRD description, which incorrectly described it as a "Spark cluster stacklet" rather than a Spark application ([#705]). - BREAKING: make application templates namespaced instead of cluster wide objects ([#694]). +### Deprecated + +- `SparkApplication`/`SparkApplicationTemplate` `spec.job` is deprecated and ignored since `v1alpha2` (renamed to `deprecatedJob` in that version). The driver `Job` is now built from `spec.driver` ([#711]). +- `SparkApplication`/`SparkApplicationTemplate` `spec.mode` is deprecated and ignored: the operator always runs the driver in client mode internally ([#711]). + [#674]: https://github.com/stackabletech/spark-k8s-operator/pull/674 [#679]: https://github.com/stackabletech/spark-k8s-operator/pull/679 [#680]: https://github.com/stackabletech/spark-k8s-operator/pull/680 @@ -36,6 +44,7 @@ All notable changes to this project will be documented in this file. [#694]: https://github.com/stackabletech/spark-k8s-operator/pull/694 [#696]: https://github.com/stackabletech/spark-k8s-operator/pull/696 [#705]: https://github.com/stackabletech/spark-k8s-operator/pull/705 +[#711]: https://github.com/stackabletech/spark-k8s-operator/pull/711 ## [26.3.0] - 2026-03-16 diff --git a/deploy/helm/spark-k8s-operator/templates/roles.yaml b/deploy/helm/spark-k8s-operator/templates/roles.yaml index e3fab4ff..4804aade 100644 --- a/deploy/helm/spark-k8s-operator/templates/roles.yaml +++ b/deploy/helm/spark-k8s-operator/templates/roles.yaml @@ -13,16 +13,14 @@ rules: - nodes/proxy verbs: - get - # The pod-driver controller watches Spark driver pods - # (labelled spark-role=driver) to track SparkApplication completion. It also - # deletes driver pods once the application reaches a terminal phase (Succeeded - # or Failed). + # The pod-driver controller watches Spark driver pods (labelled spark-role=driver) to track + # SparkApplication completion. Driver pods are owned by the driver Job and cleaned up via the + # Job's ttlSecondsAfterFinished, so the operator no longer deletes them. - apiGroups: - "" resources: - pods verbs: - - delete - get - list - watch @@ -43,9 +41,10 @@ rules: - patch - watch # Services expose Spark History Server and Spark Connect Server for metrics and - # inter-component communication. Applied via Server-Side Apply and tracked for orphan - # cleanup by the history and connect controllers. The history and connect controllers - # watch Services via .owns(Service) to trigger re-reconciliation on change. + # inter-component communication, and the app controller creates a headless driver Service per + # SparkApplication so executors can reach the driver in client mode. Applied via Server-Side + # Apply and tracked for orphan cleanup by the history and connect controllers. The app, history + # and connect controllers watch Services via .owns(Service) to trigger re-reconciliation on change. # get is required for the ReconciliationPaused strategy in cluster_resources.add(). - apiGroups: - "" @@ -115,17 +114,20 @@ rules: - list - patch - watch - # A Kubernetes Job is created per SparkApplication via Server-Side Apply to run - # spark-submit. The app controller applies Jobs directly (not via cluster_resources), - # so only create + patch (SSA) are needed. Jobs are not watched and not tracked for - # orphan cleanup by any controller. + # The driver Job is created per SparkApplication via Server-Side Apply. The app controller + # applies Jobs directly (create + patch via SSA) and watches them via .owns(Job) to track + # SparkApplication progress, so get + list + watch are also required. Jobs are garbage collected + # via their owner reference and ttlSecondsAfterFinished, so no explicit delete is needed. - apiGroups: - batch resources: - jobs verbs: - create + - get + - list - patch + - watch # PodDisruptionBudgets limit voluntary disruptions to Spark History Server pods. # Applied via Server-Side Apply and tracked for orphan cleanup by the history # controller. No controller watches PDBs via .owns(). diff --git a/docs/modules/spark-k8s/pages/getting_started/first_steps.adoc b/docs/modules/spark-k8s/pages/getting_started/first_steps.adoc index e59a2e0a..a19cad6f 100644 --- a/docs/modules/spark-k8s/pages/getting_started/first_steps.adoc +++ b/docs/modules/spark-k8s/pages/getting_started/first_steps.adoc @@ -6,11 +6,13 @@ Afterwards you can <<_verify_that_it_works, verify that it works>> by looking at == Starting a Spark job -A Spark application is made of up three components: +A Spark application is made up of two components: -* Job: this builds a `spark-submit` command from the resource, passing this to internal spark code together with templates for building the driver and executor pods -* Driver: the driver starts the designated number of executors and removes them when the job is completed. -* Executor(s): responsible for executing the job itself +* Driver: the operator creates a Kubernetes `Job` (built from `spec.driver`) whose pod runs `spark-submit` in client mode and therefore _is_ the Spark driver. + The driver starts the designated number of executors and removes them when the job is completed. +* Executor(s): responsible for executing the job itself. They are created by the driver via Spark's Kubernetes backend and connect back to the driver through a headless `Service` created by the operator. + +NOTE: Previous versions ran an additional `spark-submit` process in a dedicated pod that then created the driver pod. This is no longer the case; `spec.job` and `spec.mode` are deprecated and ignored. Create a Spark application by running: @@ -27,22 +29,21 @@ include::example$getting_started/application.yaml[Create a Spark application] ---- <1> `metadata.name` contains the name of the SparkApplication <2> `spec.sparkImage`: the image used by the job, driver and executor pods. This can be a custom image built by the user or an official Stackable image. Available official images are stored in the Stackable https://oci.stackable.tech/[image registry,window=_blank]. Information on how to browse the registry can be found xref:contributor:project-overview.adoc#docker-images[here,window=_blank]. -<3> `spec.mode`: only `cluster` is currently supported +<3> `spec.mode`: deprecated and ignored; the driver always runs in client mode internally. <4> `spec.mainApplicationFile`: the artifact (Java, Scala or Python) that forms the basis of the Spark job. This path is relative to the image, so in this case an example python script (that calculates the value of pi) is running: it is bundled with the Spark code and therefore already present in the job image -<5> `spec.job`: submit command specific settings. -<6> `spec.driver`: driver-specific settings. +<5> `spec.job`: deprecated and ignored. In previous versions this configured the dedicated `spark-submit` job. +<6> `spec.driver`: driver-specific settings. Used to build the driver `Job` pod. <7> `spec.executor`: executor-specific settings. == Verify that it works -As mentioned above, the SparkApplication that has just been created builds a `spark-submit` command and pass it to the driver Pod, which in turn creates executor Pods that run for the duration of the job before being clean up. +As mentioned above, the operator creates a driver `Job` whose pod runs `spark-submit` in client mode and thus acts as the Spark driver, which in turn creates executor Pods that run for the duration of the job before being cleaned up. A running process looks like this: image::getting_started/spark_running.png[Spark job] -* `pyspark-pi-xxxx`: this is the initializing job that creates the spark-submit command (named as `metadata.name` with a unique suffix) -* `pyspark-pi-xxxxxxx-driver`: the driver pod that drives the execution +* `pyspark-pi-xxxxxxx-driver`: the driver pod (created by the driver `Job`) that drives the execution * `pythonpi-xxxxxxxxx-exec-x`: the set of executors started by the driver (in the example `spec.executor.instances` was set to 3 which is why 3 executors are running) Job progress can be followed by issuing this command: diff --git a/docs/modules/spark-k8s/pages/usage-guide/examples.adoc b/docs/modules/spark-k8s/pages/usage-guide/examples.adoc index 8e9211e1..182198eb 100644 --- a/docs/modules/spark-k8s/pages/usage-guide/examples.adoc +++ b/docs/modules/spark-k8s/pages/usage-guide/examples.adoc @@ -6,7 +6,7 @@ The following examples have the following `spec` fields in common: * `version`: the current version is "1.0" * `sparkImage`: the docker image that is used by job, driver and executor pods. This can be provided by the user. -* `mode`: only `cluster` is currently supported +* `mode`: deprecated and ignored; the driver always runs in client mode internally * `mainApplicationFile`: the artifact (Java, Scala or Python) that forms the basis of the Spark job. * `args`: these are the arguments passed directly to the application. In the examples below it is e.g. the input path for part of the public New York taxi dataset. * `sparkConf`: these list spark configuration settings that are passed directly to `spark-submit` and which are best defined explicitly by the user. Since the `SparkApplication` "knows" that there is an external dependency (the s3 bucket where the data and/or the application is located) and how that dependency should be treated (i.e. what type of credential checks are required, if any), it is better to have these things declared together. diff --git a/extra/crds.yaml b/extra/crds.yaml index 040c7856..12dc6cd2 100644 --- a/extra/crds.yaml +++ b/extra/crds.yaml @@ -15,7 +15,7 @@ spec: scope: Namespaced versions: - additionalPrinterColumns: [] - name: v1alpha1 + name: v1alpha2 schema: openAPIV3Schema: description: Auto-generated derived type for SparkApplicationSpec via `CustomResource` @@ -35,6 +35,212 @@ spec: items: type: string type: array + deprecatedJob: + description: |- + Deprecated and ignored. + + In previous versions this configured the dedicated `spark-submit` Job. Since v1alpha2 the + operator launches the driver directly as a Kubernetes Job (built from `spec.driver`), so + there is no separate submit process to configure. This field is kept for backwards + compatibility but has no effect. + nullable: true + properties: + cliOverrides: + additionalProperties: + type: string + default: {} + type: object + config: + default: {} + properties: + affinity: + default: + nodeAffinity: null + nodeSelector: null + podAffinity: null + podAntiAffinity: null + description: |- + These configuration settings control + [Pod placement](https://docs.stackable.tech/home/nightly/concepts/operations/pod_placement). + properties: + nodeAffinity: + description: Same as the `spec.affinity.nodeAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + nodeSelector: + additionalProperties: + type: string + description: Simple key-value pairs forming a nodeSelector, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true + type: object + podAffinity: + description: Same as the `spec.affinity.podAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + podAntiAffinity: + description: Same as the `spec.affinity.podAntiAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + requestedSecretLifetime: + description: |- + Request secret (currently only autoTls certificates) lifetime from the secret operator, e.g. `7d`, or `30d`. + This can be shortened by the `maxCertificateLifetime` setting on the SecretClass issuing the TLS certificate. + nullable: true + type: string + resources: + default: + cpu: + max: null + min: null + memory: + limit: null + runtimeLimits: {} + storage: {} + description: |- + Resource usage is configured here, this includes CPU usage, memory usage and disk storage + usage, if this role needs any. + properties: + cpu: + default: + max: null + min: null + properties: + max: + description: |- + The maximum amount of CPU cores that can be requested by Pods. + Equivalent to the `limit` for Pod resource configuration. + Cores are specified either as a decimal point number or as milli units. + For example:`1.5` will be 1.5 cores, also written as `1500m`. + nullable: true + x-kubernetes-int-or-string: true + min: + description: |- + The minimal amount of CPU cores that Pods need to run. + Equivalent to the `request` for Pod resource configuration. + Cores are specified either as a decimal point number or as milli units. + For example:`1.5` will be 1.5 cores, also written as `1500m`. + nullable: true + x-kubernetes-int-or-string: true + type: object + memory: + properties: + limit: + description: |- + The maximum amount of memory that should be available to the Pod. + Specified as a byte [Quantity](https://kubernetes.io/docs/reference/kubernetes-api/common-definitions/quantity/), + which means these suffixes are supported: E, P, T, G, M, k. + You can also use the power-of-two equivalents: Ei, Pi, Ti, Gi, Mi, Ki. + For example, the following represent roughly the same value: + `128974848, 129e6, 129M, 128974848000m, 123Mi` + nullable: true + x-kubernetes-int-or-string: true + runtimeLimits: + description: Additional options that can be specified. + type: object + type: object + storage: + type: object + type: object + retryOnFailureCount: + description: Number of times to retry the submit job on failure. Default is `0` (no retries). + format: uint16 + maximum: 65535.0 + minimum: 0.0 + nullable: true + type: integer + volumeMounts: + description: Volume mounts for the spark-submit, driver and executor pods. + items: + type: object + x-kubernetes-preserve-unknown-fields: true + type: array + type: object + configOverrides: + description: |- + The `configOverrides` can be used to configure properties in product config files + that are not exposed in the CRD. Read the + [config overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#config-overrides) + and consult the operator specific usage guide documentation for details on the + available config files and settings for the specific product. + properties: + security.properties: + additionalProperties: + type: string + description: |- + Flat key-value overrides for `*.properties`, Hadoop XML, etc. + + This is backwards-compatible with the existing flat key-value YAML format + used by `HashMap`. + nullable: true + type: object + spark-env.sh: + additionalProperties: + type: string + description: |- + Flat key-value overrides for `*.properties`, Hadoop XML, etc. + + This is backwards-compatible with the existing flat key-value YAML format + used by `HashMap`. + nullable: true + type: object + type: object + envOverrides: + additionalProperties: + type: string + default: {} + description: |- + `envOverrides` configure environment variables to be set in the Pods. + It is a map from strings to strings - environment variables and the value to set. + Read the + [environment variable overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#env-overrides) + for more information and consult the operator specific usage guide to find out about + the product specific environment variables that are available. + type: object + jvmArgumentOverrides: + default: + add: [] + remove: [] + removeRegex: [] + description: |- + Allows overriding JVM arguments. + Please read on the [JVM argument overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#jvm-argument-overrides) + for details on the usage. + properties: + add: + default: [] + description: JVM arguments to be added + items: + type: string + type: array + remove: + default: [] + description: JVM arguments to be removed by exact match + items: + type: string + type: array + removeRegex: + default: [] + description: JVM arguments matching any of this regexes will be removed + items: + type: string + type: array + type: object + podOverrides: + default: {} + description: |- + In the `podOverrides` property you can define a + [PodTemplateSpec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.34/#podtemplatespec-v1-core) + to override any property that can be set on a Kubernetes Pod. + Read the + [Pod overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#pod-overrides) + for more information. + type: object + x-kubernetes-preserve-unknown-fields: true + type: object deps: default: excludePackages: [] @@ -1572,231 +1778,27 @@ spec: See the [examples](https://docs.stackable.tech/home/nightly/spark-k8s/usage-guide/examples). nullable: true type: string - job: - description: |- - The job builds a spark-submit command, complete with arguments and referenced dependencies - such as templates, and passes it on to Spark. - The reason this property uses its own type (SubmitConfigFragment) is because logging is not - supported for spark-submit processes. + logFileDirectory: + description: The log file directory definition used by the Spark history server. nullable: true + oneOf: + - required: + - s3 + - required: + - customLogDirectory properties: - cliOverrides: - additionalProperties: - type: string - default: {} - type: object - config: - default: {} + customLogDirectory: + description: A custom log directory + type: string + s3: + description: An S3 bucket storing the log events properties: - affinity: - default: - nodeAffinity: null - nodeSelector: null - podAffinity: null - podAntiAffinity: null - description: |- - These configuration settings control - [Pod placement](https://docs.stackable.tech/home/nightly/concepts/operations/pod_placement). - properties: - nodeAffinity: - description: Same as the `spec.affinity.nodeAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) - nullable: true - type: object - x-kubernetes-preserve-unknown-fields: true - nodeSelector: - additionalProperties: - type: string - description: Simple key-value pairs forming a nodeSelector, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) - nullable: true - type: object - podAffinity: - description: Same as the `spec.affinity.podAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) - nullable: true - type: object - x-kubernetes-preserve-unknown-fields: true - podAntiAffinity: - description: Same as the `spec.affinity.podAntiAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) - nullable: true - type: object - x-kubernetes-preserve-unknown-fields: true - type: object - requestedSecretLifetime: - description: |- - Request secret (currently only autoTls certificates) lifetime from the secret operator, e.g. `7d`, or `30d`. - This can be shortened by the `maxCertificateLifetime` setting on the SecretClass issuing the TLS certificate. - nullable: true - type: string - resources: - default: - cpu: - max: null - min: null - memory: - limit: null - runtimeLimits: {} - storage: {} - description: |- - Resource usage is configured here, this includes CPU usage, memory usage and disk storage - usage, if this role needs any. - properties: - cpu: - default: - max: null - min: null - properties: - max: - description: |- - The maximum amount of CPU cores that can be requested by Pods. - Equivalent to the `limit` for Pod resource configuration. - Cores are specified either as a decimal point number or as milli units. - For example:`1.5` will be 1.5 cores, also written as `1500m`. - nullable: true - x-kubernetes-int-or-string: true - min: - description: |- - The minimal amount of CPU cores that Pods need to run. - Equivalent to the `request` for Pod resource configuration. - Cores are specified either as a decimal point number or as milli units. - For example:`1.5` will be 1.5 cores, also written as `1500m`. - nullable: true - x-kubernetes-int-or-string: true - type: object - memory: - properties: - limit: - description: |- - The maximum amount of memory that should be available to the Pod. - Specified as a byte [Quantity](https://kubernetes.io/docs/reference/kubernetes-api/common-definitions/quantity/), - which means these suffixes are supported: E, P, T, G, M, k. - You can also use the power-of-two equivalents: Ei, Pi, Ti, Gi, Mi, Ki. - For example, the following represent roughly the same value: - `128974848, 129e6, 129M, 128974848000m, 123Mi` - nullable: true - x-kubernetes-int-or-string: true - runtimeLimits: - description: Additional options that can be specified. - type: object - type: object - storage: - type: object - type: object - retryOnFailureCount: - description: Number of times to retry the submit job on failure. Default is `0` (no retries). - format: uint16 - maximum: 65535.0 - minimum: 0.0 - nullable: true - type: integer - volumeMounts: - description: Volume mounts for the spark-submit, driver and executor pods. - items: - type: object - x-kubernetes-preserve-unknown-fields: true - type: array - type: object - configOverrides: - description: |- - The `configOverrides` can be used to configure properties in product config files - that are not exposed in the CRD. Read the - [config overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#config-overrides) - and consult the operator specific usage guide documentation for details on the - available config files and settings for the specific product. - properties: - security.properties: - additionalProperties: - type: string - description: |- - Flat key-value overrides for `*.properties`, Hadoop XML, etc. - - This is backwards-compatible with the existing flat key-value YAML format - used by `HashMap`. - nullable: true - type: object - spark-env.sh: - additionalProperties: - type: string - description: |- - Flat key-value overrides for `*.properties`, Hadoop XML, etc. - - This is backwards-compatible with the existing flat key-value YAML format - used by `HashMap`. - nullable: true - type: object - type: object - envOverrides: - additionalProperties: - type: string - default: {} - description: |- - `envOverrides` configure environment variables to be set in the Pods. - It is a map from strings to strings - environment variables and the value to set. - Read the - [environment variable overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#env-overrides) - for more information and consult the operator specific usage guide to find out about - the product specific environment variables that are available. - type: object - jvmArgumentOverrides: - default: - add: [] - remove: [] - removeRegex: [] - description: |- - Allows overriding JVM arguments. - Please read on the [JVM argument overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#jvm-argument-overrides) - for details on the usage. - properties: - add: - default: [] - description: JVM arguments to be added - items: - type: string - type: array - remove: - default: [] - description: JVM arguments to be removed by exact match - items: - type: string - type: array - removeRegex: - default: [] - description: JVM arguments matching any of this regexes will be removed - items: - type: string - type: array - type: object - podOverrides: - default: {} - description: |- - In the `podOverrides` property you can define a - [PodTemplateSpec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.34/#podtemplatespec-v1-core) - to override any property that can be set on a Kubernetes Pod. - Read the - [Pod overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#pod-overrides) - for more information. - type: object - x-kubernetes-preserve-unknown-fields: true - type: object - logFileDirectory: - description: The log file directory definition used by the Spark history server. - nullable: true - oneOf: - - required: - - s3 - - required: - - customLogDirectory - properties: - customLogDirectory: - description: A custom log directory - type: string - s3: - description: An S3 bucket storing the log events - properties: - bucket: - oneOf: - - required: - - inline - - required: - - reference + bucket: + oneOf: + - required: + - inline + - required: + - reference properties: inline: description: |- @@ -1976,7 +1978,13 @@ spec: nullable: true type: string mode: - description: 'Mode: cluster or client. Currently only cluster is supported.' + description: |- + Deprecated and ignored. + + Since v1alpha2 the operator no longer runs a separate `spark-submit` process. The driver + is launched directly as a Kubernetes Job running in client mode, so the deploy mode is + always client internally. This field is kept for backwards compatibility but has no + effect. enum: - cluster - client @@ -2222,12 +2230,24 @@ spec: - sparkImage type: object status: + description: |- + Status of a SparkApplication. + + Defined outside the `versioned` module because it does not change between CRD versions and is + shared by all versions as the status subresource. nullable: true properties: phase: type: string resolvedTemplateRef: items: + description: |- + A reference to a [`SparkApplicationTemplate`](template_spec::SparkApplicationTemplate) that has + been merged into a SparkApplication. Stored in the application status. + + This type is intentionally defined outside the `versioned` module because it does not change + between CRD versions and is nested inside a `Vec` in the status, which the auto-generated + version conversions cannot handle for versioned types. properties: name: type: string @@ -2249,292 +2269,70 @@ spec: storage: true subresources: status: {} ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: sparkhistoryservers.spark.stackable.tech -spec: - group: spark.stackable.tech - names: - categories: [] - kind: SparkHistoryServer - plural: sparkhistoryservers - shortNames: - - sparkhist - - sparkhistory - - shs - singular: sparkhistoryserver - scope: Namespaced - versions: - additionalPrinterColumns: [] name: v1alpha1 schema: openAPIV3Schema: - description: Auto-generated derived type for SparkHistoryServerSpec via `CustomResource` + description: Auto-generated derived type for SparkApplicationSpec via `CustomResource` properties: spec: description: |- - A Spark cluster history server component. This resource is managed by the Stackable operator - for Apache Spark. Find more information on how to use it in the - [operator documentation](https://docs.stackable.tech/home/nightly/spark-k8s/usage-guide/history-server). + A Spark application run on Kubernetes by the Stackable operator for Apache Spark. + Find more information on how to use it and the resources that the operator generates in the + [operator documentation](https://docs.stackable.tech/home/nightly/spark-k8s/). + + The SparkApplication CRD looks a little different than the CRDs of the other products on the + Stackable Data Platform. properties: - image: - anyOf: - - required: - - custom - - productVersion - - required: - - productVersion + args: + default: [] + description: Arguments passed directly to the job artifact. + items: + type: string + type: array + deps: + default: + excludePackages: [] + packages: [] + repositories: [] + requirements: [] description: |- - Specify which image to use, the easiest way is to only configure the `productVersion`. - You can also configure a custom image registry to pull from, as well as completely custom - images. - - Consult the [Product image selection documentation](https://docs.stackable.tech/home/nightly/concepts/product_image_selection) - for details. + Job dependencies: a list of python packages that will be installed via pip, a list of packages + or repositories that is passed directly to spark-submit, or a list of excluded packages + (also passed directly to spark-submit). properties: - custom: - description: |- - Provide a custom container image. - - Specify the full container image name, e.g. `oci.example.tech/namespace/superset:1.4.1-my-tag` - type: string - productVersion: - description: Version of the product, e.g. `1.4.1`. - type: string - pullPolicy: - default: Always - description: '[Pull policy](https://kubernetes.io/docs/concepts/containers/images/#image-pull-policy) used when pulling the image.' - enum: - - IfNotPresent - - Always - - Never - type: string - pullSecrets: - description: '[Image pull secrets](https://kubernetes.io/docs/concepts/containers/images/#specifying-imagepullsecrets-on-a-pod) to pull images from a private registry.' + excludePackages: + default: [] + description: A list of excluded packages that is passed directly to `spark-submit`. items: - description: LocalObjectReference contains enough information to let you locate the referenced object inside the same namespace. - properties: - name: - description: 'Name of the referent. This field is effectively required, but due to backwards compatibility is allowed to be empty. Instances of this type with an empty value here are almost certainly wrong. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' - type: string - required: - - name - type: object - nullable: true + type: string type: array - repo: - description: |- - The repository on the container image registry where the container image is located, e.g. - `oci.example.com/namespace`. - - If not specified, the operator will use the image registry provided via the operator - environment options. - nullable: true - type: string - stackableVersion: + packages: + default: [] + description: A list of packages that is passed directly to `spark-submit`. + items: + type: string + type: array + repositories: + default: [] + description: A list of repositories that is passed directly to `spark-submit`. + items: + type: string + type: array + requirements: + default: [] description: |- - Stackable version of the product, e.g. `23.4`, `23.4.1` or `0.0.0-dev`. - - If not specified, the operator will use its own version, e.g. `23.4.1`. When using a nightly - operator or a PR version, it will use the nightly `0.0.0-dev` image. - nullable: true - type: string - type: object - logFileDirectory: - description: The log file directory definition used by the Spark history server. - oneOf: - - required: - - s3 - - required: - - customLogDirectory - properties: - customLogDirectory: - description: A custom log directory - type: string - s3: - description: An S3 bucket storing the log events - properties: - bucket: - oneOf: - - required: - - inline - - required: - - reference - properties: - inline: - description: |- - S3 bucket specification containing the bucket name and an inlined or referenced connection specification. - Learn more on the [S3 concept documentation](https://docs.stackable.tech/home/nightly/concepts/s3). - properties: - bucketName: - description: The name of the S3 bucket. - type: string - connection: - description: The definition of an S3 connection, either inline or as a reference. - oneOf: - - required: - - inline - - required: - - reference - properties: - inline: - description: |- - S3 connection definition as a resource. - Learn more on the [S3 concept documentation](https://docs.stackable.tech/home/nightly/concepts/s3). - properties: - accessStyle: - default: VirtualHosted - description: |- - Which access style to use. - Defaults to virtual hosted-style as most of the data products out there. - Have a look at the [AWS documentation](https://docs.aws.amazon.com/AmazonS3/latest/userguide/VirtualHosting.html). - enum: - - Path - - VirtualHosted - type: string - credentials: - description: |- - If the S3 uses authentication you have to specify you S3 credentials. - In the most cases a [SecretClass](https://docs.stackable.tech/home/nightly/secret-operator/secretclass) - providing `accessKey` and `secretKey` is sufficient. - nullable: true - properties: - scope: - description: |- - [Scope](https://docs.stackable.tech/home/nightly/secret-operator/scope) of the - [SecretClass](https://docs.stackable.tech/home/nightly/secret-operator/secretclass). - nullable: true - properties: - listenerVolumes: - default: [] - description: |- - The listener volume scope allows Node and Service scopes to be inferred from the applicable listeners. - This must correspond to Volume names in the Pod that mount Listeners. - items: - type: string - type: array - node: - default: false - description: |- - The node scope is resolved to the name of the Kubernetes Node object that the Pod is running on. - This will typically be the DNS name of the node. - type: boolean - pod: - default: false - description: |- - The pod scope is resolved to the name of the Kubernetes Pod. - This allows the secret to differentiate between StatefulSet replicas. - type: boolean - services: - default: [] - description: |- - The service scope allows Pod objects to specify custom scopes. - This should typically correspond to Service objects that the Pod participates in. - items: - type: string - type: array - type: object - secretClass: - description: '[SecretClass](https://docs.stackable.tech/home/nightly/secret-operator/secretclass) containing the LDAP bind credentials.' - type: string - required: - - secretClass - type: object - host: - description: 'Host of the S3 server without any protocol or port. For example: `west1.my-cloud.com`.' - type: string - port: - description: |- - Port the S3 server listens on. - If not specified the product will determine the port to use. - format: uint16 - maximum: 65535.0 - minimum: 0.0 - nullable: true - type: integer - region: - default: - name: us-east-1 - description: |- - Bucket region used for signing headers (sigv4). - - This defaults to `us-east-1` which is compatible with other implementations such as Minio. - - WARNING: Some products use the Hadoop S3 implementation which falls back to us-east-2. - properties: - name: - default: us-east-1 - type: string - type: object - tls: - description: Use a TLS connection. If not specified no TLS will be used. - nullable: true - properties: - verification: - description: The verification method used to verify the certificates of the server and/or the client. - oneOf: - - required: - - none - - required: - - server - properties: - none: - description: Use TLS but don't verify certificates. - type: object - server: - description: Use TLS and a CA certificate to verify the server. - properties: - caCert: - description: CA cert to verify the server. - oneOf: - - required: - - webPki - - required: - - secretClass - properties: - secretClass: - description: |- - Name of the [SecretClass](https://docs.stackable.tech/home/nightly/secret-operator/secretclass) which will provide the CA certificate. - Note that a SecretClass does not need to have a key but can also work with just a CA certificate, - so if you got provided with a CA cert but don't have access to the key you can still use this method. - type: string - webPki: - description: |- - Use TLS and the CA certificates trusted by the common web browsers to verify the server. - This can be useful when you e.g. use public AWS S3 or other public available services. - type: object - type: object - required: - - caCert - type: object - type: object - required: - - verification - type: object - required: - - host - type: object - reference: - type: string - type: object - required: - - bucketName - - connection - type: object - reference: - type: string - type: object - prefix: - type: string - required: - - bucket - - prefix - type: object + Under the `requirements` you can specify Python dependencies that will be installed with `pip`. + Example: `tabulate==0.8.9` + items: + type: string + type: array type: object - nodes: - description: A history server node role definition. + driver: + description: |- + The driver role specifies the configuration that, together with the driver pod template, is used by + Spark to create driver pods. + nullable: true properties: cliOverrides: additionalProperties: @@ -2576,9 +2374,6 @@ spec: type: object x-kubernetes-preserve-unknown-fields: true type: object - cleaner: - nullable: true - type: boolean logging: default: containers: {} @@ -2588,7 +2383,7 @@ spec: containers: description: Log configuration per container. properties: - spark-history: + job: anyOf: - required: - custom @@ -2668,7 +2463,7 @@ spec: description: Configuration per logger type: object type: object - vector: + requirements: anyOf: - required: - custom @@ -2748,1075 +2543,5814 @@ spec: description: Configuration per logger type: object type: object - type: object - enableVectorAgent: - description: Whether or not to deploy a container with the Vector log agent. - nullable: true - type: boolean - type: object - requestedSecretLifetime: - description: |- - Request secret (currently only autoTls certificates) lifetime from the secret operator, e.g. `7d`, or `30d`. - This can be shortened by the `maxCertificateLifetime` setting on the SecretClass issuing the TLS certificate. - nullable: true - type: string - resources: - default: - cpu: - max: null - min: null - memory: - limit: null - runtimeLimits: {} - storage: {} - description: |- - Resource usage is configured here, this includes CPU usage, memory usage and disk storage - usage, if this role needs any. - properties: - cpu: - default: - max: null - min: null - properties: - max: - description: |- - The maximum amount of CPU cores that can be requested by Pods. - Equivalent to the `limit` for Pod resource configuration. - Cores are specified either as a decimal point number or as milli units. - For example:`1.5` will be 1.5 cores, also written as `1500m`. - nullable: true - x-kubernetes-int-or-string: true - min: - description: |- - The minimal amount of CPU cores that Pods need to run. - Equivalent to the `request` for Pod resource configuration. - Cores are specified either as a decimal point number or as milli units. - For example:`1.5` will be 1.5 cores, also written as `1500m`. - nullable: true - x-kubernetes-int-or-string: true - type: object - memory: - properties: - limit: - description: |- - The maximum amount of memory that should be available to the Pod. - Specified as a byte [Quantity](https://kubernetes.io/docs/reference/kubernetes-api/common-definitions/quantity/), - which means these suffixes are supported: E, P, T, G, M, k. - You can also use the power-of-two equivalents: Ei, Pi, Ti, Gi, Mi, Ki. - For example, the following represent roughly the same value: - `128974848, 129e6, 129M, 128974848000m, 123Mi` - nullable: true - x-kubernetes-int-or-string: true - runtimeLimits: - description: Additional options that can be specified. + spark: + anyOf: + - required: + - custom + - {} + - {} + description: Log configuration of the container + properties: + console: + description: Configuration for the console appender + nullable: true + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + custom: + description: Log configuration provided in a ConfigMap + properties: + configMap: + description: ConfigMap containing the log configuration files + nullable: true + type: string + type: object + file: + description: Configuration for the file appender + nullable: true + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + loggers: + additionalProperties: + description: Configuration of a logger + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + default: {} + description: Configuration per logger + type: object type: object - type: object - storage: - type: object - type: object - type: object - configOverrides: - description: |- - The `configOverrides` can be used to configure properties in product config files - that are not exposed in the CRD. Read the - [config overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#config-overrides) - and consult the operator specific usage guide documentation for details on the - available config files and settings for the specific product. - properties: - security.properties: - additionalProperties: - type: string - description: |- - Flat key-value overrides for `*.properties`, Hadoop XML, etc. - - This is backwards-compatible with the existing flat key-value YAML format - used by `HashMap`. - nullable: true - type: object - spark-defaults.conf: - additionalProperties: - type: string - description: |- - Flat key-value overrides for `*.properties`, Hadoop XML, etc. - - This is backwards-compatible with the existing flat key-value YAML format - used by `HashMap`. - nullable: true - type: object - spark-env.sh: - additionalProperties: - type: string - description: |- - Flat key-value overrides for `*.properties`, Hadoop XML, etc. - - This is backwards-compatible with the existing flat key-value YAML format - used by `HashMap`. - nullable: true - type: object - type: object - envOverrides: - additionalProperties: - type: string - default: {} - description: |- - `envOverrides` configure environment variables to be set in the Pods. - It is a map from strings to strings - environment variables and the value to set. - Read the - [environment variable overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#env-overrides) - for more information and consult the operator specific usage guide to find out about - the product specific environment variables that are available. - type: object - jvmArgumentOverrides: - default: - add: [] - remove: [] - removeRegex: [] - description: |- - Allows overriding JVM arguments. - Please read on the [JVM argument overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#jvm-argument-overrides) - for details on the usage. - properties: - add: - default: [] - description: JVM arguments to be added - items: - type: string - type: array - remove: - default: [] - description: JVM arguments to be removed by exact match - items: - type: string - type: array - removeRegex: - default: [] - description: JVM arguments matching any of this regexes will be removed - items: - type: string - type: array - type: object - podOverrides: - default: {} - description: |- - In the `podOverrides` property you can define a - [PodTemplateSpec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.34/#podtemplatespec-v1-core) - to override any property that can be set on a Kubernetes Pod. - Read the - [Pod overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#pod-overrides) - for more information. - type: object - x-kubernetes-preserve-unknown-fields: true - roleConfig: - default: - listenerClass: cluster-internal - podDisruptionBudget: - enabled: true - maxUnavailable: null - description: This is a product-agnostic RoleConfig, which is sufficient for most of the products. - properties: - listenerClass: - default: cluster-internal - description: |- - This field controls which [ListenerClass](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass.html) - is used to expose the history server. - type: string - podDisruptionBudget: - default: - enabled: true + spark-submit: + anyOf: + - required: + - custom + - {} + - {} + description: Log configuration of the container + properties: + console: + description: Configuration for the console appender + nullable: true + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + custom: + description: Log configuration provided in a ConfigMap + properties: + configMap: + description: ConfigMap containing the log configuration files + nullable: true + type: string + type: object + file: + description: Configuration for the file appender + nullable: true + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + loggers: + additionalProperties: + description: Configuration of a logger + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + default: {} + description: Configuration per logger + type: object + type: object + tls: + anyOf: + - required: + - custom + - {} + - {} + description: Log configuration of the container + properties: + console: + description: Configuration for the console appender + nullable: true + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + custom: + description: Log configuration provided in a ConfigMap + properties: + configMap: + description: ConfigMap containing the log configuration files + nullable: true + type: string + type: object + file: + description: Configuration for the file appender + nullable: true + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + loggers: + additionalProperties: + description: Configuration of a logger + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + default: {} + description: Configuration per logger + type: object + type: object + vector: + anyOf: + - required: + - custom + - {} + - {} + description: Log configuration of the container + properties: + console: + description: Configuration for the console appender + nullable: true + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + custom: + description: Log configuration provided in a ConfigMap + properties: + configMap: + description: ConfigMap containing the log configuration files + nullable: true + type: string + type: object + file: + description: Configuration for the file appender + nullable: true + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + loggers: + additionalProperties: + description: Configuration of a logger + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + default: {} + description: Configuration per logger + type: object + type: object + type: object + enableVectorAgent: + description: Whether or not to deploy a container with the Vector log agent. + nullable: true + type: boolean + type: object + requestedSecretLifetime: + description: |- + Request secret (currently only autoTls certificates) lifetime from the secret operator, e.g. `7d`, or `30d`. + This can be shortened by the `maxCertificateLifetime` setting on the SecretClass issuing the TLS certificate. + nullable: true + type: string + resources: + default: + cpu: + max: null + min: null + memory: + limit: null + runtimeLimits: {} + storage: {} + description: |- + Resource usage is configured here, this includes CPU usage, memory usage and disk storage + usage, if this role needs any. + properties: + cpu: + default: + max: null + min: null + properties: + max: + description: |- + The maximum amount of CPU cores that can be requested by Pods. + Equivalent to the `limit` for Pod resource configuration. + Cores are specified either as a decimal point number or as milli units. + For example:`1.5` will be 1.5 cores, also written as `1500m`. + nullable: true + x-kubernetes-int-or-string: true + min: + description: |- + The minimal amount of CPU cores that Pods need to run. + Equivalent to the `request` for Pod resource configuration. + Cores are specified either as a decimal point number or as milli units. + For example:`1.5` will be 1.5 cores, also written as `1500m`. + nullable: true + x-kubernetes-int-or-string: true + type: object + memory: + properties: + limit: + description: |- + The maximum amount of memory that should be available to the Pod. + Specified as a byte [Quantity](https://kubernetes.io/docs/reference/kubernetes-api/common-definitions/quantity/), + which means these suffixes are supported: E, P, T, G, M, k. + You can also use the power-of-two equivalents: Ei, Pi, Ti, Gi, Mi, Ki. + For example, the following represent roughly the same value: + `128974848, 129e6, 129M, 128974848000m, 123Mi` + nullable: true + x-kubernetes-int-or-string: true + runtimeLimits: + description: Additional options that can be specified. + type: object + type: object + storage: + type: object + type: object + volumeMounts: + description: Volume mounts for the spark-submit, driver and executor pods. + items: + type: object + x-kubernetes-preserve-unknown-fields: true + type: array + type: object + configOverrides: + description: |- + The `configOverrides` can be used to configure properties in product config files + that are not exposed in the CRD. Read the + [config overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#config-overrides) + and consult the operator specific usage guide documentation for details on the + available config files and settings for the specific product. + properties: + security.properties: + additionalProperties: + type: string + description: |- + Flat key-value overrides for `*.properties`, Hadoop XML, etc. + + This is backwards-compatible with the existing flat key-value YAML format + used by `HashMap`. + nullable: true + type: object + spark-env.sh: + additionalProperties: + type: string + description: |- + Flat key-value overrides for `*.properties`, Hadoop XML, etc. + + This is backwards-compatible with the existing flat key-value YAML format + used by `HashMap`. + nullable: true + type: object + type: object + envOverrides: + additionalProperties: + type: string + default: {} + description: |- + `envOverrides` configure environment variables to be set in the Pods. + It is a map from strings to strings - environment variables and the value to set. + Read the + [environment variable overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#env-overrides) + for more information and consult the operator specific usage guide to find out about + the product specific environment variables that are available. + type: object + jvmArgumentOverrides: + default: + add: [] + remove: [] + removeRegex: [] + description: |- + Allows overriding JVM arguments. + Please read on the [JVM argument overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#jvm-argument-overrides) + for details on the usage. + properties: + add: + default: [] + description: JVM arguments to be added + items: + type: string + type: array + remove: + default: [] + description: JVM arguments to be removed by exact match + items: + type: string + type: array + removeRegex: + default: [] + description: JVM arguments matching any of this regexes will be removed + items: + type: string + type: array + type: object + podOverrides: + default: {} + description: |- + In the `podOverrides` property you can define a + [PodTemplateSpec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.34/#podtemplatespec-v1-core) + to override any property that can be set on a Kubernetes Pod. + Read the + [Pod overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#pod-overrides) + for more information. + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + env: + default: [] + description: |- + A list of environment variables that will be set in the job pod and the driver and executor + pod templates. + items: + description: EnvVar represents an environment variable present in a Container. + properties: + name: + description: Name of the environment variable. May consist of any printable ASCII characters except '='. + type: string + value: + description: 'Variable references $(VAR_NAME) are expanded using the previously defined environment variables in the container and any service environment variables. If a variable cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless of whether the variable exists or not. Defaults to "".' + type: string + valueFrom: + description: Source for the environment variable's value. Cannot be used if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + description: 'Name of the referent. This field is effectively required, but due to backwards compatibility is allowed to be empty. Instances of this type with an empty value here are almost certainly wrong. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + optional: + description: Specify whether the ConfigMap or its key must be defined + type: boolean + required: + - key + - name + type: object + fieldRef: + description: 'Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['''']`, `metadata.annotations['''']`, spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs.' + properties: + apiVersion: + description: Version of the schema the FieldPath is written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select in the specified API version. + type: string + required: + - fieldPath + type: object + fileKeyRef: + description: FileKeyRef selects a key of the env file. Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env file. An invalid key will prevent the pod from starting. The keys defined within a source may consist of any printable ASCII characters except '='. During Alpha stage of the EnvFiles feature gate, the key size is limited to 128 characters. + type: string + optional: + description: |- + Specify whether the file or its key must be defined. If the file or key does not exist, then the env var is not published. If optional is set to true and the specified key does not exist, the environment variable will not be set in the Pod's containers. + + If optional is set to false and the specified key does not exist, an error will be returned during Pod creation. + type: boolean + path: + description: The path within the volume from which to select the file. Must be relative and may not contain the '..' path or start with '..'. + type: string + volumeName: + description: The name of the volume mount containing the env file. + type: string + required: + - key + - path + - volumeName + type: object + resourceFieldRef: + description: 'Selects a resource of the container: only resources limits and requests (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported.' + properties: + containerName: + description: 'Container name: required for volumes, optional for env vars' + type: string + divisor: + description: Specifies the output format of the exposed resources, defaults to "1" + nullable: true + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + secretKeyRef: + description: Selects a key of a secret in the pod's namespace + properties: + key: + description: The key of the secret to select from. Must be a valid secret key. + type: string + name: + description: 'Name of the referent. This field is effectively required, but due to backwards compatibility is allowed to be empty. Instances of this type with an empty value here are almost certainly wrong. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + optional: + description: Specify whether the Secret or its key must be defined + type: boolean + required: + - key + - name + type: object + type: object + required: + - name + type: object + type: array + executor: + description: |- + The executor role specifies the configuration that, together with the driver pod template, is used by + Spark to create the executor pods. + This is RoleGroup instead of plain CommonConfiguration because it needs to allow for the number of replicas. + to be specified. + nullable: true + properties: + cliOverrides: + additionalProperties: + type: string + default: {} + type: object + config: + default: {} + properties: + affinity: + default: + nodeAffinity: null + nodeSelector: null + podAffinity: null + podAntiAffinity: null + description: |- + These configuration settings control + [Pod placement](https://docs.stackable.tech/home/nightly/concepts/operations/pod_placement). + properties: + nodeAffinity: + description: Same as the `spec.affinity.nodeAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + nodeSelector: + additionalProperties: + type: string + description: Simple key-value pairs forming a nodeSelector, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true + type: object + podAffinity: + description: Same as the `spec.affinity.podAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + podAntiAffinity: + description: Same as the `spec.affinity.podAntiAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + logging: + default: + containers: {} + enableVectorAgent: null + description: Logging configuration, learn more in the [logging concept documentation](https://docs.stackable.tech/home/nightly/concepts/logging). + properties: + containers: + description: Log configuration per container. + properties: + job: + anyOf: + - required: + - custom + - {} + - {} + description: Log configuration of the container + properties: + console: + description: Configuration for the console appender + nullable: true + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + custom: + description: Log configuration provided in a ConfigMap + properties: + configMap: + description: ConfigMap containing the log configuration files + nullable: true + type: string + type: object + file: + description: Configuration for the file appender + nullable: true + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + loggers: + additionalProperties: + description: Configuration of a logger + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + default: {} + description: Configuration per logger + type: object + type: object + requirements: + anyOf: + - required: + - custom + - {} + - {} + description: Log configuration of the container + properties: + console: + description: Configuration for the console appender + nullable: true + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + custom: + description: Log configuration provided in a ConfigMap + properties: + configMap: + description: ConfigMap containing the log configuration files + nullable: true + type: string + type: object + file: + description: Configuration for the file appender + nullable: true + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + loggers: + additionalProperties: + description: Configuration of a logger + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + default: {} + description: Configuration per logger + type: object + type: object + spark: + anyOf: + - required: + - custom + - {} + - {} + description: Log configuration of the container + properties: + console: + description: Configuration for the console appender + nullable: true + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + custom: + description: Log configuration provided in a ConfigMap + properties: + configMap: + description: ConfigMap containing the log configuration files + nullable: true + type: string + type: object + file: + description: Configuration for the file appender + nullable: true + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + loggers: + additionalProperties: + description: Configuration of a logger + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + default: {} + description: Configuration per logger + type: object + type: object + spark-submit: + anyOf: + - required: + - custom + - {} + - {} + description: Log configuration of the container + properties: + console: + description: Configuration for the console appender + nullable: true + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + custom: + description: Log configuration provided in a ConfigMap + properties: + configMap: + description: ConfigMap containing the log configuration files + nullable: true + type: string + type: object + file: + description: Configuration for the file appender + nullable: true + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + loggers: + additionalProperties: + description: Configuration of a logger + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + default: {} + description: Configuration per logger + type: object + type: object + tls: + anyOf: + - required: + - custom + - {} + - {} + description: Log configuration of the container + properties: + console: + description: Configuration for the console appender + nullable: true + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + custom: + description: Log configuration provided in a ConfigMap + properties: + configMap: + description: ConfigMap containing the log configuration files + nullable: true + type: string + type: object + file: + description: Configuration for the file appender + nullable: true + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + loggers: + additionalProperties: + description: Configuration of a logger + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + default: {} + description: Configuration per logger + type: object + type: object + vector: + anyOf: + - required: + - custom + - {} + - {} + description: Log configuration of the container + properties: + console: + description: Configuration for the console appender + nullable: true + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + custom: + description: Log configuration provided in a ConfigMap + properties: + configMap: + description: ConfigMap containing the log configuration files + nullable: true + type: string + type: object + file: + description: Configuration for the file appender + nullable: true + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + loggers: + additionalProperties: + description: Configuration of a logger + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + default: {} + description: Configuration per logger + type: object + type: object + type: object + enableVectorAgent: + description: Whether or not to deploy a container with the Vector log agent. + nullable: true + type: boolean + type: object + requestedSecretLifetime: + description: |- + Request secret (currently only autoTls certificates) lifetime from the secret operator, e.g. `7d`, or `30d`. + This can be shortened by the `maxCertificateLifetime` setting on the SecretClass issuing the TLS certificate. + nullable: true + type: string + resources: + default: + cpu: + max: null + min: null + memory: + limit: null + runtimeLimits: {} + storage: {} + description: |- + Resource usage is configured here, this includes CPU usage, memory usage and disk storage + usage, if this role needs any. + properties: + cpu: + default: + max: null + min: null + properties: + max: + description: |- + The maximum amount of CPU cores that can be requested by Pods. + Equivalent to the `limit` for Pod resource configuration. + Cores are specified either as a decimal point number or as milli units. + For example:`1.5` will be 1.5 cores, also written as `1500m`. + nullable: true + x-kubernetes-int-or-string: true + min: + description: |- + The minimal amount of CPU cores that Pods need to run. + Equivalent to the `request` for Pod resource configuration. + Cores are specified either as a decimal point number or as milli units. + For example:`1.5` will be 1.5 cores, also written as `1500m`. + nullable: true + x-kubernetes-int-or-string: true + type: object + memory: + properties: + limit: + description: |- + The maximum amount of memory that should be available to the Pod. + Specified as a byte [Quantity](https://kubernetes.io/docs/reference/kubernetes-api/common-definitions/quantity/), + which means these suffixes are supported: E, P, T, G, M, k. + You can also use the power-of-two equivalents: Ei, Pi, Ti, Gi, Mi, Ki. + For example, the following represent roughly the same value: + `128974848, 129e6, 129M, 128974848000m, 123Mi` + nullable: true + x-kubernetes-int-or-string: true + runtimeLimits: + description: Additional options that can be specified. + type: object + type: object + storage: + type: object + type: object + volumeMounts: + description: Volume mounts for the spark-submit, driver and executor pods. + items: + type: object + x-kubernetes-preserve-unknown-fields: true + type: array + type: object + configOverrides: + description: |- + The `configOverrides` can be used to configure properties in product config files + that are not exposed in the CRD. Read the + [config overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#config-overrides) + and consult the operator specific usage guide documentation for details on the + available config files and settings for the specific product. + properties: + security.properties: + additionalProperties: + type: string + description: |- + Flat key-value overrides for `*.properties`, Hadoop XML, etc. + + This is backwards-compatible with the existing flat key-value YAML format + used by `HashMap`. + nullable: true + type: object + spark-env.sh: + additionalProperties: + type: string + description: |- + Flat key-value overrides for `*.properties`, Hadoop XML, etc. + + This is backwards-compatible with the existing flat key-value YAML format + used by `HashMap`. + nullable: true + type: object + type: object + envOverrides: + additionalProperties: + type: string + default: {} + description: |- + `envOverrides` configure environment variables to be set in the Pods. + It is a map from strings to strings - environment variables and the value to set. + Read the + [environment variable overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#env-overrides) + for more information and consult the operator specific usage guide to find out about + the product specific environment variables that are available. + type: object + jvmArgumentOverrides: + default: + add: [] + remove: [] + removeRegex: [] + description: |- + Allows overriding JVM arguments. + Please read on the [JVM argument overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#jvm-argument-overrides) + for details on the usage. + properties: + add: + default: [] + description: JVM arguments to be added + items: + type: string + type: array + remove: + default: [] + description: JVM arguments to be removed by exact match + items: + type: string + type: array + removeRegex: + default: [] + description: JVM arguments matching any of this regexes will be removed + items: + type: string + type: array + type: object + podOverrides: + default: {} + description: |- + In the `podOverrides` property you can define a + [PodTemplateSpec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.34/#podtemplatespec-v1-core) + to override any property that can be set on a Kubernetes Pod. + Read the + [Pod overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#pod-overrides) + for more information. + type: object + x-kubernetes-preserve-unknown-fields: true + replicas: + format: uint16 + maximum: 65535.0 + minimum: 0.0 + nullable: true + type: integer + type: object + image: + description: |- + User-supplied image containing spark-job dependencies that will be copied to the specified volume mount. + See the [examples](https://docs.stackable.tech/home/nightly/spark-k8s/usage-guide/examples). + nullable: true + type: string + job: + description: |- + Deprecated and ignored. + + In previous versions this configured the dedicated `spark-submit` Job. Since v1alpha2 the + operator launches the driver directly as a Kubernetes Job (built from `spec.driver`), so + there is no separate submit process to configure. This field is kept for backwards + compatibility but has no effect. + nullable: true + properties: + cliOverrides: + additionalProperties: + type: string + default: {} + type: object + config: + default: {} + properties: + affinity: + default: + nodeAffinity: null + nodeSelector: null + podAffinity: null + podAntiAffinity: null + description: |- + These configuration settings control + [Pod placement](https://docs.stackable.tech/home/nightly/concepts/operations/pod_placement). + properties: + nodeAffinity: + description: Same as the `spec.affinity.nodeAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + nodeSelector: + additionalProperties: + type: string + description: Simple key-value pairs forming a nodeSelector, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true + type: object + podAffinity: + description: Same as the `spec.affinity.podAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + podAntiAffinity: + description: Same as the `spec.affinity.podAntiAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + requestedSecretLifetime: + description: |- + Request secret (currently only autoTls certificates) lifetime from the secret operator, e.g. `7d`, or `30d`. + This can be shortened by the `maxCertificateLifetime` setting on the SecretClass issuing the TLS certificate. + nullable: true + type: string + resources: + default: + cpu: + max: null + min: null + memory: + limit: null + runtimeLimits: {} + storage: {} + description: |- + Resource usage is configured here, this includes CPU usage, memory usage and disk storage + usage, if this role needs any. + properties: + cpu: + default: + max: null + min: null + properties: + max: + description: |- + The maximum amount of CPU cores that can be requested by Pods. + Equivalent to the `limit` for Pod resource configuration. + Cores are specified either as a decimal point number or as milli units. + For example:`1.5` will be 1.5 cores, also written as `1500m`. + nullable: true + x-kubernetes-int-or-string: true + min: + description: |- + The minimal amount of CPU cores that Pods need to run. + Equivalent to the `request` for Pod resource configuration. + Cores are specified either as a decimal point number or as milli units. + For example:`1.5` will be 1.5 cores, also written as `1500m`. + nullable: true + x-kubernetes-int-or-string: true + type: object + memory: + properties: + limit: + description: |- + The maximum amount of memory that should be available to the Pod. + Specified as a byte [Quantity](https://kubernetes.io/docs/reference/kubernetes-api/common-definitions/quantity/), + which means these suffixes are supported: E, P, T, G, M, k. + You can also use the power-of-two equivalents: Ei, Pi, Ti, Gi, Mi, Ki. + For example, the following represent roughly the same value: + `128974848, 129e6, 129M, 128974848000m, 123Mi` + nullable: true + x-kubernetes-int-or-string: true + runtimeLimits: + description: Additional options that can be specified. + type: object + type: object + storage: + type: object + type: object + retryOnFailureCount: + description: Number of times to retry the submit job on failure. Default is `0` (no retries). + format: uint16 + maximum: 65535.0 + minimum: 0.0 + nullable: true + type: integer + volumeMounts: + description: Volume mounts for the spark-submit, driver and executor pods. + items: + type: object + x-kubernetes-preserve-unknown-fields: true + type: array + type: object + configOverrides: + description: |- + The `configOverrides` can be used to configure properties in product config files + that are not exposed in the CRD. Read the + [config overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#config-overrides) + and consult the operator specific usage guide documentation for details on the + available config files and settings for the specific product. + properties: + security.properties: + additionalProperties: + type: string + description: |- + Flat key-value overrides for `*.properties`, Hadoop XML, etc. + + This is backwards-compatible with the existing flat key-value YAML format + used by `HashMap`. + nullable: true + type: object + spark-env.sh: + additionalProperties: + type: string + description: |- + Flat key-value overrides for `*.properties`, Hadoop XML, etc. + + This is backwards-compatible with the existing flat key-value YAML format + used by `HashMap`. + nullable: true + type: object + type: object + envOverrides: + additionalProperties: + type: string + default: {} + description: |- + `envOverrides` configure environment variables to be set in the Pods. + It is a map from strings to strings - environment variables and the value to set. + Read the + [environment variable overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#env-overrides) + for more information and consult the operator specific usage guide to find out about + the product specific environment variables that are available. + type: object + jvmArgumentOverrides: + default: + add: [] + remove: [] + removeRegex: [] + description: |- + Allows overriding JVM arguments. + Please read on the [JVM argument overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#jvm-argument-overrides) + for details on the usage. + properties: + add: + default: [] + description: JVM arguments to be added + items: + type: string + type: array + remove: + default: [] + description: JVM arguments to be removed by exact match + items: + type: string + type: array + removeRegex: + default: [] + description: JVM arguments matching any of this regexes will be removed + items: + type: string + type: array + type: object + podOverrides: + default: {} + description: |- + In the `podOverrides` property you can define a + [PodTemplateSpec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.34/#podtemplatespec-v1-core) + to override any property that can be set on a Kubernetes Pod. + Read the + [Pod overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#pod-overrides) + for more information. + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + logFileDirectory: + description: The log file directory definition used by the Spark history server. + nullable: true + oneOf: + - required: + - s3 + - required: + - customLogDirectory + properties: + customLogDirectory: + description: A custom log directory + type: string + s3: + description: An S3 bucket storing the log events + properties: + bucket: + oneOf: + - required: + - inline + - required: + - reference + properties: + inline: + description: |- + S3 bucket specification containing the bucket name and an inlined or referenced connection specification. + Learn more on the [S3 concept documentation](https://docs.stackable.tech/home/nightly/concepts/s3). + properties: + bucketName: + description: The name of the S3 bucket. + type: string + connection: + description: The definition of an S3 connection, either inline or as a reference. + oneOf: + - required: + - inline + - required: + - reference + properties: + inline: + description: |- + S3 connection definition as a resource. + Learn more on the [S3 concept documentation](https://docs.stackable.tech/home/nightly/concepts/s3). + properties: + accessStyle: + default: VirtualHosted + description: |- + Which access style to use. + Defaults to virtual hosted-style as most of the data products out there. + Have a look at the [AWS documentation](https://docs.aws.amazon.com/AmazonS3/latest/userguide/VirtualHosting.html). + enum: + - Path + - VirtualHosted + type: string + credentials: + description: |- + If the S3 uses authentication you have to specify you S3 credentials. + In the most cases a [SecretClass](https://docs.stackable.tech/home/nightly/secret-operator/secretclass) + providing `accessKey` and `secretKey` is sufficient. + nullable: true + properties: + scope: + description: |- + [Scope](https://docs.stackable.tech/home/nightly/secret-operator/scope) of the + [SecretClass](https://docs.stackable.tech/home/nightly/secret-operator/secretclass). + nullable: true + properties: + listenerVolumes: + default: [] + description: |- + The listener volume scope allows Node and Service scopes to be inferred from the applicable listeners. + This must correspond to Volume names in the Pod that mount Listeners. + items: + type: string + type: array + node: + default: false + description: |- + The node scope is resolved to the name of the Kubernetes Node object that the Pod is running on. + This will typically be the DNS name of the node. + type: boolean + pod: + default: false + description: |- + The pod scope is resolved to the name of the Kubernetes Pod. + This allows the secret to differentiate between StatefulSet replicas. + type: boolean + services: + default: [] + description: |- + The service scope allows Pod objects to specify custom scopes. + This should typically correspond to Service objects that the Pod participates in. + items: + type: string + type: array + type: object + secretClass: + description: '[SecretClass](https://docs.stackable.tech/home/nightly/secret-operator/secretclass) containing the LDAP bind credentials.' + type: string + required: + - secretClass + type: object + host: + description: 'Host of the S3 server without any protocol or port. For example: `west1.my-cloud.com`.' + type: string + port: + description: |- + Port the S3 server listens on. + If not specified the product will determine the port to use. + format: uint16 + maximum: 65535.0 + minimum: 0.0 + nullable: true + type: integer + region: + default: + name: us-east-1 + description: |- + Bucket region used for signing headers (sigv4). + + This defaults to `us-east-1` which is compatible with other implementations such as Minio. + + WARNING: Some products use the Hadoop S3 implementation which falls back to us-east-2. + properties: + name: + default: us-east-1 + type: string + type: object + tls: + description: Use a TLS connection. If not specified no TLS will be used. + nullable: true + properties: + verification: + description: The verification method used to verify the certificates of the server and/or the client. + oneOf: + - required: + - none + - required: + - server + properties: + none: + description: Use TLS but don't verify certificates. + type: object + server: + description: Use TLS and a CA certificate to verify the server. + properties: + caCert: + description: CA cert to verify the server. + oneOf: + - required: + - webPki + - required: + - secretClass + properties: + secretClass: + description: |- + Name of the [SecretClass](https://docs.stackable.tech/home/nightly/secret-operator/secretclass) which will provide the CA certificate. + Note that a SecretClass does not need to have a key but can also work with just a CA certificate, + so if you got provided with a CA cert but don't have access to the key you can still use this method. + type: string + webPki: + description: |- + Use TLS and the CA certificates trusted by the common web browsers to verify the server. + This can be useful when you e.g. use public AWS S3 or other public available services. + type: object + type: object + required: + - caCert + type: object + type: object + required: + - verification + type: object + required: + - host + type: object + reference: + type: string + type: object + required: + - bucketName + - connection + type: object + reference: + type: string + type: object + prefix: + type: string + required: + - bucket + - prefix + type: object + type: object + mainApplicationFile: + description: The actual application file that will be called by `spark-submit`. + type: string + mainClass: + description: The main class - i.e. entry point - for JVM artifacts. + nullable: true + type: string + mode: + description: |- + Deprecated and ignored. + + Since v1alpha2 the operator no longer runs a separate `spark-submit` process. The driver + is launched directly as a Kubernetes Job running in client mode, so the deploy mode is + always client internally. This field is kept for backwards compatibility but has no + effect. + enum: + - cluster + - client + type: string + s3connection: + description: |- + Configure an S3 connection that the SparkApplication has access to. + Read more in the [Spark S3 usage guide](https://docs.stackable.tech/home/nightly/spark-k8s/usage-guide/s3). + nullable: true + oneOf: + - required: + - inline + - required: + - reference + properties: + inline: + description: |- + S3 connection definition as a resource. + Learn more on the [S3 concept documentation](https://docs.stackable.tech/home/nightly/concepts/s3). + properties: + accessStyle: + default: VirtualHosted + description: |- + Which access style to use. + Defaults to virtual hosted-style as most of the data products out there. + Have a look at the [AWS documentation](https://docs.aws.amazon.com/AmazonS3/latest/userguide/VirtualHosting.html). + enum: + - Path + - VirtualHosted + type: string + credentials: + description: |- + If the S3 uses authentication you have to specify you S3 credentials. + In the most cases a [SecretClass](https://docs.stackable.tech/home/nightly/secret-operator/secretclass) + providing `accessKey` and `secretKey` is sufficient. + nullable: true + properties: + scope: + description: |- + [Scope](https://docs.stackable.tech/home/nightly/secret-operator/scope) of the + [SecretClass](https://docs.stackable.tech/home/nightly/secret-operator/secretclass). + nullable: true + properties: + listenerVolumes: + default: [] + description: |- + The listener volume scope allows Node and Service scopes to be inferred from the applicable listeners. + This must correspond to Volume names in the Pod that mount Listeners. + items: + type: string + type: array + node: + default: false + description: |- + The node scope is resolved to the name of the Kubernetes Node object that the Pod is running on. + This will typically be the DNS name of the node. + type: boolean + pod: + default: false + description: |- + The pod scope is resolved to the name of the Kubernetes Pod. + This allows the secret to differentiate between StatefulSet replicas. + type: boolean + services: + default: [] + description: |- + The service scope allows Pod objects to specify custom scopes. + This should typically correspond to Service objects that the Pod participates in. + items: + type: string + type: array + type: object + secretClass: + description: '[SecretClass](https://docs.stackable.tech/home/nightly/secret-operator/secretclass) containing the LDAP bind credentials.' + type: string + required: + - secretClass + type: object + host: + description: 'Host of the S3 server without any protocol or port. For example: `west1.my-cloud.com`.' + type: string + port: + description: |- + Port the S3 server listens on. + If not specified the product will determine the port to use. + format: uint16 + maximum: 65535.0 + minimum: 0.0 + nullable: true + type: integer + region: + default: + name: us-east-1 + description: |- + Bucket region used for signing headers (sigv4). + + This defaults to `us-east-1` which is compatible with other implementations such as Minio. + + WARNING: Some products use the Hadoop S3 implementation which falls back to us-east-2. + properties: + name: + default: us-east-1 + type: string + type: object + tls: + description: Use a TLS connection. If not specified no TLS will be used. + nullable: true + properties: + verification: + description: The verification method used to verify the certificates of the server and/or the client. + oneOf: + - required: + - none + - required: + - server + properties: + none: + description: Use TLS but don't verify certificates. + type: object + server: + description: Use TLS and a CA certificate to verify the server. + properties: + caCert: + description: CA cert to verify the server. + oneOf: + - required: + - webPki + - required: + - secretClass + properties: + secretClass: + description: |- + Name of the [SecretClass](https://docs.stackable.tech/home/nightly/secret-operator/secretclass) which will provide the CA certificate. + Note that a SecretClass does not need to have a key but can also work with just a CA certificate, + so if you got provided with a CA cert but don't have access to the key you can still use this method. + type: string + webPki: + description: |- + Use TLS and the CA certificates trusted by the common web browsers to verify the server. + This can be useful when you e.g. use public AWS S3 or other public available services. + type: object + type: object + required: + - caCert + type: object + type: object + required: + - verification + type: object + required: + - host + type: object + reference: + type: string + type: object + sparkConf: + additionalProperties: + type: string + default: {} + description: A map of key/value strings that will be passed directly to spark-submit. + type: object + sparkImage: + anyOf: + - required: + - custom + - productVersion + - required: + - productVersion + description: |- + Specify which image to use, the easiest way is to only configure the `productVersion`. + You can also configure a custom image registry to pull from, as well as completely custom + images. + + Consult the [Product image selection documentation](https://docs.stackable.tech/home/nightly/concepts/product_image_selection) + for details. + properties: + custom: + description: |- + Provide a custom container image. + + Specify the full container image name, e.g. `oci.example.tech/namespace/superset:1.4.1-my-tag` + type: string + productVersion: + description: Version of the product, e.g. `1.4.1`. + type: string + pullPolicy: + default: Always + description: '[Pull policy](https://kubernetes.io/docs/concepts/containers/images/#image-pull-policy) used when pulling the image.' + enum: + - IfNotPresent + - Always + - Never + type: string + pullSecrets: + description: '[Image pull secrets](https://kubernetes.io/docs/concepts/containers/images/#specifying-imagepullsecrets-on-a-pod) to pull images from a private registry.' + items: + description: LocalObjectReference contains enough information to let you locate the referenced object inside the same namespace. + properties: + name: + description: 'Name of the referent. This field is effectively required, but due to backwards compatibility is allowed to be empty. Instances of this type with an empty value here are almost certainly wrong. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + required: + - name + type: object + nullable: true + type: array + repo: + description: |- + The repository on the container image registry where the container image is located, e.g. + `oci.example.com/namespace`. + + If not specified, the operator will use the image registry provided via the operator + environment options. + nullable: true + type: string + stackableVersion: + description: |- + Stackable version of the product, e.g. `23.4`, `23.4.1` or `0.0.0-dev`. + + If not specified, the operator will use its own version, e.g. `23.4.1`. When using a nightly + operator or a PR version, it will use the nightly `0.0.0-dev` image. + nullable: true + type: string + type: object + vectorAggregatorConfigMapName: + description: |- + Name of the Vector aggregator [discovery ConfigMap](https://docs.stackable.tech/home/nightly/concepts/service_discovery). + It must contain the key `ADDRESS` with the address of the Vector aggregator. + Follow the [logging tutorial](https://docs.stackable.tech/home/nightly/tutorials/logging-vector-aggregator) + to learn how to configure log aggregation with Vector. + nullable: true + type: string + volumes: + default: [] + description: A list of volumes that can be made available to the job, driver or executors via their volume mounts. + items: + type: object + x-kubernetes-preserve-unknown-fields: true + type: array + required: + - mainApplicationFile + - mode + - sparkImage + type: object + status: + description: |- + Status of a SparkApplication. + + Defined outside the `versioned` module because it does not change between CRD versions and is + shared by all versions as the status subresource. + nullable: true + properties: + phase: + type: string + resolvedTemplateRef: + items: + description: |- + A reference to a [`SparkApplicationTemplate`](template_spec::SparkApplicationTemplate) that has + been merged into a SparkApplication. Stored in the application status. + + This type is intentionally defined outside the `versioned` module because it does not change + between CRD versions and is nested inside a `Vec` in the status, which the auto-generated + version conversions cannot handle for versioned types. + properties: + name: + type: string + uid: + nullable: true + type: string + required: + - name + type: object + type: array + required: + - phase + type: object + required: + - spec + title: SparkApplication + type: object + served: true + storage: false + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: sparkhistoryservers.spark.stackable.tech +spec: + group: spark.stackable.tech + names: + categories: [] + kind: SparkHistoryServer + plural: sparkhistoryservers + shortNames: + - sparkhist + - sparkhistory + - shs + singular: sparkhistoryserver + scope: Namespaced + versions: + - additionalPrinterColumns: [] + name: v1alpha1 + schema: + openAPIV3Schema: + description: Auto-generated derived type for SparkHistoryServerSpec via `CustomResource` + properties: + spec: + description: |- + A Spark cluster history server component. This resource is managed by the Stackable operator + for Apache Spark. Find more information on how to use it in the + [operator documentation](https://docs.stackable.tech/home/nightly/spark-k8s/usage-guide/history-server). + properties: + image: + anyOf: + - required: + - custom + - productVersion + - required: + - productVersion + description: |- + Specify which image to use, the easiest way is to only configure the `productVersion`. + You can also configure a custom image registry to pull from, as well as completely custom + images. + + Consult the [Product image selection documentation](https://docs.stackable.tech/home/nightly/concepts/product_image_selection) + for details. + properties: + custom: + description: |- + Provide a custom container image. + + Specify the full container image name, e.g. `oci.example.tech/namespace/superset:1.4.1-my-tag` + type: string + productVersion: + description: Version of the product, e.g. `1.4.1`. + type: string + pullPolicy: + default: Always + description: '[Pull policy](https://kubernetes.io/docs/concepts/containers/images/#image-pull-policy) used when pulling the image.' + enum: + - IfNotPresent + - Always + - Never + type: string + pullSecrets: + description: '[Image pull secrets](https://kubernetes.io/docs/concepts/containers/images/#specifying-imagepullsecrets-on-a-pod) to pull images from a private registry.' + items: + description: LocalObjectReference contains enough information to let you locate the referenced object inside the same namespace. + properties: + name: + description: 'Name of the referent. This field is effectively required, but due to backwards compatibility is allowed to be empty. Instances of this type with an empty value here are almost certainly wrong. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + required: + - name + type: object + nullable: true + type: array + repo: + description: |- + The repository on the container image registry where the container image is located, e.g. + `oci.example.com/namespace`. + + If not specified, the operator will use the image registry provided via the operator + environment options. + nullable: true + type: string + stackableVersion: + description: |- + Stackable version of the product, e.g. `23.4`, `23.4.1` or `0.0.0-dev`. + + If not specified, the operator will use its own version, e.g. `23.4.1`. When using a nightly + operator or a PR version, it will use the nightly `0.0.0-dev` image. + nullable: true + type: string + type: object + logFileDirectory: + description: The log file directory definition used by the Spark history server. + oneOf: + - required: + - s3 + - required: + - customLogDirectory + properties: + customLogDirectory: + description: A custom log directory + type: string + s3: + description: An S3 bucket storing the log events + properties: + bucket: + oneOf: + - required: + - inline + - required: + - reference + properties: + inline: + description: |- + S3 bucket specification containing the bucket name and an inlined or referenced connection specification. + Learn more on the [S3 concept documentation](https://docs.stackable.tech/home/nightly/concepts/s3). + properties: + bucketName: + description: The name of the S3 bucket. + type: string + connection: + description: The definition of an S3 connection, either inline or as a reference. + oneOf: + - required: + - inline + - required: + - reference + properties: + inline: + description: |- + S3 connection definition as a resource. + Learn more on the [S3 concept documentation](https://docs.stackable.tech/home/nightly/concepts/s3). + properties: + accessStyle: + default: VirtualHosted + description: |- + Which access style to use. + Defaults to virtual hosted-style as most of the data products out there. + Have a look at the [AWS documentation](https://docs.aws.amazon.com/AmazonS3/latest/userguide/VirtualHosting.html). + enum: + - Path + - VirtualHosted + type: string + credentials: + description: |- + If the S3 uses authentication you have to specify you S3 credentials. + In the most cases a [SecretClass](https://docs.stackable.tech/home/nightly/secret-operator/secretclass) + providing `accessKey` and `secretKey` is sufficient. + nullable: true + properties: + scope: + description: |- + [Scope](https://docs.stackable.tech/home/nightly/secret-operator/scope) of the + [SecretClass](https://docs.stackable.tech/home/nightly/secret-operator/secretclass). + nullable: true + properties: + listenerVolumes: + default: [] + description: |- + The listener volume scope allows Node and Service scopes to be inferred from the applicable listeners. + This must correspond to Volume names in the Pod that mount Listeners. + items: + type: string + type: array + node: + default: false + description: |- + The node scope is resolved to the name of the Kubernetes Node object that the Pod is running on. + This will typically be the DNS name of the node. + type: boolean + pod: + default: false + description: |- + The pod scope is resolved to the name of the Kubernetes Pod. + This allows the secret to differentiate between StatefulSet replicas. + type: boolean + services: + default: [] + description: |- + The service scope allows Pod objects to specify custom scopes. + This should typically correspond to Service objects that the Pod participates in. + items: + type: string + type: array + type: object + secretClass: + description: '[SecretClass](https://docs.stackable.tech/home/nightly/secret-operator/secretclass) containing the LDAP bind credentials.' + type: string + required: + - secretClass + type: object + host: + description: 'Host of the S3 server without any protocol or port. For example: `west1.my-cloud.com`.' + type: string + port: + description: |- + Port the S3 server listens on. + If not specified the product will determine the port to use. + format: uint16 + maximum: 65535.0 + minimum: 0.0 + nullable: true + type: integer + region: + default: + name: us-east-1 + description: |- + Bucket region used for signing headers (sigv4). + + This defaults to `us-east-1` which is compatible with other implementations such as Minio. + + WARNING: Some products use the Hadoop S3 implementation which falls back to us-east-2. + properties: + name: + default: us-east-1 + type: string + type: object + tls: + description: Use a TLS connection. If not specified no TLS will be used. + nullable: true + properties: + verification: + description: The verification method used to verify the certificates of the server and/or the client. + oneOf: + - required: + - none + - required: + - server + properties: + none: + description: Use TLS but don't verify certificates. + type: object + server: + description: Use TLS and a CA certificate to verify the server. + properties: + caCert: + description: CA cert to verify the server. + oneOf: + - required: + - webPki + - required: + - secretClass + properties: + secretClass: + description: |- + Name of the [SecretClass](https://docs.stackable.tech/home/nightly/secret-operator/secretclass) which will provide the CA certificate. + Note that a SecretClass does not need to have a key but can also work with just a CA certificate, + so if you got provided with a CA cert but don't have access to the key you can still use this method. + type: string + webPki: + description: |- + Use TLS and the CA certificates trusted by the common web browsers to verify the server. + This can be useful when you e.g. use public AWS S3 or other public available services. + type: object + type: object + required: + - caCert + type: object + type: object + required: + - verification + type: object + required: + - host + type: object + reference: + type: string + type: object + required: + - bucketName + - connection + type: object + reference: + type: string + type: object + prefix: + type: string + required: + - bucket + - prefix + type: object + type: object + nodes: + description: A history server node role definition. + properties: + cliOverrides: + additionalProperties: + type: string + default: {} + type: object + config: + default: {} + properties: + affinity: + default: + nodeAffinity: null + nodeSelector: null + podAffinity: null + podAntiAffinity: null + description: |- + These configuration settings control + [Pod placement](https://docs.stackable.tech/home/nightly/concepts/operations/pod_placement). + properties: + nodeAffinity: + description: Same as the `spec.affinity.nodeAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + nodeSelector: + additionalProperties: + type: string + description: Simple key-value pairs forming a nodeSelector, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true + type: object + podAffinity: + description: Same as the `spec.affinity.podAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + podAntiAffinity: + description: Same as the `spec.affinity.podAntiAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + cleaner: + nullable: true + type: boolean + logging: + default: + containers: {} + enableVectorAgent: null + description: Logging configuration, learn more in the [logging concept documentation](https://docs.stackable.tech/home/nightly/concepts/logging). + properties: + containers: + description: Log configuration per container. + properties: + spark-history: + anyOf: + - required: + - custom + - {} + - {} + description: Log configuration of the container + properties: + console: + description: Configuration for the console appender + nullable: true + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + custom: + description: Log configuration provided in a ConfigMap + properties: + configMap: + description: ConfigMap containing the log configuration files + nullable: true + type: string + type: object + file: + description: Configuration for the file appender + nullable: true + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + loggers: + additionalProperties: + description: Configuration of a logger + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + default: {} + description: Configuration per logger + type: object + type: object + vector: + anyOf: + - required: + - custom + - {} + - {} + description: Log configuration of the container + properties: + console: + description: Configuration for the console appender + nullable: true + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + custom: + description: Log configuration provided in a ConfigMap + properties: + configMap: + description: ConfigMap containing the log configuration files + nullable: true + type: string + type: object + file: + description: Configuration for the file appender + nullable: true + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + loggers: + additionalProperties: + description: Configuration of a logger + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + default: {} + description: Configuration per logger + type: object + type: object + type: object + enableVectorAgent: + description: Whether or not to deploy a container with the Vector log agent. + nullable: true + type: boolean + type: object + requestedSecretLifetime: + description: |- + Request secret (currently only autoTls certificates) lifetime from the secret operator, e.g. `7d`, or `30d`. + This can be shortened by the `maxCertificateLifetime` setting on the SecretClass issuing the TLS certificate. + nullable: true + type: string + resources: + default: + cpu: + max: null + min: null + memory: + limit: null + runtimeLimits: {} + storage: {} + description: |- + Resource usage is configured here, this includes CPU usage, memory usage and disk storage + usage, if this role needs any. + properties: + cpu: + default: + max: null + min: null + properties: + max: + description: |- + The maximum amount of CPU cores that can be requested by Pods. + Equivalent to the `limit` for Pod resource configuration. + Cores are specified either as a decimal point number or as milli units. + For example:`1.5` will be 1.5 cores, also written as `1500m`. + nullable: true + x-kubernetes-int-or-string: true + min: + description: |- + The minimal amount of CPU cores that Pods need to run. + Equivalent to the `request` for Pod resource configuration. + Cores are specified either as a decimal point number or as milli units. + For example:`1.5` will be 1.5 cores, also written as `1500m`. + nullable: true + x-kubernetes-int-or-string: true + type: object + memory: + properties: + limit: + description: |- + The maximum amount of memory that should be available to the Pod. + Specified as a byte [Quantity](https://kubernetes.io/docs/reference/kubernetes-api/common-definitions/quantity/), + which means these suffixes are supported: E, P, T, G, M, k. + You can also use the power-of-two equivalents: Ei, Pi, Ti, Gi, Mi, Ki. + For example, the following represent roughly the same value: + `128974848, 129e6, 129M, 128974848000m, 123Mi` + nullable: true + x-kubernetes-int-or-string: true + runtimeLimits: + description: Additional options that can be specified. + type: object + type: object + storage: + type: object + type: object + type: object + configOverrides: + description: |- + The `configOverrides` can be used to configure properties in product config files + that are not exposed in the CRD. Read the + [config overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#config-overrides) + and consult the operator specific usage guide documentation for details on the + available config files and settings for the specific product. + properties: + security.properties: + additionalProperties: + type: string + description: |- + Flat key-value overrides for `*.properties`, Hadoop XML, etc. + + This is backwards-compatible with the existing flat key-value YAML format + used by `HashMap`. + nullable: true + type: object + spark-defaults.conf: + additionalProperties: + type: string + description: |- + Flat key-value overrides for `*.properties`, Hadoop XML, etc. + + This is backwards-compatible with the existing flat key-value YAML format + used by `HashMap`. + nullable: true + type: object + spark-env.sh: + additionalProperties: + type: string + description: |- + Flat key-value overrides for `*.properties`, Hadoop XML, etc. + + This is backwards-compatible with the existing flat key-value YAML format + used by `HashMap`. + nullable: true + type: object + type: object + envOverrides: + additionalProperties: + type: string + default: {} + description: |- + `envOverrides` configure environment variables to be set in the Pods. + It is a map from strings to strings - environment variables and the value to set. + Read the + [environment variable overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#env-overrides) + for more information and consult the operator specific usage guide to find out about + the product specific environment variables that are available. + type: object + jvmArgumentOverrides: + default: + add: [] + remove: [] + removeRegex: [] + description: |- + Allows overriding JVM arguments. + Please read on the [JVM argument overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#jvm-argument-overrides) + for details on the usage. + properties: + add: + default: [] + description: JVM arguments to be added + items: + type: string + type: array + remove: + default: [] + description: JVM arguments to be removed by exact match + items: + type: string + type: array + removeRegex: + default: [] + description: JVM arguments matching any of this regexes will be removed + items: + type: string + type: array + type: object + podOverrides: + default: {} + description: |- + In the `podOverrides` property you can define a + [PodTemplateSpec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.34/#podtemplatespec-v1-core) + to override any property that can be set on a Kubernetes Pod. + Read the + [Pod overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#pod-overrides) + for more information. + type: object + x-kubernetes-preserve-unknown-fields: true + roleConfig: + default: + listenerClass: cluster-internal + podDisruptionBudget: + enabled: true + maxUnavailable: null + description: This is a product-agnostic RoleConfig, which is sufficient for most of the products. + properties: + listenerClass: + default: cluster-internal + description: |- + This field controls which [ListenerClass](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass.html) + is used to expose the history server. + type: string + podDisruptionBudget: + default: + enabled: true maxUnavailable: null description: |- - This struct is used to configure: + This struct is used to configure: + + 1. If PodDisruptionBudgets are created by the operator + 2. The allowed number of Pods to be unavailable (`maxUnavailable`) + + Learn more in the + [allowed Pod disruptions documentation](https://docs.stackable.tech/home/nightly/concepts/operations/pod_disruptions). + properties: + enabled: + default: true + description: |- + Whether a PodDisruptionBudget should be written out for this role. + Disabling this enables you to specify your own - custom - one. + Defaults to true. + type: boolean + maxUnavailable: + description: |- + The number of Pods that are allowed to be down because of voluntary disruptions. + If you don't explicitly set this, the operator will use a sane default based + upon knowledge about the individual product. + format: uint16 + maximum: 65535.0 + minimum: 0.0 + nullable: true + type: integer + type: object + type: object + roleGroups: + additionalProperties: + properties: + cliOverrides: + additionalProperties: + type: string + default: {} + type: object + config: + default: {} + properties: + affinity: + default: + nodeAffinity: null + nodeSelector: null + podAffinity: null + podAntiAffinity: null + description: |- + These configuration settings control + [Pod placement](https://docs.stackable.tech/home/nightly/concepts/operations/pod_placement). + properties: + nodeAffinity: + description: Same as the `spec.affinity.nodeAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + nodeSelector: + additionalProperties: + type: string + description: Simple key-value pairs forming a nodeSelector, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true + type: object + podAffinity: + description: Same as the `spec.affinity.podAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + podAntiAffinity: + description: Same as the `spec.affinity.podAntiAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + cleaner: + nullable: true + type: boolean + logging: + default: + containers: {} + enableVectorAgent: null + description: Logging configuration, learn more in the [logging concept documentation](https://docs.stackable.tech/home/nightly/concepts/logging). + properties: + containers: + description: Log configuration per container. + properties: + spark-history: + anyOf: + - required: + - custom + - {} + - {} + description: Log configuration of the container + properties: + console: + description: Configuration for the console appender + nullable: true + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + custom: + description: Log configuration provided in a ConfigMap + properties: + configMap: + description: ConfigMap containing the log configuration files + nullable: true + type: string + type: object + file: + description: Configuration for the file appender + nullable: true + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + loggers: + additionalProperties: + description: Configuration of a logger + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + default: {} + description: Configuration per logger + type: object + type: object + vector: + anyOf: + - required: + - custom + - {} + - {} + description: Log configuration of the container + properties: + console: + description: Configuration for the console appender + nullable: true + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + custom: + description: Log configuration provided in a ConfigMap + properties: + configMap: + description: ConfigMap containing the log configuration files + nullable: true + type: string + type: object + file: + description: Configuration for the file appender + nullable: true + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + loggers: + additionalProperties: + description: Configuration of a logger + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + default: {} + description: Configuration per logger + type: object + type: object + type: object + enableVectorAgent: + description: Whether or not to deploy a container with the Vector log agent. + nullable: true + type: boolean + type: object + requestedSecretLifetime: + description: |- + Request secret (currently only autoTls certificates) lifetime from the secret operator, e.g. `7d`, or `30d`. + This can be shortened by the `maxCertificateLifetime` setting on the SecretClass issuing the TLS certificate. + nullable: true + type: string + resources: + default: + cpu: + max: null + min: null + memory: + limit: null + runtimeLimits: {} + storage: {} + description: |- + Resource usage is configured here, this includes CPU usage, memory usage and disk storage + usage, if this role needs any. + properties: + cpu: + default: + max: null + min: null + properties: + max: + description: |- + The maximum amount of CPU cores that can be requested by Pods. + Equivalent to the `limit` for Pod resource configuration. + Cores are specified either as a decimal point number or as milli units. + For example:`1.5` will be 1.5 cores, also written as `1500m`. + nullable: true + x-kubernetes-int-or-string: true + min: + description: |- + The minimal amount of CPU cores that Pods need to run. + Equivalent to the `request` for Pod resource configuration. + Cores are specified either as a decimal point number or as milli units. + For example:`1.5` will be 1.5 cores, also written as `1500m`. + nullable: true + x-kubernetes-int-or-string: true + type: object + memory: + properties: + limit: + description: |- + The maximum amount of memory that should be available to the Pod. + Specified as a byte [Quantity](https://kubernetes.io/docs/reference/kubernetes-api/common-definitions/quantity/), + which means these suffixes are supported: E, P, T, G, M, k. + You can also use the power-of-two equivalents: Ei, Pi, Ti, Gi, Mi, Ki. + For example, the following represent roughly the same value: + `128974848, 129e6, 129M, 128974848000m, 123Mi` + nullable: true + x-kubernetes-int-or-string: true + runtimeLimits: + description: Additional options that can be specified. + type: object + type: object + storage: + type: object + type: object + type: object + configOverrides: + description: |- + The `configOverrides` can be used to configure properties in product config files + that are not exposed in the CRD. Read the + [config overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#config-overrides) + and consult the operator specific usage guide documentation for details on the + available config files and settings for the specific product. + properties: + security.properties: + additionalProperties: + type: string + description: |- + Flat key-value overrides for `*.properties`, Hadoop XML, etc. + + This is backwards-compatible with the existing flat key-value YAML format + used by `HashMap`. + nullable: true + type: object + spark-defaults.conf: + additionalProperties: + type: string + description: |- + Flat key-value overrides for `*.properties`, Hadoop XML, etc. + + This is backwards-compatible with the existing flat key-value YAML format + used by `HashMap`. + nullable: true + type: object + spark-env.sh: + additionalProperties: + type: string + description: |- + Flat key-value overrides for `*.properties`, Hadoop XML, etc. + + This is backwards-compatible with the existing flat key-value YAML format + used by `HashMap`. + nullable: true + type: object + type: object + envOverrides: + additionalProperties: + type: string + default: {} + description: |- + `envOverrides` configure environment variables to be set in the Pods. + It is a map from strings to strings - environment variables and the value to set. + Read the + [environment variable overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#env-overrides) + for more information and consult the operator specific usage guide to find out about + the product specific environment variables that are available. + type: object + jvmArgumentOverrides: + default: + add: [] + remove: [] + removeRegex: [] + description: |- + Allows overriding JVM arguments. + Please read on the [JVM argument overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#jvm-argument-overrides) + for details on the usage. + properties: + add: + default: [] + description: JVM arguments to be added + items: + type: string + type: array + remove: + default: [] + description: JVM arguments to be removed by exact match + items: + type: string + type: array + removeRegex: + default: [] + description: JVM arguments matching any of this regexes will be removed + items: + type: string + type: array + type: object + podOverrides: + default: {} + description: |- + In the `podOverrides` property you can define a + [PodTemplateSpec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.34/#podtemplatespec-v1-core) + to override any property that can be set on a Kubernetes Pod. + Read the + [Pod overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#pod-overrides) + for more information. + type: object + x-kubernetes-preserve-unknown-fields: true + replicas: + format: uint16 + maximum: 65535.0 + minimum: 0.0 + nullable: true + type: integer + type: object + type: object + required: + - roleGroups + type: object + objectOverrides: + default: [] + description: |- + A list of generic Kubernetes objects, which are merged into the objects that the operator + creates. + + List entries are arbitrary YAML objects, which need to be valid Kubernetes objects. + + Read the [Object overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#object-overrides) + for more information. + items: + type: object + x-kubernetes-preserve-unknown-fields: true + type: array + sparkConf: + additionalProperties: + type: string + default: {} + description: A map of key/value strings that will be passed directly to Spark when deploying the history server. + type: object + vectorAggregatorConfigMapName: + description: |- + Name of the Vector aggregator discovery ConfigMap. + It must contain the key `ADDRESS` with the address of the Vector aggregator. + nullable: true + type: string + required: + - image + - logFileDirectory + - nodes + type: object + required: + - spec + title: SparkHistoryServer + type: object + served: true + storage: true + subresources: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: sparkconnectservers.spark.stackable.tech +spec: + group: spark.stackable.tech + names: + categories: [] + kind: SparkConnectServer + plural: sparkconnectservers + shortNames: + - sparkconnect + singular: sparkconnectserver + scope: Namespaced + versions: + - additionalPrinterColumns: [] + name: v1alpha1 + schema: + openAPIV3Schema: + description: Auto-generated derived type for SparkConnectServerSpec via `CustomResource` + properties: + spec: + description: |- + An Apache Spark Connect server component. This resource is managed by the Stackable operator + for Apache Spark. Find more information on how to use it in the + [operator documentation](https://docs.stackable.tech/home/nightly/spark-k8s/usage-guide/connect-server). + properties: + args: + default: [] + description: User provided command line arguments appended to the server entry point. + items: + type: string + type: array + clusterOperation: + default: + reconciliationPaused: false + stopped: false + description: |- + [Cluster operations](https://docs.stackable.tech/home/nightly/concepts/operations/cluster_operations) + properties, allow stopping the product instance as well as pausing reconciliation. + properties: + reconciliationPaused: + default: false + description: |- + Flag to stop cluster reconciliation by the operator. This means that all changes in the + custom resource spec are ignored until this flag is set to false or removed. The operator + will however still watch the deployed resources at the time and update the custom resource + status field. + If applied at the same time with `stopped`, `reconciliationPaused` will take precedence over + `stopped` and stop the reconciliation immediately. + type: boolean + stopped: + default: false + description: |- + Flag to stop the cluster. This means all deployed resources (e.g. Services, StatefulSets, + ConfigMaps) are kept but all deployed Pods (e.g. replicas from a StatefulSet) are scaled to 0 + and therefore stopped and removed. + If applied at the same time with `reconciliationPaused`, the latter will pause reconciliation + and `stopped` will take no effect until `reconciliationPaused` is set to false or removed. + type: boolean + type: object + connectors: + default: + s3buckets: [] + description: One or more S3 connections to be used by the Spark Connect server. + properties: + s3buckets: + default: [] + items: + oneOf: + - required: + - inline + - required: + - reference + properties: + inline: + description: |- + S3 bucket specification containing the bucket name and an inlined or referenced connection specification. + Learn more on the [S3 concept documentation](https://docs.stackable.tech/home/nightly/concepts/s3). + properties: + bucketName: + description: The name of the S3 bucket. + type: string + connection: + description: The definition of an S3 connection, either inline or as a reference. + oneOf: + - required: + - inline + - required: + - reference + properties: + inline: + description: |- + S3 connection definition as a resource. + Learn more on the [S3 concept documentation](https://docs.stackable.tech/home/nightly/concepts/s3). + properties: + accessStyle: + default: VirtualHosted + description: |- + Which access style to use. + Defaults to virtual hosted-style as most of the data products out there. + Have a look at the [AWS documentation](https://docs.aws.amazon.com/AmazonS3/latest/userguide/VirtualHosting.html). + enum: + - Path + - VirtualHosted + type: string + credentials: + description: |- + If the S3 uses authentication you have to specify you S3 credentials. + In the most cases a [SecretClass](https://docs.stackable.tech/home/nightly/secret-operator/secretclass) + providing `accessKey` and `secretKey` is sufficient. + nullable: true + properties: + scope: + description: |- + [Scope](https://docs.stackable.tech/home/nightly/secret-operator/scope) of the + [SecretClass](https://docs.stackable.tech/home/nightly/secret-operator/secretclass). + nullable: true + properties: + listenerVolumes: + default: [] + description: |- + The listener volume scope allows Node and Service scopes to be inferred from the applicable listeners. + This must correspond to Volume names in the Pod that mount Listeners. + items: + type: string + type: array + node: + default: false + description: |- + The node scope is resolved to the name of the Kubernetes Node object that the Pod is running on. + This will typically be the DNS name of the node. + type: boolean + pod: + default: false + description: |- + The pod scope is resolved to the name of the Kubernetes Pod. + This allows the secret to differentiate between StatefulSet replicas. + type: boolean + services: + default: [] + description: |- + The service scope allows Pod objects to specify custom scopes. + This should typically correspond to Service objects that the Pod participates in. + items: + type: string + type: array + type: object + secretClass: + description: '[SecretClass](https://docs.stackable.tech/home/nightly/secret-operator/secretclass) containing the LDAP bind credentials.' + type: string + required: + - secretClass + type: object + host: + description: 'Host of the S3 server without any protocol or port. For example: `west1.my-cloud.com`.' + type: string + port: + description: |- + Port the S3 server listens on. + If not specified the product will determine the port to use. + format: uint16 + maximum: 65535.0 + minimum: 0.0 + nullable: true + type: integer + region: + default: + name: us-east-1 + description: |- + Bucket region used for signing headers (sigv4). + + This defaults to `us-east-1` which is compatible with other implementations such as Minio. + + WARNING: Some products use the Hadoop S3 implementation which falls back to us-east-2. + properties: + name: + default: us-east-1 + type: string + type: object + tls: + description: Use a TLS connection. If not specified no TLS will be used. + nullable: true + properties: + verification: + description: The verification method used to verify the certificates of the server and/or the client. + oneOf: + - required: + - none + - required: + - server + properties: + none: + description: Use TLS but don't verify certificates. + type: object + server: + description: Use TLS and a CA certificate to verify the server. + properties: + caCert: + description: CA cert to verify the server. + oneOf: + - required: + - webPki + - required: + - secretClass + properties: + secretClass: + description: |- + Name of the [SecretClass](https://docs.stackable.tech/home/nightly/secret-operator/secretclass) which will provide the CA certificate. + Note that a SecretClass does not need to have a key but can also work with just a CA certificate, + so if you got provided with a CA cert but don't have access to the key you can still use this method. + type: string + webPki: + description: |- + Use TLS and the CA certificates trusted by the common web browsers to verify the server. + This can be useful when you e.g. use public AWS S3 or other public available services. + type: object + type: object + required: + - caCert + type: object + type: object + required: + - verification + type: object + required: + - host + type: object + reference: + type: string + type: object + required: + - bucketName + - connection + type: object + reference: + type: string + type: object + type: array + s3connection: + nullable: true + oneOf: + - required: + - inline + - required: + - reference + properties: + inline: + description: |- + S3 connection definition as a resource. + Learn more on the [S3 concept documentation](https://docs.stackable.tech/home/nightly/concepts/s3). + properties: + accessStyle: + default: VirtualHosted + description: |- + Which access style to use. + Defaults to virtual hosted-style as most of the data products out there. + Have a look at the [AWS documentation](https://docs.aws.amazon.com/AmazonS3/latest/userguide/VirtualHosting.html). + enum: + - Path + - VirtualHosted + type: string + credentials: + description: |- + If the S3 uses authentication you have to specify you S3 credentials. + In the most cases a [SecretClass](https://docs.stackable.tech/home/nightly/secret-operator/secretclass) + providing `accessKey` and `secretKey` is sufficient. + nullable: true + properties: + scope: + description: |- + [Scope](https://docs.stackable.tech/home/nightly/secret-operator/scope) of the + [SecretClass](https://docs.stackable.tech/home/nightly/secret-operator/secretclass). + nullable: true + properties: + listenerVolumes: + default: [] + description: |- + The listener volume scope allows Node and Service scopes to be inferred from the applicable listeners. + This must correspond to Volume names in the Pod that mount Listeners. + items: + type: string + type: array + node: + default: false + description: |- + The node scope is resolved to the name of the Kubernetes Node object that the Pod is running on. + This will typically be the DNS name of the node. + type: boolean + pod: + default: false + description: |- + The pod scope is resolved to the name of the Kubernetes Pod. + This allows the secret to differentiate between StatefulSet replicas. + type: boolean + services: + default: [] + description: |- + The service scope allows Pod objects to specify custom scopes. + This should typically correspond to Service objects that the Pod participates in. + items: + type: string + type: array + type: object + secretClass: + description: '[SecretClass](https://docs.stackable.tech/home/nightly/secret-operator/secretclass) containing the LDAP bind credentials.' + type: string + required: + - secretClass + type: object + host: + description: 'Host of the S3 server without any protocol or port. For example: `west1.my-cloud.com`.' + type: string + port: + description: |- + Port the S3 server listens on. + If not specified the product will determine the port to use. + format: uint16 + maximum: 65535.0 + minimum: 0.0 + nullable: true + type: integer + region: + default: + name: us-east-1 + description: |- + Bucket region used for signing headers (sigv4). + + This defaults to `us-east-1` which is compatible with other implementations such as Minio. + + WARNING: Some products use the Hadoop S3 implementation which falls back to us-east-2. + properties: + name: + default: us-east-1 + type: string + type: object + tls: + description: Use a TLS connection. If not specified no TLS will be used. + nullable: true + properties: + verification: + description: The verification method used to verify the certificates of the server and/or the client. + oneOf: + - required: + - none + - required: + - server + properties: + none: + description: Use TLS but don't verify certificates. + type: object + server: + description: Use TLS and a CA certificate to verify the server. + properties: + caCert: + description: CA cert to verify the server. + oneOf: + - required: + - webPki + - required: + - secretClass + properties: + secretClass: + description: |- + Name of the [SecretClass](https://docs.stackable.tech/home/nightly/secret-operator/secretclass) which will provide the CA certificate. + Note that a SecretClass does not need to have a key but can also work with just a CA certificate, + so if you got provided with a CA cert but don't have access to the key you can still use this method. + type: string + webPki: + description: |- + Use TLS and the CA certificates trusted by the common web browsers to verify the server. + This can be useful when you e.g. use public AWS S3 or other public available services. + type: object + type: object + required: + - caCert + type: object + type: object + required: + - verification + type: object + required: + - host + type: object + reference: + type: string + type: object + type: object + executor: + description: Spark Connect executor properties. + nullable: true + properties: + cliOverrides: + additionalProperties: + type: string + default: {} + type: object + config: + default: {} + properties: + affinity: + default: + nodeAffinity: null + nodeSelector: null + podAffinity: null + podAntiAffinity: null + description: |- + These configuration settings control + [Pod placement](https://docs.stackable.tech/home/nightly/concepts/operations/pod_placement). + properties: + nodeAffinity: + description: Same as the `spec.affinity.nodeAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + nodeSelector: + additionalProperties: + type: string + description: Simple key-value pairs forming a nodeSelector, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true + type: object + podAffinity: + description: Same as the `spec.affinity.podAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + podAntiAffinity: + description: Same as the `spec.affinity.podAntiAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + logging: + default: + containers: {} + enableVectorAgent: null + description: Logging configuration, learn more in the [logging concept documentation](https://docs.stackable.tech/home/nightly/concepts/logging). + properties: + containers: + description: Log configuration per container. + properties: + spark: + anyOf: + - required: + - custom + - {} + - {} + description: Log configuration of the container + properties: + console: + description: Configuration for the console appender + nullable: true + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + custom: + description: Log configuration provided in a ConfigMap + properties: + configMap: + description: ConfigMap containing the log configuration files + nullable: true + type: string + type: object + file: + description: Configuration for the file appender + nullable: true + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + loggers: + additionalProperties: + description: Configuration of a logger + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + default: {} + description: Configuration per logger + type: object + type: object + vector: + anyOf: + - required: + - custom + - {} + - {} + description: Log configuration of the container + properties: + console: + description: Configuration for the console appender + nullable: true + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + custom: + description: Log configuration provided in a ConfigMap + properties: + configMap: + description: ConfigMap containing the log configuration files + nullable: true + type: string + type: object + file: + description: Configuration for the file appender + nullable: true + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + loggers: + additionalProperties: + description: Configuration of a logger + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + default: {} + description: Configuration per logger + type: object + type: object + type: object + enableVectorAgent: + description: Whether or not to deploy a container with the Vector log agent. + nullable: true + type: boolean + type: object + requestedSecretLifetime: + description: |- + Request secret (currently only autoTls certificates) lifetime from the secret operator, e.g. `7d`, or `30d`. + This can be shortened by the `maxCertificateLifetime` setting on the SecretClass issuing the TLS certificate. + nullable: true + type: string + resources: + default: + cpu: + max: null + min: null + memory: + limit: null + runtimeLimits: {} + storage: {} + description: |- + Resource usage is configured here, this includes CPU usage, memory usage and disk storage + usage, if this role needs any. + properties: + cpu: + default: + max: null + min: null + properties: + max: + description: |- + The maximum amount of CPU cores that can be requested by Pods. + Equivalent to the `limit` for Pod resource configuration. + Cores are specified either as a decimal point number or as milli units. + For example:`1.5` will be 1.5 cores, also written as `1500m`. + nullable: true + x-kubernetes-int-or-string: true + min: + description: |- + The minimal amount of CPU cores that Pods need to run. + Equivalent to the `request` for Pod resource configuration. + Cores are specified either as a decimal point number or as milli units. + For example:`1.5` will be 1.5 cores, also written as `1500m`. + nullable: true + x-kubernetes-int-or-string: true + type: object + memory: + properties: + limit: + description: |- + The maximum amount of memory that should be available to the Pod. + Specified as a byte [Quantity](https://kubernetes.io/docs/reference/kubernetes-api/common-definitions/quantity/), + which means these suffixes are supported: E, P, T, G, M, k. + You can also use the power-of-two equivalents: Ei, Pi, Ti, Gi, Mi, Ki. + For example, the following represent roughly the same value: + `128974848, 129e6, 129M, 128974848000m, 123Mi` + nullable: true + x-kubernetes-int-or-string: true + runtimeLimits: + description: Additional options that can be specified. + type: object + type: object + storage: + type: object + type: object + type: object + configOverrides: + description: |- + The `configOverrides` can be used to configure properties in product config files + that are not exposed in the CRD. Read the + [config overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#config-overrides) + and consult the operator specific usage guide documentation for details on the + available config files and settings for the specific product. + properties: + metrics.properties: + additionalProperties: + type: string + description: |- + Flat key-value overrides for `*.properties`, Hadoop XML, etc. + + This is backwards-compatible with the existing flat key-value YAML format + used by `HashMap`. + nullable: true + type: object + security.properties: + additionalProperties: + type: string + description: |- + Flat key-value overrides for `*.properties`, Hadoop XML, etc. + + This is backwards-compatible with the existing flat key-value YAML format + used by `HashMap`. + nullable: true + type: object + spark-defaults.conf: + additionalProperties: + type: string + description: |- + Flat key-value overrides for `*.properties`, Hadoop XML, etc. + + This is backwards-compatible with the existing flat key-value YAML format + used by `HashMap`. + nullable: true + type: object + type: object + envOverrides: + additionalProperties: + type: string + default: {} + description: |- + `envOverrides` configure environment variables to be set in the Pods. + It is a map from strings to strings - environment variables and the value to set. + Read the + [environment variable overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#env-overrides) + for more information and consult the operator specific usage guide to find out about + the product specific environment variables that are available. + type: object + jvmArgumentOverrides: + default: + add: [] + remove: [] + removeRegex: [] + description: |- + Allows overriding JVM arguments. + Please read on the [JVM argument overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#jvm-argument-overrides) + for details on the usage. + properties: + add: + default: [] + description: JVM arguments to be added + items: + type: string + type: array + remove: + default: [] + description: JVM arguments to be removed by exact match + items: + type: string + type: array + removeRegex: + default: [] + description: JVM arguments matching any of this regexes will be removed + items: + type: string + type: array + type: object + podOverrides: + default: {} + description: |- + In the `podOverrides` property you can define a + [PodTemplateSpec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.34/#podtemplatespec-v1-core) + to override any property that can be set on a Kubernetes Pod. + Read the + [Pod overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#pod-overrides) + for more information. + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + image: + anyOf: + - required: + - custom + - productVersion + - required: + - productVersion + description: |- + Specify which image to use, the easiest way is to only configure the `productVersion`. + You can also configure a custom image registry to pull from, as well as completely custom + images. + + Consult the [Product image selection documentation](https://docs.stackable.tech/home/nightly/concepts/product_image_selection) + for details. + properties: + custom: + description: |- + Provide a custom container image. + + Specify the full container image name, e.g. `oci.example.tech/namespace/superset:1.4.1-my-tag` + type: string + productVersion: + description: Version of the product, e.g. `1.4.1`. + type: string + pullPolicy: + default: Always + description: '[Pull policy](https://kubernetes.io/docs/concepts/containers/images/#image-pull-policy) used when pulling the image.' + enum: + - IfNotPresent + - Always + - Never + type: string + pullSecrets: + description: '[Image pull secrets](https://kubernetes.io/docs/concepts/containers/images/#specifying-imagepullsecrets-on-a-pod) to pull images from a private registry.' + items: + description: LocalObjectReference contains enough information to let you locate the referenced object inside the same namespace. + properties: + name: + description: 'Name of the referent. This field is effectively required, but due to backwards compatibility is allowed to be empty. Instances of this type with an empty value here are almost certainly wrong. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + required: + - name + type: object + nullable: true + type: array + repo: + description: |- + The repository on the container image registry where the container image is located, e.g. + `oci.example.com/namespace`. + + If not specified, the operator will use the image registry provided via the operator + environment options. + nullable: true + type: string + stackableVersion: + description: |- + Stackable version of the product, e.g. `23.4`, `23.4.1` or `0.0.0-dev`. + + If not specified, the operator will use its own version, e.g. `23.4.1`. When using a nightly + operator or a PR version, it will use the nightly `0.0.0-dev` image. + nullable: true + type: string + type: object + objectOverrides: + default: [] + description: |- + A list of generic Kubernetes objects, which are merged into the objects that the operator + creates. + + List entries are arbitrary YAML objects, which need to be valid Kubernetes objects. + + Read the [Object overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#object-overrides) + for more information. + items: + type: object + x-kubernetes-preserve-unknown-fields: true + type: array + server: + default: + roleConfig: + listenerClass: cluster-internal + description: A Spark Connect server definition. + properties: + cliOverrides: + additionalProperties: + type: string + default: {} + type: object + config: + default: {} + properties: + logging: + default: + containers: {} + enableVectorAgent: null + description: Logging configuration, learn more in the [logging concept documentation](https://docs.stackable.tech/home/nightly/concepts/logging). + properties: + containers: + description: Log configuration per container. + properties: + spark: + anyOf: + - required: + - custom + - {} + - {} + description: Log configuration of the container + properties: + console: + description: Configuration for the console appender + nullable: true + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + custom: + description: Log configuration provided in a ConfigMap + properties: + configMap: + description: ConfigMap containing the log configuration files + nullable: true + type: string + type: object + file: + description: Configuration for the file appender + nullable: true + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + loggers: + additionalProperties: + description: Configuration of a logger + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + default: {} + description: Configuration per logger + type: object + type: object + vector: + anyOf: + - required: + - custom + - {} + - {} + description: Log configuration of the container + properties: + console: + description: Configuration for the console appender + nullable: true + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + custom: + description: Log configuration provided in a ConfigMap + properties: + configMap: + description: ConfigMap containing the log configuration files + nullable: true + type: string + type: object + file: + description: Configuration for the file appender + nullable: true + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + loggers: + additionalProperties: + description: Configuration of a logger + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + default: {} + description: Configuration per logger + type: object + type: object + type: object + enableVectorAgent: + description: Whether or not to deploy a container with the Vector log agent. + nullable: true + type: boolean + type: object + requestedSecretLifetime: + description: |- + Request secret (currently only autoTls certificates) lifetime from the secret operator, e.g. `7d`, or `30d`. + This can be shortened by the `maxCertificateLifetime` setting on the SecretClass issuing the TLS certificate. + nullable: true + type: string + resources: + default: + cpu: + max: null + min: null + memory: + limit: null + runtimeLimits: {} + storage: {} + description: |- + Resource usage is configured here, this includes CPU usage, memory usage and disk storage + usage, if this role needs any. + properties: + cpu: + default: + max: null + min: null + properties: + max: + description: |- + The maximum amount of CPU cores that can be requested by Pods. + Equivalent to the `limit` for Pod resource configuration. + Cores are specified either as a decimal point number or as milli units. + For example:`1.5` will be 1.5 cores, also written as `1500m`. + nullable: true + x-kubernetes-int-or-string: true + min: + description: |- + The minimal amount of CPU cores that Pods need to run. + Equivalent to the `request` for Pod resource configuration. + Cores are specified either as a decimal point number or as milli units. + For example:`1.5` will be 1.5 cores, also written as `1500m`. + nullable: true + x-kubernetes-int-or-string: true + type: object + memory: + properties: + limit: + description: |- + The maximum amount of memory that should be available to the Pod. + Specified as a byte [Quantity](https://kubernetes.io/docs/reference/kubernetes-api/common-definitions/quantity/), + which means these suffixes are supported: E, P, T, G, M, k. + You can also use the power-of-two equivalents: Ei, Pi, Ti, Gi, Mi, Ki. + For example, the following represent roughly the same value: + `128974848, 129e6, 129M, 128974848000m, 123Mi` + nullable: true + x-kubernetes-int-or-string: true + runtimeLimits: + description: Additional options that can be specified. + type: object + type: object + storage: + type: object + type: object + type: object + configOverrides: + description: |- + The `configOverrides` can be used to configure properties in product config files + that are not exposed in the CRD. Read the + [config overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#config-overrides) + and consult the operator specific usage guide documentation for details on the + available config files and settings for the specific product. + properties: + metrics.properties: + additionalProperties: + type: string + description: |- + Flat key-value overrides for `*.properties`, Hadoop XML, etc. + + This is backwards-compatible with the existing flat key-value YAML format + used by `HashMap`. + nullable: true + type: object + security.properties: + additionalProperties: + type: string + description: |- + Flat key-value overrides for `*.properties`, Hadoop XML, etc. + + This is backwards-compatible with the existing flat key-value YAML format + used by `HashMap`. + nullable: true + type: object + spark-defaults.conf: + additionalProperties: + type: string + description: |- + Flat key-value overrides for `*.properties`, Hadoop XML, etc. + + This is backwards-compatible with the existing flat key-value YAML format + used by `HashMap`. + nullable: true + type: object + type: object + envOverrides: + additionalProperties: + type: string + default: {} + description: |- + `envOverrides` configure environment variables to be set in the Pods. + It is a map from strings to strings - environment variables and the value to set. + Read the + [environment variable overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#env-overrides) + for more information and consult the operator specific usage guide to find out about + the product specific environment variables that are available. + type: object + jvmArgumentOverrides: + default: + add: [] + remove: [] + removeRegex: [] + description: |- + Allows overriding JVM arguments. + Please read on the [JVM argument overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#jvm-argument-overrides) + for details on the usage. + properties: + add: + default: [] + description: JVM arguments to be added + items: + type: string + type: array + remove: + default: [] + description: JVM arguments to be removed by exact match + items: + type: string + type: array + removeRegex: + default: [] + description: JVM arguments matching any of this regexes will be removed + items: + type: string + type: array + type: object + podOverrides: + default: {} + description: |- + In the `podOverrides` property you can define a + [PodTemplateSpec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.34/#podtemplatespec-v1-core) + to override any property that can be set on a Kubernetes Pod. + Read the + [Pod overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#pod-overrides) + for more information. + type: object + x-kubernetes-preserve-unknown-fields: true + roleConfig: + default: + listenerClass: cluster-internal + description: Global role config settings for the Spark Connect Server. + properties: + listenerClass: + default: cluster-internal + description: |- + This field controls which [ListenerClass](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass.html) + is used to expose the Spark Connect services. + type: string + type: object + type: object + vectorAggregatorConfigMapName: + description: |- + Name of the Vector aggregator discovery ConfigMap. + It must contain the key `ADDRESS` with the address of the Vector aggregator. + nullable: true + type: string + required: + - image + type: object + status: + nullable: true + properties: + conditions: + default: [] + items: + properties: + lastTransitionTime: + description: Last time the condition transitioned from one status to another. + format: date-time + nullable: true + type: string + message: + description: A human readable message indicating details about the transition. + nullable: true + type: string + reason: + description: The reason for the condition's last transition. + nullable: true + type: string + status: + description: Status of the condition, one of True, False, Unknown. + enum: + - 'True' + - 'False' + - Unknown + type: string + type: + description: Type of deployment condition. + enum: + - Available + - Degraded + - Progressing + - ReconciliationPaused + - Stopped + type: string + required: + - status + - type + type: object + type: array + type: object + required: + - spec + title: SparkConnectServer + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: sparkapptemplates.spark.stackable.tech +spec: + group: spark.stackable.tech + names: + categories: [] + kind: SparkApplicationTemplate + plural: sparkapptemplates + shortNames: + - sparkapptemplate + singular: sparkapplicationtemplate + scope: Namespaced + versions: + - additionalPrinterColumns: [] + name: v1alpha2 + schema: + openAPIV3Schema: + description: Auto-generated derived type for SparkApplicationTemplateSpec via `CustomResource` + properties: + spec: + description: |- + A Spark application template. This resource is managed by the Stackable operator for Apache Spark. + Find more information on how to use it and the resources that the operator generates in the + [operator documentation](https://docs.stackable.tech/home/nightly/spark-k8s/). + properties: + args: + default: [] + description: Arguments passed directly to the job artifact. + items: + type: string + type: array + deprecatedJob: + description: |- + Deprecated and ignored. + + In previous versions this configured the dedicated `spark-submit` Job. Since v1alpha2 the + operator launches the driver directly as a Kubernetes Job (built from `spec.driver`), so + there is no separate submit process to configure. This field is kept for backwards + compatibility but has no effect. + nullable: true + properties: + cliOverrides: + additionalProperties: + type: string + default: {} + type: object + config: + default: {} + properties: + affinity: + default: + nodeAffinity: null + nodeSelector: null + podAffinity: null + podAntiAffinity: null + description: |- + These configuration settings control + [Pod placement](https://docs.stackable.tech/home/nightly/concepts/operations/pod_placement). + properties: + nodeAffinity: + description: Same as the `spec.affinity.nodeAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + nodeSelector: + additionalProperties: + type: string + description: Simple key-value pairs forming a nodeSelector, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true + type: object + podAffinity: + description: Same as the `spec.affinity.podAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + podAntiAffinity: + description: Same as the `spec.affinity.podAntiAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + requestedSecretLifetime: + description: |- + Request secret (currently only autoTls certificates) lifetime from the secret operator, e.g. `7d`, or `30d`. + This can be shortened by the `maxCertificateLifetime` setting on the SecretClass issuing the TLS certificate. + nullable: true + type: string + resources: + default: + cpu: + max: null + min: null + memory: + limit: null + runtimeLimits: {} + storage: {} + description: |- + Resource usage is configured here, this includes CPU usage, memory usage and disk storage + usage, if this role needs any. + properties: + cpu: + default: + max: null + min: null + properties: + max: + description: |- + The maximum amount of CPU cores that can be requested by Pods. + Equivalent to the `limit` for Pod resource configuration. + Cores are specified either as a decimal point number or as milli units. + For example:`1.5` will be 1.5 cores, also written as `1500m`. + nullable: true + x-kubernetes-int-or-string: true + min: + description: |- + The minimal amount of CPU cores that Pods need to run. + Equivalent to the `request` for Pod resource configuration. + Cores are specified either as a decimal point number or as milli units. + For example:`1.5` will be 1.5 cores, also written as `1500m`. + nullable: true + x-kubernetes-int-or-string: true + type: object + memory: + properties: + limit: + description: |- + The maximum amount of memory that should be available to the Pod. + Specified as a byte [Quantity](https://kubernetes.io/docs/reference/kubernetes-api/common-definitions/quantity/), + which means these suffixes are supported: E, P, T, G, M, k. + You can also use the power-of-two equivalents: Ei, Pi, Ti, Gi, Mi, Ki. + For example, the following represent roughly the same value: + `128974848, 129e6, 129M, 128974848000m, 123Mi` + nullable: true + x-kubernetes-int-or-string: true + runtimeLimits: + description: Additional options that can be specified. + type: object + type: object + storage: + type: object + type: object + retryOnFailureCount: + description: Number of times to retry the submit job on failure. Default is `0` (no retries). + format: uint16 + maximum: 65535.0 + minimum: 0.0 + nullable: true + type: integer + volumeMounts: + description: Volume mounts for the spark-submit, driver and executor pods. + items: + type: object + x-kubernetes-preserve-unknown-fields: true + type: array + type: object + configOverrides: + description: |- + The `configOverrides` can be used to configure properties in product config files + that are not exposed in the CRD. Read the + [config overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#config-overrides) + and consult the operator specific usage guide documentation for details on the + available config files and settings for the specific product. + properties: + security.properties: + additionalProperties: + type: string + description: |- + Flat key-value overrides for `*.properties`, Hadoop XML, etc. - 1. If PodDisruptionBudgets are created by the operator - 2. The allowed number of Pods to be unavailable (`maxUnavailable`) + This is backwards-compatible with the existing flat key-value YAML format + used by `HashMap`. + nullable: true + type: object + spark-env.sh: + additionalProperties: + type: string + description: |- + Flat key-value overrides for `*.properties`, Hadoop XML, etc. - Learn more in the - [allowed Pod disruptions documentation](https://docs.stackable.tech/home/nightly/concepts/operations/pod_disruptions). + This is backwards-compatible with the existing flat key-value YAML format + used by `HashMap`. + nullable: true + type: object + type: object + envOverrides: + additionalProperties: + type: string + default: {} + description: |- + `envOverrides` configure environment variables to be set in the Pods. + It is a map from strings to strings - environment variables and the value to set. + Read the + [environment variable overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#env-overrides) + for more information and consult the operator specific usage guide to find out about + the product specific environment variables that are available. + type: object + jvmArgumentOverrides: + default: + add: [] + remove: [] + removeRegex: [] + description: |- + Allows overriding JVM arguments. + Please read on the [JVM argument overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#jvm-argument-overrides) + for details on the usage. + properties: + add: + default: [] + description: JVM arguments to be added + items: + type: string + type: array + remove: + default: [] + description: JVM arguments to be removed by exact match + items: + type: string + type: array + removeRegex: + default: [] + description: JVM arguments matching any of this regexes will be removed + items: + type: string + type: array + type: object + podOverrides: + default: {} + description: |- + In the `podOverrides` property you can define a + [PodTemplateSpec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.34/#podtemplatespec-v1-core) + to override any property that can be set on a Kubernetes Pod. + Read the + [Pod overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#pod-overrides) + for more information. + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + deps: + default: + excludePackages: [] + packages: [] + repositories: [] + requirements: [] + description: |- + Job dependencies: a list of python packages that will be installed via pip, a list of packages + or repositories that is passed directly to spark-submit, or a list of excluded packages + (also passed directly to spark-submit). + properties: + excludePackages: + default: [] + description: A list of excluded packages that is passed directly to `spark-submit`. + items: + type: string + type: array + packages: + default: [] + description: A list of packages that is passed directly to `spark-submit`. + items: + type: string + type: array + repositories: + default: [] + description: A list of repositories that is passed directly to `spark-submit`. + items: + type: string + type: array + requirements: + default: [] + description: |- + Under the `requirements` you can specify Python dependencies that will be installed with `pip`. + Example: `tabulate==0.8.9` + items: + type: string + type: array + type: object + driver: + description: |- + The driver role specifies the configuration that, together with the driver pod template, is used by + Spark to create driver pods. + nullable: true + properties: + cliOverrides: + additionalProperties: + type: string + default: {} + type: object + config: + default: {} + properties: + affinity: + default: + nodeAffinity: null + nodeSelector: null + podAffinity: null + podAntiAffinity: null + description: |- + These configuration settings control + [Pod placement](https://docs.stackable.tech/home/nightly/concepts/operations/pod_placement). properties: - enabled: - default: true - description: |- - Whether a PodDisruptionBudget should be written out for this role. - Disabling this enables you to specify your own - custom - one. - Defaults to true. - type: boolean - maxUnavailable: - description: |- - The number of Pods that are allowed to be down because of voluntary disruptions. - If you don't explicitly set this, the operator will use a sane default based - upon knowledge about the individual product. - format: uint16 - maximum: 65535.0 - minimum: 0.0 + nodeAffinity: + description: Same as the `spec.affinity.nodeAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + nodeSelector: + additionalProperties: + type: string + description: Simple key-value pairs forming a nodeSelector, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) nullable: true - type: integer + type: object + podAffinity: + description: Same as the `spec.affinity.podAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + podAntiAffinity: + description: Same as the `spec.affinity.podAntiAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true type: object - type: object - roleGroups: - additionalProperties: - properties: - cliOverrides: - additionalProperties: - type: string - default: {} - type: object - config: - default: {} - properties: - affinity: - default: - nodeAffinity: null - nodeSelector: null - podAffinity: null - podAntiAffinity: null - description: |- - These configuration settings control - [Pod placement](https://docs.stackable.tech/home/nightly/concepts/operations/pod_placement). - properties: - nodeAffinity: - description: Same as the `spec.affinity.nodeAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) - nullable: true - type: object - x-kubernetes-preserve-unknown-fields: true - nodeSelector: - additionalProperties: - type: string - description: Simple key-value pairs forming a nodeSelector, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) - nullable: true - type: object - podAffinity: - description: Same as the `spec.affinity.podAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) - nullable: true - type: object - x-kubernetes-preserve-unknown-fields: true - podAntiAffinity: - description: Same as the `spec.affinity.podAntiAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) - nullable: true - type: object - x-kubernetes-preserve-unknown-fields: true - type: object - cleaner: - nullable: true - type: boolean - logging: - default: - containers: {} - enableVectorAgent: null - description: Logging configuration, learn more in the [logging concept documentation](https://docs.stackable.tech/home/nightly/concepts/logging). - properties: - containers: - description: Log configuration per container. - properties: - spark-history: - anyOf: - - required: - - custom - - {} - - {} - description: Log configuration of the container + logging: + default: + containers: {} + enableVectorAgent: null + description: Logging configuration, learn more in the [logging concept documentation](https://docs.stackable.tech/home/nightly/concepts/logging). + properties: + containers: + description: Log configuration per container. + properties: + job: + anyOf: + - required: + - custom + - {} + - {} + description: Log configuration of the container + properties: + console: + description: Configuration for the console appender + nullable: true + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + custom: + description: Log configuration provided in a ConfigMap + properties: + configMap: + description: ConfigMap containing the log configuration files + nullable: true + type: string + type: object + file: + description: Configuration for the file appender + nullable: true + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + loggers: + additionalProperties: + description: Configuration of a logger + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + default: {} + description: Configuration per logger + type: object + type: object + requirements: + anyOf: + - required: + - custom + - {} + - {} + description: Log configuration of the container + properties: + console: + description: Configuration for the console appender + nullable: true + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + custom: + description: Log configuration provided in a ConfigMap + properties: + configMap: + description: ConfigMap containing the log configuration files + nullable: true + type: string + type: object + file: + description: Configuration for the file appender + nullable: true + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + loggers: + additionalProperties: + description: Configuration of a logger + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + default: {} + description: Configuration per logger + type: object + type: object + spark: + anyOf: + - required: + - custom + - {} + - {} + description: Log configuration of the container + properties: + console: + description: Configuration for the console appender + nullable: true + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + custom: + description: Log configuration provided in a ConfigMap + properties: + configMap: + description: ConfigMap containing the log configuration files + nullable: true + type: string + type: object + file: + description: Configuration for the file appender + nullable: true + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + loggers: + additionalProperties: + description: Configuration of a logger properties: - console: - description: Configuration for the console appender + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null nullable: true - properties: - level: - description: |- - The log level threshold. - Log events with a lower log level are discarded. - enum: - - TRACE - - DEBUG - - INFO - - WARN - - ERROR - - FATAL - - NONE - - null - nullable: true - type: string - type: object - custom: - description: Log configuration provided in a ConfigMap - properties: - configMap: - description: ConfigMap containing the log configuration files - nullable: true - type: string - type: object - file: - description: Configuration for the file appender + type: string + type: object + default: {} + description: Configuration per logger + type: object + type: object + spark-submit: + anyOf: + - required: + - custom + - {} + - {} + description: Log configuration of the container + properties: + console: + description: Configuration for the console appender + nullable: true + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + custom: + description: Log configuration provided in a ConfigMap + properties: + configMap: + description: ConfigMap containing the log configuration files + nullable: true + type: string + type: object + file: + description: Configuration for the file appender + nullable: true + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + loggers: + additionalProperties: + description: Configuration of a logger + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null nullable: true - properties: - level: - description: |- - The log level threshold. - Log events with a lower log level are discarded. - enum: - - TRACE - - DEBUG - - INFO - - WARN - - ERROR - - FATAL - - NONE - - null - nullable: true - type: string - type: object - loggers: - additionalProperties: - description: Configuration of a logger - properties: - level: - description: |- - The log level threshold. - Log events with a lower log level are discarded. - enum: - - TRACE - - DEBUG - - INFO - - WARN - - ERROR - - FATAL - - NONE - - null - nullable: true - type: string - type: object - default: {} - description: Configuration per logger - type: object + type: string type: object - vector: - anyOf: - - required: - - custom - - {} - - {} - description: Log configuration of the container + default: {} + description: Configuration per logger + type: object + type: object + tls: + anyOf: + - required: + - custom + - {} + - {} + description: Log configuration of the container + properties: + console: + description: Configuration for the console appender + nullable: true + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + custom: + description: Log configuration provided in a ConfigMap + properties: + configMap: + description: ConfigMap containing the log configuration files + nullable: true + type: string + type: object + file: + description: Configuration for the file appender + nullable: true + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + loggers: + additionalProperties: + description: Configuration of a logger properties: - console: - description: Configuration for the console appender - nullable: true - properties: - level: - description: |- - The log level threshold. - Log events with a lower log level are discarded. - enum: - - TRACE - - DEBUG - - INFO - - WARN - - ERROR - - FATAL - - NONE - - null - nullable: true - type: string - type: object - custom: - description: Log configuration provided in a ConfigMap - properties: - configMap: - description: ConfigMap containing the log configuration files - nullable: true - type: string - type: object - file: - description: Configuration for the file appender + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null nullable: true - properties: - level: - description: |- - The log level threshold. - Log events with a lower log level are discarded. - enum: - - TRACE - - DEBUG - - INFO - - WARN - - ERROR - - FATAL - - NONE - - null - nullable: true - type: string - type: object - loggers: - additionalProperties: - description: Configuration of a logger - properties: - level: - description: |- - The log level threshold. - Log events with a lower log level are discarded. - enum: - - TRACE - - DEBUG - - INFO - - WARN - - ERROR - - FATAL - - NONE - - null - nullable: true - type: string - type: object - default: {} - description: Configuration per logger - type: object - type: object - type: object - enableVectorAgent: - description: Whether or not to deploy a container with the Vector log agent. - nullable: true - type: boolean - type: object - requestedSecretLifetime: - description: |- - Request secret (currently only autoTls certificates) lifetime from the secret operator, e.g. `7d`, or `30d`. - This can be shortened by the `maxCertificateLifetime` setting on the SecretClass issuing the TLS certificate. - nullable: true - type: string - resources: - default: - cpu: - max: null - min: null - memory: - limit: null - runtimeLimits: {} - storage: {} - description: |- - Resource usage is configured here, this includes CPU usage, memory usage and disk storage - usage, if this role needs any. - properties: - cpu: - default: - max: null - min: null - properties: - max: - description: |- - The maximum amount of CPU cores that can be requested by Pods. - Equivalent to the `limit` for Pod resource configuration. - Cores are specified either as a decimal point number or as milli units. - For example:`1.5` will be 1.5 cores, also written as `1500m`. - nullable: true - x-kubernetes-int-or-string: true - min: - description: |- - The minimal amount of CPU cores that Pods need to run. - Equivalent to the `request` for Pod resource configuration. - Cores are specified either as a decimal point number or as milli units. - For example:`1.5` will be 1.5 cores, also written as `1500m`. - nullable: true - x-kubernetes-int-or-string: true - type: object - memory: - properties: - limit: - description: |- - The maximum amount of memory that should be available to the Pod. - Specified as a byte [Quantity](https://kubernetes.io/docs/reference/kubernetes-api/common-definitions/quantity/), - which means these suffixes are supported: E, P, T, G, M, k. - You can also use the power-of-two equivalents: Ei, Pi, Ti, Gi, Mi, Ki. - For example, the following represent roughly the same value: - `128974848, 129e6, 129M, 128974848000m, 123Mi` - nullable: true - x-kubernetes-int-or-string: true - runtimeLimits: - description: Additional options that can be specified. + type: string type: object - type: object - storage: - type: object - type: object + default: {} + description: Configuration per logger + type: object + type: object + vector: + anyOf: + - required: + - custom + - {} + - {} + description: Log configuration of the container + properties: + console: + description: Configuration for the console appender + nullable: true + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + custom: + description: Log configuration provided in a ConfigMap + properties: + configMap: + description: ConfigMap containing the log configuration files + nullable: true + type: string + type: object + file: + description: Configuration for the file appender + nullable: true + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + loggers: + additionalProperties: + description: Configuration of a logger + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + default: {} + description: Configuration per logger + type: object + type: object + type: object + enableVectorAgent: + description: Whether or not to deploy a container with the Vector log agent. + nullable: true + type: boolean + type: object + requestedSecretLifetime: + description: |- + Request secret (currently only autoTls certificates) lifetime from the secret operator, e.g. `7d`, or `30d`. + This can be shortened by the `maxCertificateLifetime` setting on the SecretClass issuing the TLS certificate. + nullable: true + type: string + resources: + default: + cpu: + max: null + min: null + memory: + limit: null + runtimeLimits: {} + storage: {} + description: |- + Resource usage is configured here, this includes CPU usage, memory usage and disk storage + usage, if this role needs any. + properties: + cpu: + default: + max: null + min: null + properties: + max: + description: |- + The maximum amount of CPU cores that can be requested by Pods. + Equivalent to the `limit` for Pod resource configuration. + Cores are specified either as a decimal point number or as milli units. + For example:`1.5` will be 1.5 cores, also written as `1500m`. + nullable: true + x-kubernetes-int-or-string: true + min: + description: |- + The minimal amount of CPU cores that Pods need to run. + Equivalent to the `request` for Pod resource configuration. + Cores are specified either as a decimal point number or as milli units. + For example:`1.5` will be 1.5 cores, also written as `1500m`. + nullable: true + x-kubernetes-int-or-string: true + type: object + memory: + properties: + limit: + description: |- + The maximum amount of memory that should be available to the Pod. + Specified as a byte [Quantity](https://kubernetes.io/docs/reference/kubernetes-api/common-definitions/quantity/), + which means these suffixes are supported: E, P, T, G, M, k. + You can also use the power-of-two equivalents: Ei, Pi, Ti, Gi, Mi, Ki. + For example, the following represent roughly the same value: + `128974848, 129e6, 129M, 128974848000m, 123Mi` + nullable: true + x-kubernetes-int-or-string: true + runtimeLimits: + description: Additional options that can be specified. + type: object + type: object + storage: + type: object + type: object + volumeMounts: + description: Volume mounts for the spark-submit, driver and executor pods. + items: type: object - configOverrides: - description: |- - The `configOverrides` can be used to configure properties in product config files - that are not exposed in the CRD. Read the - [config overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#config-overrides) - and consult the operator specific usage guide documentation for details on the - available config files and settings for the specific product. - properties: - security.properties: - additionalProperties: - type: string - description: |- - Flat key-value overrides for `*.properties`, Hadoop XML, etc. + x-kubernetes-preserve-unknown-fields: true + type: array + type: object + configOverrides: + description: |- + The `configOverrides` can be used to configure properties in product config files + that are not exposed in the CRD. Read the + [config overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#config-overrides) + and consult the operator specific usage guide documentation for details on the + available config files and settings for the specific product. + properties: + security.properties: + additionalProperties: + type: string + description: |- + Flat key-value overrides for `*.properties`, Hadoop XML, etc. - This is backwards-compatible with the existing flat key-value YAML format - used by `HashMap`. - nullable: true - type: object - spark-defaults.conf: - additionalProperties: - type: string - description: |- - Flat key-value overrides for `*.properties`, Hadoop XML, etc. + This is backwards-compatible with the existing flat key-value YAML format + used by `HashMap`. + nullable: true + type: object + spark-env.sh: + additionalProperties: + type: string + description: |- + Flat key-value overrides for `*.properties`, Hadoop XML, etc. - This is backwards-compatible with the existing flat key-value YAML format - used by `HashMap`. - nullable: true - type: object - spark-env.sh: - additionalProperties: - type: string + This is backwards-compatible with the existing flat key-value YAML format + used by `HashMap`. + nullable: true + type: object + type: object + envOverrides: + additionalProperties: + type: string + default: {} + description: |- + `envOverrides` configure environment variables to be set in the Pods. + It is a map from strings to strings - environment variables and the value to set. + Read the + [environment variable overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#env-overrides) + for more information and consult the operator specific usage guide to find out about + the product specific environment variables that are available. + type: object + jvmArgumentOverrides: + default: + add: [] + remove: [] + removeRegex: [] + description: |- + Allows overriding JVM arguments. + Please read on the [JVM argument overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#jvm-argument-overrides) + for details on the usage. + properties: + add: + default: [] + description: JVM arguments to be added + items: + type: string + type: array + remove: + default: [] + description: JVM arguments to be removed by exact match + items: + type: string + type: array + removeRegex: + default: [] + description: JVM arguments matching any of this regexes will be removed + items: + type: string + type: array + type: object + podOverrides: + default: {} + description: |- + In the `podOverrides` property you can define a + [PodTemplateSpec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.34/#podtemplatespec-v1-core) + to override any property that can be set on a Kubernetes Pod. + Read the + [Pod overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#pod-overrides) + for more information. + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + env: + default: [] + description: |- + A list of environment variables that will be set in the job pod and the driver and executor + pod templates. + items: + description: EnvVar represents an environment variable present in a Container. + properties: + name: + description: Name of the environment variable. May consist of any printable ASCII characters except '='. + type: string + value: + description: 'Variable references $(VAR_NAME) are expanded using the previously defined environment variables in the container and any service environment variables. If a variable cannot be resolved, the reference in the input string will be unchanged. Double $$ are reduced to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". Escaped references will never be expanded, regardless of whether the variable exists or not. Defaults to "".' + type: string + valueFrom: + description: Source for the environment variable's value. Cannot be used if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + description: 'Name of the referent. This field is effectively required, but due to backwards compatibility is allowed to be empty. Instances of this type with an empty value here are almost certainly wrong. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + optional: + description: Specify whether the ConfigMap or its key must be defined + type: boolean + required: + - key + - name + type: object + fieldRef: + description: 'Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['''']`, `metadata.annotations['''']`, spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs.' + properties: + apiVersion: + description: Version of the schema the FieldPath is written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select in the specified API version. + type: string + required: + - fieldPath + type: object + fileKeyRef: + description: FileKeyRef selects a key of the env file. Requires the EnvFiles feature gate to be enabled. + properties: + key: + description: The key within the env file. An invalid key will prevent the pod from starting. The keys defined within a source may consist of any printable ASCII characters except '='. During Alpha stage of the EnvFiles feature gate, the key size is limited to 128 characters. + type: string + optional: description: |- - Flat key-value overrides for `*.properties`, Hadoop XML, etc. + Specify whether the file or its key must be defined. If the file or key does not exist, then the env var is not published. If optional is set to true and the specified key does not exist, the environment variable will not be set in the Pod's containers. - This is backwards-compatible with the existing flat key-value YAML format - used by `HashMap`. - nullable: true - type: object - type: object - envOverrides: - additionalProperties: - type: string - default: {} - description: |- - `envOverrides` configure environment variables to be set in the Pods. - It is a map from strings to strings - environment variables and the value to set. - Read the - [environment variable overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#env-overrides) - for more information and consult the operator specific usage guide to find out about - the product specific environment variables that are available. + If optional is set to false and the specified key does not exist, an error will be returned during Pod creation. + type: boolean + path: + description: The path within the volume from which to select the file. Must be relative and may not contain the '..' path or start with '..'. + type: string + volumeName: + description: The name of the volume mount containing the env file. + type: string + required: + - key + - path + - volumeName type: object - jvmArgumentOverrides: - default: - add: [] - remove: [] - removeRegex: [] - description: |- - Allows overriding JVM arguments. - Please read on the [JVM argument overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#jvm-argument-overrides) - for details on the usage. + resourceFieldRef: + description: 'Selects a resource of the container: only resources limits and requests (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported.' properties: - add: - default: [] - description: JVM arguments to be added - items: - type: string - type: array - remove: - default: [] - description: JVM arguments to be removed by exact match - items: - type: string - type: array - removeRegex: - default: [] - description: JVM arguments matching any of this regexes will be removed - items: - type: string - type: array + containerName: + description: 'Container name: required for volumes, optional for env vars' + type: string + divisor: + description: Specifies the output format of the exposed resources, defaults to "1" + nullable: true + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource type: object - podOverrides: - default: {} - description: |- - In the `podOverrides` property you can define a - [PodTemplateSpec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.34/#podtemplatespec-v1-core) - to override any property that can be set on a Kubernetes Pod. - Read the - [Pod overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#pod-overrides) - for more information. + secretKeyRef: + description: Selects a key of a secret in the pod's namespace + properties: + key: + description: The key of the secret to select from. Must be a valid secret key. + type: string + name: + description: 'Name of the referent. This field is effectively required, but due to backwards compatibility is allowed to be empty. Instances of this type with an empty value here are almost certainly wrong. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + optional: + description: Specify whether the Secret or its key must be defined + type: boolean + required: + - key + - name type: object - x-kubernetes-preserve-unknown-fields: true - replicas: - format: uint16 - maximum: 65535.0 - minimum: 0.0 - nullable: true - type: integer type: object - type: object - required: - - roleGroups - type: object - objectOverrides: - default: [] - description: |- - A list of generic Kubernetes objects, which are merged into the objects that the operator - creates. - - List entries are arbitrary YAML objects, which need to be valid Kubernetes objects. - - Read the [Object overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#object-overrides) - for more information. - items: + required: + - name type: object - x-kubernetes-preserve-unknown-fields: true type: array - sparkConf: - additionalProperties: - type: string - default: {} - description: A map of key/value strings that will be passed directly to Spark when deploying the history server. - type: object - vectorAggregatorConfigMapName: + executor: description: |- - Name of the Vector aggregator discovery ConfigMap. - It must contain the key `ADDRESS` with the address of the Vector aggregator. + The executor role specifies the configuration that, together with the driver pod template, is used by + Spark to create the executor pods. + This is RoleGroup instead of plain CommonConfiguration because it needs to allow for the number of replicas. + to be specified. nullable: true - type: string - required: - - image - - logFileDirectory - - nodes - type: object - required: - - spec - title: SparkHistoryServer - type: object - served: true - storage: true - subresources: {} ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: sparkconnectservers.spark.stackable.tech -spec: - group: spark.stackable.tech - names: - categories: [] - kind: SparkConnectServer - plural: sparkconnectservers - shortNames: - - sparkconnect - singular: sparkconnectserver - scope: Namespaced - versions: - - additionalPrinterColumns: [] - name: v1alpha1 - schema: - openAPIV3Schema: - description: Auto-generated derived type for SparkConnectServerSpec via `CustomResource` - properties: - spec: - description: |- - An Apache Spark Connect server component. This resource is managed by the Stackable operator - for Apache Spark. Find more information on how to use it in the - [operator documentation](https://docs.stackable.tech/home/nightly/spark-k8s/usage-guide/connect-server). - properties: - args: - default: [] - description: User provided command line arguments appended to the server entry point. - items: - type: string - type: array - clusterOperation: - default: - reconciliationPaused: false - stopped: false - description: |- - [Cluster operations](https://docs.stackable.tech/home/nightly/concepts/operations/cluster_operations) - properties, allow stopping the product instance as well as pausing reconciliation. - properties: - reconciliationPaused: - default: false - description: |- - Flag to stop cluster reconciliation by the operator. This means that all changes in the - custom resource spec are ignored until this flag is set to false or removed. The operator - will however still watch the deployed resources at the time and update the custom resource - status field. - If applied at the same time with `stopped`, `reconciliationPaused` will take precedence over - `stopped` and stop the reconciliation immediately. - type: boolean - stopped: - default: false - description: |- - Flag to stop the cluster. This means all deployed resources (e.g. Services, StatefulSets, - ConfigMaps) are kept but all deployed Pods (e.g. replicas from a StatefulSet) are scaled to 0 - and therefore stopped and removed. - If applied at the same time with `reconciliationPaused`, the latter will pause reconciliation - and `stopped` will take no effect until `reconciliationPaused` is set to false or removed. - type: boolean - type: object - connectors: - default: - s3buckets: [] - description: One or more S3 connections to be used by the Spark Connect server. properties: - s3buckets: - default: [] - items: - oneOf: - - required: - - inline - - required: - - reference - properties: - inline: - description: |- - S3 bucket specification containing the bucket name and an inlined or referenced connection specification. - Learn more on the [S3 concept documentation](https://docs.stackable.tech/home/nightly/concepts/s3). - properties: - bucketName: - description: The name of the S3 bucket. - type: string - connection: - description: The definition of an S3 connection, either inline or as a reference. - oneOf: - - required: - - inline - - required: - - reference - properties: - inline: - description: |- - S3 connection definition as a resource. - Learn more on the [S3 concept documentation](https://docs.stackable.tech/home/nightly/concepts/s3). - properties: - accessStyle: - default: VirtualHosted - description: |- - Which access style to use. - Defaults to virtual hosted-style as most of the data products out there. - Have a look at the [AWS documentation](https://docs.aws.amazon.com/AmazonS3/latest/userguide/VirtualHosting.html). - enum: - - Path - - VirtualHosted - type: string - credentials: - description: |- - If the S3 uses authentication you have to specify you S3 credentials. - In the most cases a [SecretClass](https://docs.stackable.tech/home/nightly/secret-operator/secretclass) - providing `accessKey` and `secretKey` is sufficient. - nullable: true + cliOverrides: + additionalProperties: + type: string + default: {} + type: object + config: + default: {} + properties: + affinity: + default: + nodeAffinity: null + nodeSelector: null + podAffinity: null + podAntiAffinity: null + description: |- + These configuration settings control + [Pod placement](https://docs.stackable.tech/home/nightly/concepts/operations/pod_placement). + properties: + nodeAffinity: + description: Same as the `spec.affinity.nodeAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + nodeSelector: + additionalProperties: + type: string + description: Simple key-value pairs forming a nodeSelector, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true + type: object + podAffinity: + description: Same as the `spec.affinity.podAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + podAntiAffinity: + description: Same as the `spec.affinity.podAntiAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + logging: + default: + containers: {} + enableVectorAgent: null + description: Logging configuration, learn more in the [logging concept documentation](https://docs.stackable.tech/home/nightly/concepts/logging). + properties: + containers: + description: Log configuration per container. + properties: + job: + anyOf: + - required: + - custom + - {} + - {} + description: Log configuration of the container + properties: + console: + description: Configuration for the console appender + nullable: true + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + custom: + description: Log configuration provided in a ConfigMap + properties: + configMap: + description: ConfigMap containing the log configuration files + nullable: true + type: string + type: object + file: + description: Configuration for the file appender + nullable: true + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + loggers: + additionalProperties: + description: Configuration of a logger properties: - scope: + level: description: |- - [Scope](https://docs.stackable.tech/home/nightly/secret-operator/scope) of the - [SecretClass](https://docs.stackable.tech/home/nightly/secret-operator/secretclass). + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null nullable: true - properties: - listenerVolumes: - default: [] - description: |- - The listener volume scope allows Node and Service scopes to be inferred from the applicable listeners. - This must correspond to Volume names in the Pod that mount Listeners. - items: - type: string - type: array - node: - default: false - description: |- - The node scope is resolved to the name of the Kubernetes Node object that the Pod is running on. - This will typically be the DNS name of the node. - type: boolean - pod: - default: false - description: |- - The pod scope is resolved to the name of the Kubernetes Pod. - This allows the secret to differentiate between StatefulSet replicas. - type: boolean - services: - default: [] - description: |- - The service scope allows Pod objects to specify custom scopes. - This should typically correspond to Service objects that the Pod participates in. - items: - type: string - type: array - type: object - secretClass: - description: '[SecretClass](https://docs.stackable.tech/home/nightly/secret-operator/secretclass) containing the LDAP bind credentials.' type: string - required: - - secretClass type: object - host: - description: 'Host of the S3 server without any protocol or port. For example: `west1.my-cloud.com`.' - type: string - port: - description: |- - Port the S3 server listens on. - If not specified the product will determine the port to use. - format: uint16 - maximum: 65535.0 - minimum: 0.0 - nullable: true - type: integer - region: - default: - name: us-east-1 - description: |- - Bucket region used for signing headers (sigv4). - - This defaults to `us-east-1` which is compatible with other implementations such as Minio. - - WARNING: Some products use the Hadoop S3 implementation which falls back to us-east-2. + default: {} + description: Configuration per logger + type: object + type: object + requirements: + anyOf: + - required: + - custom + - {} + - {} + description: Log configuration of the container + properties: + console: + description: Configuration for the console appender + nullable: true + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + custom: + description: Log configuration provided in a ConfigMap + properties: + configMap: + description: ConfigMap containing the log configuration files + nullable: true + type: string + type: object + file: + description: Configuration for the file appender + nullable: true + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + loggers: + additionalProperties: + description: Configuration of a logger + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + default: {} + description: Configuration per logger + type: object + type: object + spark: + anyOf: + - required: + - custom + - {} + - {} + description: Log configuration of the container + properties: + console: + description: Configuration for the console appender + nullable: true + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + custom: + description: Log configuration provided in a ConfigMap + properties: + configMap: + description: ConfigMap containing the log configuration files + nullable: true + type: string + type: object + file: + description: Configuration for the file appender + nullable: true + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + loggers: + additionalProperties: + description: Configuration of a logger properties: - name: - default: us-east-1 + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true type: string type: object - tls: - description: Use a TLS connection. If not specified no TLS will be used. - nullable: true - properties: - verification: - description: The verification method used to verify the certificates of the server and/or the client. - oneOf: - - required: - - none - - required: - - server - properties: - none: - description: Use TLS but don't verify certificates. - type: object - server: - description: Use TLS and a CA certificate to verify the server. - properties: - caCert: - description: CA cert to verify the server. - oneOf: - - required: - - webPki - - required: - - secretClass - properties: - secretClass: - description: |- - Name of the [SecretClass](https://docs.stackable.tech/home/nightly/secret-operator/secretclass) which will provide the CA certificate. - Note that a SecretClass does not need to have a key but can also work with just a CA certificate, - so if you got provided with a CA cert but don't have access to the key you can still use this method. - type: string - webPki: - description: |- - Use TLS and the CA certificates trusted by the common web browsers to verify the server. - This can be useful when you e.g. use public AWS S3 or other public available services. - type: object - type: object - required: - - caCert - type: object - type: object - required: - - verification - type: object - required: - - host - type: object - reference: - type: string - type: object - required: - - bucketName - - connection - type: object - reference: - type: string - type: object - type: array - s3connection: - nullable: true - oneOf: - - required: - - inline - - required: - - reference - properties: - inline: - description: |- - S3 connection definition as a resource. - Learn more on the [S3 concept documentation](https://docs.stackable.tech/home/nightly/concepts/s3). - properties: - accessStyle: - default: VirtualHosted - description: |- - Which access style to use. - Defaults to virtual hosted-style as most of the data products out there. - Have a look at the [AWS documentation](https://docs.aws.amazon.com/AmazonS3/latest/userguide/VirtualHosting.html). - enum: - - Path - - VirtualHosted - type: string - credentials: - description: |- - If the S3 uses authentication you have to specify you S3 credentials. - In the most cases a [SecretClass](https://docs.stackable.tech/home/nightly/secret-operator/secretclass) - providing `accessKey` and `secretKey` is sufficient. - nullable: true - properties: - scope: - description: |- - [Scope](https://docs.stackable.tech/home/nightly/secret-operator/scope) of the - [SecretClass](https://docs.stackable.tech/home/nightly/secret-operator/secretclass). - nullable: true - properties: - listenerVolumes: - default: [] - description: |- - The listener volume scope allows Node and Service scopes to be inferred from the applicable listeners. - This must correspond to Volume names in the Pod that mount Listeners. - items: - type: string - type: array - node: - default: false - description: |- - The node scope is resolved to the name of the Kubernetes Node object that the Pod is running on. - This will typically be the DNS name of the node. - type: boolean - pod: - default: false - description: |- - The pod scope is resolved to the name of the Kubernetes Pod. - This allows the secret to differentiate between StatefulSet replicas. - type: boolean - services: - default: [] - description: |- - The service scope allows Pod objects to specify custom scopes. - This should typically correspond to Service objects that the Pod participates in. - items: - type: string - type: array + default: {} + description: Configuration per logger + type: object type: object - secretClass: - description: '[SecretClass](https://docs.stackable.tech/home/nightly/secret-operator/secretclass) containing the LDAP bind credentials.' - type: string - required: - - secretClass - type: object - host: - description: 'Host of the S3 server without any protocol or port. For example: `west1.my-cloud.com`.' - type: string - port: - description: |- - Port the S3 server listens on. - If not specified the product will determine the port to use. - format: uint16 - maximum: 65535.0 - minimum: 0.0 - nullable: true - type: integer - region: - default: - name: us-east-1 - description: |- - Bucket region used for signing headers (sigv4). - - This defaults to `us-east-1` which is compatible with other implementations such as Minio. - - WARNING: Some products use the Hadoop S3 implementation which falls back to us-east-2. - properties: - name: - default: us-east-1 - type: string - type: object - tls: - description: Use a TLS connection. If not specified no TLS will be used. - nullable: true - properties: - verification: - description: The verification method used to verify the certificates of the server and/or the client. - oneOf: - - required: - - none + spark-submit: + anyOf: - required: - - server + - custom + - {} + - {} + description: Log configuration of the container properties: - none: - description: Use TLS but don't verify certificates. + console: + description: Configuration for the console appender + nullable: true + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string type: object - server: - description: Use TLS and a CA certificate to verify the server. + custom: + description: Log configuration provided in a ConfigMap properties: - caCert: - description: CA cert to verify the server. - oneOf: - - required: - - webPki - - required: - - secretClass - properties: - secretClass: - description: |- - Name of the [SecretClass](https://docs.stackable.tech/home/nightly/secret-operator/secretclass) which will provide the CA certificate. - Note that a SecretClass does not need to have a key but can also work with just a CA certificate, - so if you got provided with a CA cert but don't have access to the key you can still use this method. - type: string - webPki: - description: |- - Use TLS and the CA certificates trusted by the common web browsers to verify the server. - This can be useful when you e.g. use public AWS S3 or other public available services. - type: object - type: object - required: - - caCert + configMap: + description: ConfigMap containing the log configuration files + nullable: true + type: string + type: object + file: + description: Configuration for the file appender + nullable: true + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + loggers: + additionalProperties: + description: Configuration of a logger + properties: + level: + description: |- + The log level threshold. + Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + - null + nullable: true + type: string + type: object + default: {} + description: Configuration per logger type: object type: object - required: - - verification - type: object - required: - - host - type: object - reference: - type: string - type: object - type: object - executor: - description: Spark Connect executor properties. - nullable: true - properties: - cliOverrides: - additionalProperties: - type: string - default: {} - type: object - config: - default: {} - properties: - affinity: - default: - nodeAffinity: null - nodeSelector: null - podAffinity: null - podAntiAffinity: null - description: |- - These configuration settings control - [Pod placement](https://docs.stackable.tech/home/nightly/concepts/operations/pod_placement). - properties: - nodeAffinity: - description: Same as the `spec.affinity.nodeAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) - nullable: true - type: object - x-kubernetes-preserve-unknown-fields: true - nodeSelector: - additionalProperties: - type: string - description: Simple key-value pairs forming a nodeSelector, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) - nullable: true - type: object - podAffinity: - description: Same as the `spec.affinity.podAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) - nullable: true - type: object - x-kubernetes-preserve-unknown-fields: true - podAntiAffinity: - description: Same as the `spec.affinity.podAntiAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) - nullable: true - type: object - x-kubernetes-preserve-unknown-fields: true - type: object - logging: - default: - containers: {} - enableVectorAgent: null - description: Logging configuration, learn more in the [logging concept documentation](https://docs.stackable.tech/home/nightly/concepts/logging). - properties: - containers: - description: Log configuration per container. - properties: - spark: + tls: anyOf: - required: - custom @@ -4042,6 +8576,12 @@ spec: storage: type: object type: object + volumeMounts: + description: Volume mounts for the spark-submit, driver and executor pods. + items: + type: object + x-kubernetes-preserve-unknown-fields: true + type: array type: object configOverrides: description: |- @@ -4051,16 +8591,6 @@ spec: and consult the operator specific usage guide documentation for details on the available config files and settings for the specific product. properties: - metrics.properties: - additionalProperties: - type: string - description: |- - Flat key-value overrides for `*.properties`, Hadoop XML, etc. - - This is backwards-compatible with the existing flat key-value YAML format - used by `HashMap`. - nullable: true - type: object security.properties: additionalProperties: type: string @@ -4071,7 +8601,7 @@ spec: used by `HashMap`. nullable: true type: object - spark-defaults.conf: + spark-env.sh: additionalProperties: type: string description: |- @@ -4134,513 +8664,477 @@ spec: for more information. type: object x-kubernetes-preserve-unknown-fields: true - type: object - image: - anyOf: - - required: - - custom - - productVersion - - required: - - productVersion - description: |- - Specify which image to use, the easiest way is to only configure the `productVersion`. - You can also configure a custom image registry to pull from, as well as completely custom - images. - - Consult the [Product image selection documentation](https://docs.stackable.tech/home/nightly/concepts/product_image_selection) - for details. - properties: - custom: - description: |- - Provide a custom container image. - - Specify the full container image name, e.g. `oci.example.tech/namespace/superset:1.4.1-my-tag` - type: string - productVersion: - description: Version of the product, e.g. `1.4.1`. - type: string - pullPolicy: - default: Always - description: '[Pull policy](https://kubernetes.io/docs/concepts/containers/images/#image-pull-policy) used when pulling the image.' - enum: - - IfNotPresent - - Always - - Never - type: string - pullSecrets: - description: '[Image pull secrets](https://kubernetes.io/docs/concepts/containers/images/#specifying-imagepullsecrets-on-a-pod) to pull images from a private registry.' - items: - description: LocalObjectReference contains enough information to let you locate the referenced object inside the same namespace. - properties: - name: - description: 'Name of the referent. This field is effectively required, but due to backwards compatibility is allowed to be empty. Instances of this type with an empty value here are almost certainly wrong. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' - type: string - required: - - name - type: object - nullable: true - type: array - repo: - description: |- - The repository on the container image registry where the container image is located, e.g. - `oci.example.com/namespace`. - - If not specified, the operator will use the image registry provided via the operator - environment options. - nullable: true - type: string - stackableVersion: - description: |- - Stackable version of the product, e.g. `23.4`, `23.4.1` or `0.0.0-dev`. - - If not specified, the operator will use its own version, e.g. `23.4.1`. When using a nightly - operator or a PR version, it will use the nightly `0.0.0-dev` image. + replicas: + format: uint16 + maximum: 65535.0 + minimum: 0.0 nullable: true - type: string + type: integer type: object - objectOverrides: - default: [] + image: description: |- - A list of generic Kubernetes objects, which are merged into the objects that the operator - creates. - - List entries are arbitrary YAML objects, which need to be valid Kubernetes objects. - - Read the [Object overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#object-overrides) - for more information. - items: - type: object - x-kubernetes-preserve-unknown-fields: true - type: array - server: - default: - roleConfig: - listenerClass: cluster-internal - description: A Spark Connect server definition. + User-supplied image containing spark-job dependencies that will be copied to the specified volume mount. + See the [examples](https://docs.stackable.tech/home/nightly/spark-k8s/usage-guide/examples). + nullable: true + type: string + logFileDirectory: + description: The log file directory definition used by the Spark history server. + nullable: true + oneOf: + - required: + - s3 + - required: + - customLogDirectory properties: - cliOverrides: - additionalProperties: - type: string - default: {} - type: object - config: - default: {} + customLogDirectory: + description: A custom log directory + type: string + s3: + description: An S3 bucket storing the log events properties: - logging: - default: - containers: {} - enableVectorAgent: null - description: Logging configuration, learn more in the [logging concept documentation](https://docs.stackable.tech/home/nightly/concepts/logging). + bucket: + oneOf: + - required: + - inline + - required: + - reference properties: - containers: - description: Log configuration per container. + inline: + description: |- + S3 bucket specification containing the bucket name and an inlined or referenced connection specification. + Learn more on the [S3 concept documentation](https://docs.stackable.tech/home/nightly/concepts/s3). properties: - spark: - anyOf: + bucketName: + description: The name of the S3 bucket. + type: string + connection: + description: The definition of an S3 connection, either inline or as a reference. + oneOf: - required: - - custom - - {} - - {} - description: Log configuration of the container + - inline + - required: + - reference properties: - console: - description: Configuration for the console appender - nullable: true + inline: + description: |- + S3 connection definition as a resource. + Learn more on the [S3 concept documentation](https://docs.stackable.tech/home/nightly/concepts/s3). properties: - level: + accessStyle: + default: VirtualHosted description: |- - The log level threshold. - Log events with a lower log level are discarded. + Which access style to use. + Defaults to virtual hosted-style as most of the data products out there. + Have a look at the [AWS documentation](https://docs.aws.amazon.com/AmazonS3/latest/userguide/VirtualHosting.html). enum: - - TRACE - - DEBUG - - INFO - - WARN - - ERROR - - FATAL - - NONE - - null - nullable: true - type: string - type: object - custom: - description: Log configuration provided in a ConfigMap - properties: - configMap: - description: ConfigMap containing the log configuration files - nullable: true + - Path + - VirtualHosted type: string - type: object - file: - description: Configuration for the file appender - nullable: true - properties: - level: + credentials: description: |- - The log level threshold. - Log events with a lower log level are discarded. - enum: - - TRACE - - DEBUG - - INFO - - WARN - - ERROR - - FATAL - - NONE - - null + If the S3 uses authentication you have to specify you S3 credentials. + In the most cases a [SecretClass](https://docs.stackable.tech/home/nightly/secret-operator/secretclass) + providing `accessKey` and `secretKey` is sufficient. nullable: true + properties: + scope: + description: |- + [Scope](https://docs.stackable.tech/home/nightly/secret-operator/scope) of the + [SecretClass](https://docs.stackable.tech/home/nightly/secret-operator/secretclass). + nullable: true + properties: + listenerVolumes: + default: [] + description: |- + The listener volume scope allows Node and Service scopes to be inferred from the applicable listeners. + This must correspond to Volume names in the Pod that mount Listeners. + items: + type: string + type: array + node: + default: false + description: |- + The node scope is resolved to the name of the Kubernetes Node object that the Pod is running on. + This will typically be the DNS name of the node. + type: boolean + pod: + default: false + description: |- + The pod scope is resolved to the name of the Kubernetes Pod. + This allows the secret to differentiate between StatefulSet replicas. + type: boolean + services: + default: [] + description: |- + The service scope allows Pod objects to specify custom scopes. + This should typically correspond to Service objects that the Pod participates in. + items: + type: string + type: array + type: object + secretClass: + description: '[SecretClass](https://docs.stackable.tech/home/nightly/secret-operator/secretclass) containing the LDAP bind credentials.' + type: string + required: + - secretClass + type: object + host: + description: 'Host of the S3 server without any protocol or port. For example: `west1.my-cloud.com`.' type: string - type: object - loggers: - additionalProperties: - description: Configuration of a logger - properties: - level: - description: |- - The log level threshold. - Log events with a lower log level are discarded. - enum: - - TRACE - - DEBUG - - INFO - - WARN - - ERROR - - FATAL - - NONE - - null - nullable: true - type: string - type: object - default: {} - description: Configuration per logger - type: object - type: object - vector: - anyOf: - - required: - - custom - - {} - - {} - description: Log configuration of the container - properties: - console: - description: Configuration for the console appender - nullable: true - properties: - level: + port: description: |- - The log level threshold. - Log events with a lower log level are discarded. - enum: - - TRACE - - DEBUG - - INFO - - WARN - - ERROR - - FATAL - - NONE - - null - nullable: true - type: string - type: object - custom: - description: Log configuration provided in a ConfigMap - properties: - configMap: - description: ConfigMap containing the log configuration files + Port the S3 server listens on. + If not specified the product will determine the port to use. + format: uint16 + maximum: 65535.0 + minimum: 0.0 nullable: true - type: string - type: object - file: - description: Configuration for the file appender - nullable: true - properties: - level: + type: integer + region: + default: + name: us-east-1 description: |- - The log level threshold. - Log events with a lower log level are discarded. - enum: - - TRACE - - DEBUG - - INFO - - WARN - - ERROR - - FATAL - - NONE - - null - nullable: true - type: string - type: object - loggers: - additionalProperties: - description: Configuration of a logger - properties: - level: - description: |- - The log level threshold. - Log events with a lower log level are discarded. - enum: - - TRACE - - DEBUG - - INFO - - WARN - - ERROR - - FATAL - - NONE - - null - nullable: true - type: string - type: object - default: {} - description: Configuration per logger + Bucket region used for signing headers (sigv4). + + This defaults to `us-east-1` which is compatible with other implementations such as Minio. + + WARNING: Some products use the Hadoop S3 implementation which falls back to us-east-2. + properties: + name: + default: us-east-1 + type: string + type: object + tls: + description: Use a TLS connection. If not specified no TLS will be used. + nullable: true + properties: + verification: + description: The verification method used to verify the certificates of the server and/or the client. + oneOf: + - required: + - none + - required: + - server + properties: + none: + description: Use TLS but don't verify certificates. + type: object + server: + description: Use TLS and a CA certificate to verify the server. + properties: + caCert: + description: CA cert to verify the server. + oneOf: + - required: + - webPki + - required: + - secretClass + properties: + secretClass: + description: |- + Name of the [SecretClass](https://docs.stackable.tech/home/nightly/secret-operator/secretclass) which will provide the CA certificate. + Note that a SecretClass does not need to have a key but can also work with just a CA certificate, + so if you got provided with a CA cert but don't have access to the key you can still use this method. + type: string + webPki: + description: |- + Use TLS and the CA certificates trusted by the common web browsers to verify the server. + This can be useful when you e.g. use public AWS S3 or other public available services. + type: object + type: object + required: + - caCert + type: object + type: object + required: + - verification + type: object + required: + - host type: object + reference: + type: string type: object + required: + - bucketName + - connection type: object - enableVectorAgent: - description: Whether or not to deploy a container with the Vector log agent. - nullable: true - type: boolean + reference: + type: string type: object - requestedSecretLifetime: + prefix: + type: string + required: + - bucket + - prefix + type: object + type: object + mainApplicationFile: + description: The actual application file that will be called by `spark-submit`. + type: string + mainClass: + description: The main class - i.e. entry point - for JVM artifacts. + nullable: true + type: string + mode: + description: |- + Deprecated and ignored. + + Since v1alpha2 the operator no longer runs a separate `spark-submit` process. The driver + is launched directly as a Kubernetes Job running in client mode, so the deploy mode is + always client internally. This field is kept for backwards compatibility but has no + effect. + enum: + - cluster + - client + type: string + s3connection: + description: |- + Configure an S3 connection that the SparkApplication has access to. + Read more in the [Spark S3 usage guide](https://docs.stackable.tech/home/nightly/spark-k8s/usage-guide/s3). + nullable: true + oneOf: + - required: + - inline + - required: + - reference + properties: + inline: + description: |- + S3 connection definition as a resource. + Learn more on the [S3 concept documentation](https://docs.stackable.tech/home/nightly/concepts/s3). + properties: + accessStyle: + default: VirtualHosted description: |- - Request secret (currently only autoTls certificates) lifetime from the secret operator, e.g. `7d`, or `30d`. - This can be shortened by the `maxCertificateLifetime` setting on the SecretClass issuing the TLS certificate. - nullable: true + Which access style to use. + Defaults to virtual hosted-style as most of the data products out there. + Have a look at the [AWS documentation](https://docs.aws.amazon.com/AmazonS3/latest/userguide/VirtualHosting.html). + enum: + - Path + - VirtualHosted type: string - resources: - default: - cpu: - max: null - min: null - memory: - limit: null - runtimeLimits: {} - storage: {} + credentials: description: |- - Resource usage is configured here, this includes CPU usage, memory usage and disk storage - usage, if this role needs any. + If the S3 uses authentication you have to specify you S3 credentials. + In the most cases a [SecretClass](https://docs.stackable.tech/home/nightly/secret-operator/secretclass) + providing `accessKey` and `secretKey` is sufficient. + nullable: true properties: - cpu: - default: - max: null - min: null + scope: + description: |- + [Scope](https://docs.stackable.tech/home/nightly/secret-operator/scope) of the + [SecretClass](https://docs.stackable.tech/home/nightly/secret-operator/secretclass). + nullable: true properties: - max: + listenerVolumes: + default: [] description: |- - The maximum amount of CPU cores that can be requested by Pods. - Equivalent to the `limit` for Pod resource configuration. - Cores are specified either as a decimal point number or as milli units. - For example:`1.5` will be 1.5 cores, also written as `1500m`. - nullable: true - x-kubernetes-int-or-string: true - min: + The listener volume scope allows Node and Service scopes to be inferred from the applicable listeners. + This must correspond to Volume names in the Pod that mount Listeners. + items: + type: string + type: array + node: + default: false description: |- - The minimal amount of CPU cores that Pods need to run. - Equivalent to the `request` for Pod resource configuration. - Cores are specified either as a decimal point number or as milli units. - For example:`1.5` will be 1.5 cores, also written as `1500m`. - nullable: true - x-kubernetes-int-or-string: true + The node scope is resolved to the name of the Kubernetes Node object that the Pod is running on. + This will typically be the DNS name of the node. + type: boolean + pod: + default: false + description: |- + The pod scope is resolved to the name of the Kubernetes Pod. + This allows the secret to differentiate between StatefulSet replicas. + type: boolean + services: + default: [] + description: |- + The service scope allows Pod objects to specify custom scopes. + This should typically correspond to Service objects that the Pod participates in. + items: + type: string + type: array type: object - memory: + secretClass: + description: '[SecretClass](https://docs.stackable.tech/home/nightly/secret-operator/secretclass) containing the LDAP bind credentials.' + type: string + required: + - secretClass + type: object + host: + description: 'Host of the S3 server without any protocol or port. For example: `west1.my-cloud.com`.' + type: string + port: + description: |- + Port the S3 server listens on. + If not specified the product will determine the port to use. + format: uint16 + maximum: 65535.0 + minimum: 0.0 + nullable: true + type: integer + region: + default: + name: us-east-1 + description: |- + Bucket region used for signing headers (sigv4). + + This defaults to `us-east-1` which is compatible with other implementations such as Minio. + + WARNING: Some products use the Hadoop S3 implementation which falls back to us-east-2. + properties: + name: + default: us-east-1 + type: string + type: object + tls: + description: Use a TLS connection. If not specified no TLS will be used. + nullable: true + properties: + verification: + description: The verification method used to verify the certificates of the server and/or the client. + oneOf: + - required: + - none + - required: + - server properties: - limit: - description: |- - The maximum amount of memory that should be available to the Pod. - Specified as a byte [Quantity](https://kubernetes.io/docs/reference/kubernetes-api/common-definitions/quantity/), - which means these suffixes are supported: E, P, T, G, M, k. - You can also use the power-of-two equivalents: Ei, Pi, Ti, Gi, Mi, Ki. - For example, the following represent roughly the same value: - `128974848, 129e6, 129M, 128974848000m, 123Mi` - nullable: true - x-kubernetes-int-or-string: true - runtimeLimits: - description: Additional options that can be specified. + none: + description: Use TLS but don't verify certificates. + type: object + server: + description: Use TLS and a CA certificate to verify the server. + properties: + caCert: + description: CA cert to verify the server. + oneOf: + - required: + - webPki + - required: + - secretClass + properties: + secretClass: + description: |- + Name of the [SecretClass](https://docs.stackable.tech/home/nightly/secret-operator/secretclass) which will provide the CA certificate. + Note that a SecretClass does not need to have a key but can also work with just a CA certificate, + so if you got provided with a CA cert but don't have access to the key you can still use this method. + type: string + webPki: + description: |- + Use TLS and the CA certificates trusted by the common web browsers to verify the server. + This can be useful when you e.g. use public AWS S3 or other public available services. + type: object + type: object + required: + - caCert type: object type: object - storage: - type: object + required: + - verification type: object + required: + - host type: object - configOverrides: - description: |- - The `configOverrides` can be used to configure properties in product config files - that are not exposed in the CRD. Read the - [config overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#config-overrides) - and consult the operator specific usage guide documentation for details on the - available config files and settings for the specific product. - properties: - metrics.properties: - additionalProperties: - type: string - description: |- - Flat key-value overrides for `*.properties`, Hadoop XML, etc. + reference: + type: string + type: object + sparkConf: + additionalProperties: + type: string + default: {} + description: A map of key/value strings that will be passed directly to spark-submit. + type: object + sparkImage: + anyOf: + - required: + - custom + - productVersion + - required: + - productVersion + description: |- + Specify which image to use, the easiest way is to only configure the `productVersion`. + You can also configure a custom image registry to pull from, as well as completely custom + images. - This is backwards-compatible with the existing flat key-value YAML format - used by `HashMap`. - nullable: true - type: object - security.properties: - additionalProperties: - type: string - description: |- - Flat key-value overrides for `*.properties`, Hadoop XML, etc. + Consult the [Product image selection documentation](https://docs.stackable.tech/home/nightly/concepts/product_image_selection) + for details. + properties: + custom: + description: |- + Provide a custom container image. - This is backwards-compatible with the existing flat key-value YAML format - used by `HashMap`. - nullable: true - type: object - spark-defaults.conf: - additionalProperties: + Specify the full container image name, e.g. `oci.example.tech/namespace/superset:1.4.1-my-tag` + type: string + productVersion: + description: Version of the product, e.g. `1.4.1`. + type: string + pullPolicy: + default: Always + description: '[Pull policy](https://kubernetes.io/docs/concepts/containers/images/#image-pull-policy) used when pulling the image.' + enum: + - IfNotPresent + - Always + - Never + type: string + pullSecrets: + description: '[Image pull secrets](https://kubernetes.io/docs/concepts/containers/images/#specifying-imagepullsecrets-on-a-pod) to pull images from a private registry.' + items: + description: LocalObjectReference contains enough information to let you locate the referenced object inside the same namespace. + properties: + name: + description: 'Name of the referent. This field is effectively required, but due to backwards compatibility is allowed to be empty. Instances of this type with an empty value here are almost certainly wrong. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' type: string - description: |- - Flat key-value overrides for `*.properties`, Hadoop XML, etc. - - This is backwards-compatible with the existing flat key-value YAML format - used by `HashMap`. - nullable: true - type: object - type: object - envOverrides: - additionalProperties: - type: string - default: {} - description: |- - `envOverrides` configure environment variables to be set in the Pods. - It is a map from strings to strings - environment variables and the value to set. - Read the - [environment variable overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#env-overrides) - for more information and consult the operator specific usage guide to find out about - the product specific environment variables that are available. - type: object - jvmArgumentOverrides: - default: - add: [] - remove: [] - removeRegex: [] + required: + - name + type: object + nullable: true + type: array + repo: description: |- - Allows overriding JVM arguments. - Please read on the [JVM argument overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#jvm-argument-overrides) - for details on the usage. - properties: - add: - default: [] - description: JVM arguments to be added - items: - type: string - type: array - remove: - default: [] - description: JVM arguments to be removed by exact match - items: - type: string - type: array - removeRegex: - default: [] - description: JVM arguments matching any of this regexes will be removed - items: - type: string - type: array - type: object - podOverrides: - default: {} + The repository on the container image registry where the container image is located, e.g. + `oci.example.com/namespace`. + + If not specified, the operator will use the image registry provided via the operator + environment options. + nullable: true + type: string + stackableVersion: description: |- - In the `podOverrides` property you can define a - [PodTemplateSpec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.34/#podtemplatespec-v1-core) - to override any property that can be set on a Kubernetes Pod. - Read the - [Pod overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#pod-overrides) - for more information. - type: object - x-kubernetes-preserve-unknown-fields: true - roleConfig: - default: - listenerClass: cluster-internal - description: Global role config settings for the Spark Connect Server. - properties: - listenerClass: - default: cluster-internal - description: |- - This field controls which [ListenerClass](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass.html) - is used to expose the Spark Connect services. - type: string - type: object + Stackable version of the product, e.g. `23.4`, `23.4.1` or `0.0.0-dev`. + + If not specified, the operator will use its own version, e.g. `23.4.1`. When using a nightly + operator or a PR version, it will use the nightly `0.0.0-dev` image. + nullable: true + type: string type: object vectorAggregatorConfigMapName: description: |- - Name of the Vector aggregator discovery ConfigMap. + Name of the Vector aggregator [discovery ConfigMap](https://docs.stackable.tech/home/nightly/concepts/service_discovery). It must contain the key `ADDRESS` with the address of the Vector aggregator. + Follow the [logging tutorial](https://docs.stackable.tech/home/nightly/tutorials/logging-vector-aggregator) + to learn how to configure log aggregation with Vector. nullable: true type: string - required: - - image - type: object - status: - nullable: true - properties: - conditions: + volumes: default: [] + description: A list of volumes that can be made available to the job, driver or executors via their volume mounts. items: - properties: - lastTransitionTime: - description: Last time the condition transitioned from one status to another. - format: date-time - nullable: true - type: string - message: - description: A human readable message indicating details about the transition. - nullable: true - type: string - reason: - description: The reason for the condition's last transition. - nullable: true - type: string - status: - description: Status of the condition, one of True, False, Unknown. - enum: - - 'True' - - 'False' - - Unknown - type: string - type: - description: Type of deployment condition. - enum: - - Available - - Degraded - - Progressing - - ReconciliationPaused - - Stopped - type: string - required: - - status - - type type: object + x-kubernetes-preserve-unknown-fields: true type: array + required: + - mainApplicationFile + - mode + - sparkImage type: object required: - spec - title: SparkConnectServer + title: SparkApplicationTemplate type: object served: true storage: true - subresources: - status: {} ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: sparkapptemplates.spark.stackable.tech -spec: - group: spark.stackable.tech - names: - categories: [] - kind: SparkApplicationTemplate - plural: sparkapptemplates - shortNames: - - sparkapptemplate - singular: sparkapplicationtemplate - scope: Namespaced - versions: + subresources: {} - additionalPrinterColumns: [] name: v1alpha1 schema: @@ -6198,10 +10692,12 @@ spec: type: string job: description: |- - The job builds a spark-submit command, complete with arguments and referenced dependencies - such as templates, and passes it on to Spark. - The reason this property uses its own type (SubmitConfigFragment) is because logging is not - supported for spark-submit processes. + Deprecated and ignored. + + In previous versions this configured the dedicated `spark-submit` Job. Since v1alpha2 the + operator launches the driver directly as a Kubernetes Job (built from `spec.driver`), so + there is no separate submit process to configure. This field is kept for backwards + compatibility but has no effect. nullable: true properties: cliOverrides: @@ -6600,7 +11096,13 @@ spec: nullable: true type: string mode: - description: 'Mode: cluster or client. Currently only cluster is supported.' + description: |- + Deprecated and ignored. + + Since v1alpha2 the operator no longer runs a separate `spark-submit` process. The driver + is launched directly as a Kubernetes Job running in client mode, so the deploy mode is + always client internally. This field is kept for backwards compatibility but has no + effect. enum: - cluster - client @@ -6850,5 +11352,5 @@ spec: title: SparkApplicationTemplate type: object served: true - storage: true + storage: false subresources: {} diff --git a/rust/operator-binary/src/config/jvm.rs b/rust/operator-binary/src/config/jvm.rs index 52968674..454cd29f 100644 --- a/rust/operator-binary/src/config/jvm.rs +++ b/rust/operator-binary/src/config/jvm.rs @@ -11,7 +11,7 @@ use crate::crd::{ }, logdir::ResolvedLogDir, tlscerts::tls_secret_names, - v1alpha1::SparkApplication, + v1alpha2::SparkApplication, }; #[derive(Snafu, Debug)] diff --git a/rust/operator-binary/src/crd/constants.rs b/rust/operator-binary/src/crd/constants.rs index 0b2aab32..3aa32218 100644 --- a/rust/operator-binary/src/crd/constants.rs +++ b/rust/operator-binary/src/crd/constants.rs @@ -6,9 +6,6 @@ pub const APP_NAME: &str = "spark-k8s"; pub const VOLUME_MOUNT_NAME_IVY2: &str = "ivy2"; pub const VOLUME_MOUNT_PATH_IVY2: &str = "/ivy2"; -pub const VOLUME_MOUNT_NAME_DRIVER_POD_TEMPLATES: &str = "driver-pod-template"; -pub const VOLUME_MOUNT_PATH_DRIVER_POD_TEMPLATES: &str = "/stackable/spark/driver-pod-templates"; - pub const VOLUME_MOUNT_NAME_EXECUTOR_POD_TEMPLATES: &str = "executor-pod-template"; pub const VOLUME_MOUNT_PATH_EXECUTOR_POD_TEMPLATES: &str = "/stackable/spark/executor-pod-templates"; @@ -88,7 +85,15 @@ pub const SPARK_CLUSTER_ROLE: &str = "spark-k8s-clusterrole"; pub const METRICS_PORT: u16 = 18081; pub const HISTORY_UI_PORT: u16 = 18080; +/// Label value identifying the Spark driver pod. Used both as the headless driver Service selector +/// and to let the driver/executor pods discover each other in client mode. +pub const SPARK_ROLE_LABEL: &str = "spark-role"; +pub const SPARK_ROLE_DRIVER: &str = "driver"; + +/// Fixed ports for the driver running in client mode, so the headless driver Service can expose a +/// stable address that executors connect back to. +pub const DRIVER_PORT: u16 = 7078; +pub const DRIVER_BLOCK_MANAGER_PORT: u16 = 7079; + pub const LISTENER_VOLUME_NAME: &str = "listener"; pub const LISTENER_VOLUME_DIR: &str = "/stackable/listener"; - -pub const DEFAULT_SUBMIT_JOB_RETRY_ON_FAILURE_COUNT: u16 = 0; diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index 9560b32c..23c6eb60 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -52,7 +52,7 @@ use crate::{ job_dependencies::JobDependencies, roles::{ RoleConfig, RoleConfigFragment, SparkApplicationRole, SparkContainer, SparkMode, - SubmitConfig, SubmitConfigFragment, VolumeMounts, + SubmitConfigFragment, }, }, }; @@ -133,16 +133,64 @@ pub enum Error { } pub type SparkApplicationJobRoleType = - CommonConfiguration; + CommonConfiguration; pub type SparkApplicationDriverRoleType = - CommonConfiguration; + CommonConfiguration; pub type SparkApplicationExecutorRoleType = - RoleGroup; + RoleGroup; + +/// A reference to a [`SparkApplicationTemplate`](template_spec::SparkApplicationTemplate) that has +/// been merged into a SparkApplication. Stored in the application status. +/// +/// This type is intentionally defined outside the `versioned` module because it does not change +/// between CRD versions and is nested inside a `Vec` in the status, which the auto-generated +/// version conversions cannot handle for versioned types. +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, JsonSchema)] +#[serde(rename_all = "camelCase")] +pub struct ResolvedSparkApplicationTemplate { + pub name: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub uid: Option, +} + +/// Status of a SparkApplication. +/// +/// Defined outside the `versioned` module because it does not change between CRD versions and is +/// shared by all versions as the status subresource. +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, JsonSchema)] +#[allow(clippy::derive_partial_eq_without_eq)] +#[serde(rename_all = "camelCase")] +pub struct SparkApplicationStatus { + pub phase: String, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub resolved_template_ref: Vec, +} + +/// Config file overrides accepted by a SparkApplication. +/// +/// Defined outside the `versioned` module because it does not change between CRD versions. +#[derive(Clone, Debug, Default, Deserialize, JsonSchema, PartialEq, Serialize)] +pub struct ConfigOverrides { + #[serde( + default, + rename = "spark-env.sh", + skip_serializing_if = "Option::is_none" + )] + pub spark_env_sh: Option, + + #[serde( + default, + rename = "security.properties", + skip_serializing_if = "Option::is_none" + )] + pub security_properties: Option, +} #[versioned( version(name = "v1alpha1"), + version(name = "v1alpha2"), crates( kube_core = "stackable_operator::kube::core", kube_client = "stackable_operator::kube::client", @@ -153,22 +201,6 @@ pub type SparkApplicationExecutorRoleType = )] pub mod versioned { - #[derive(Clone, Debug, Deserialize, PartialEq, Serialize, JsonSchema)] - #[serde(rename_all = "camelCase")] - pub(crate) struct ResolvedSparkApplicationTemplate { - pub name: String, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub uid: Option, - } - #[derive(Clone, Debug, Deserialize, PartialEq, Serialize, JsonSchema)] - #[allow(clippy::derive_partial_eq_without_eq)] - #[serde(rename_all = "camelCase")] - pub struct SparkApplicationStatus { - pub phase: String, - #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub resolved_template_ref: Vec, - } - /// A Spark application run on Kubernetes by the Stackable operator for Apache Spark. /// Find more information on how to use it and the resources that the operator generates in the /// [operator documentation](DOCS_BASE_URL_PLACEHOLDER/spark-k8s/). @@ -184,7 +216,12 @@ pub mod versioned { #[derive(Clone, CustomResource, Debug, Deserialize, JsonSchema, Serialize)] #[serde(rename_all = "camelCase")] pub struct SparkApplicationSpec { - /// Mode: cluster or client. Currently only cluster is supported. + /// Deprecated and ignored. + /// + /// Since v1alpha2 the operator no longer runs a separate `spark-submit` process. The driver + /// is launched directly as a Kubernetes Job running in client mode, so the deploy mode is + /// always client internally. This field is kept for backwards compatibility but has no + /// effect. pub mode: SparkMode, /// The main class - i.e. entry point - for JVM artifacts. @@ -209,15 +246,18 @@ pub mod versioned { #[serde(skip_serializing_if = "Option::is_none")] pub vector_aggregator_config_map_name: Option, - /// The job builds a spark-submit command, complete with arguments and referenced dependencies - /// such as templates, and passes it on to Spark. - /// The reason this property uses its own type (SubmitConfigFragment) is because logging is not - /// supported for spark-submit processes. - // - // IMPORTANT: Please note that the jvmArgumentOverrides have no effect here! - // However, due to product-config things I wasn't able to remove them. + /// Deprecated and ignored. + /// + /// In previous versions this configured the dedicated `spark-submit` Job. Since v1alpha2 the + /// operator launches the driver directly as a Kubernetes Job (built from `spec.driver`), so + /// there is no separate submit process to configure. This field is kept for backwards + /// compatibility but has no effect. + #[versioned(deprecated( + since = "v1alpha2", + note = "the operator no longer runs a separate spark-submit process; the driver Job is built from spec.driver" + ))] #[serde(default, skip_serializing_if = "Option::is_none")] - pub job: Option, + pub deprecated_job: Option, /// The driver role specifies the configuration that, together with the driver pod template, is used by /// Spark to create driver pods. @@ -264,26 +304,9 @@ pub mod versioned { #[serde(default, skip_serializing_if = "Option::is_none")] pub log_file_directory: Option, } - - #[derive(Clone, Debug, Default, Deserialize, JsonSchema, PartialEq, Serialize)] - pub struct ConfigOverrides { - #[serde( - default, - rename = "spark-env.sh", - skip_serializing_if = "Option::is_none" - )] - pub spark_env_sh: Option, - - #[serde( - default, - rename = "security.properties", - skip_serializing_if = "Option::is_none" - )] - pub security_properties: Option, - } } -impl KeyValueOverridesProvider for v1alpha1::ConfigOverrides { +impl KeyValueOverridesProvider for ConfigOverrides { fn get_key_value_overrides(&self, file: &str) -> BTreeMap> { let field = match file { SPARK_ENV_SH_FILE_NAME => self.spark_env_sh.as_ref(), @@ -296,7 +319,7 @@ impl KeyValueOverridesProvider for v1alpha1::ConfigOverrides { } } -impl Merge for v1alpha1::ConfigOverrides { +impl Merge for ConfigOverrides { fn merge(&mut self, defaults: &Self) { merge_key_value_config_overrides(&mut self.spark_env_sh, &defaults.spark_env_sh); merge_key_value_config_overrides( @@ -316,8 +339,8 @@ fn merge_key_value_config_overrides( } } -impl v1alpha1::SparkApplication { - /// Returns if this [`SparkApplication`] has already created a Kubernetes Job doing the actual `spark-submit`. +impl v1alpha2::SparkApplication { + /// Returns if this [`SparkApplication`] has already created the driver Kubernetes Job. /// /// This is needed because Kubernetes will remove the succeeded Job after some time. When the spark-k8s-operator is /// restarted it would re-create the Job, resulting in the Spark job running multiple times. This function assumes @@ -332,14 +355,25 @@ impl v1alpha1::SparkApplication { .unwrap_or_default() } - pub fn submit_job_config_map_name(&self) -> String { - format!("{app_name}-submit-job", app_name = self.name_any()) - } - pub fn pod_template_config_map_name(&self, role: SparkApplicationRole) -> String { format!("{app_name}-{role}-pod-template", app_name = self.name_any()) } + /// Name of the headless Service that exposes the driver pod so that executors can connect back + /// to it. Required because the driver now runs in client mode inside the operator-created Job. + pub fn driver_service_name(&self) -> String { + format!("{app_name}-driver", app_name = self.name_any()) + } + + /// The in-cluster DNS name executors use to reach the driver (`spark.driver.host`). + pub fn driver_host(&self) -> Result { + let namespace = self.metadata.namespace.as_ref().context(NoNamespaceSnafu)?; + Ok(format!( + "{service}.{namespace}.svc.cluster.local", + service = self.driver_service_name() + )) + } + pub fn application_artifact(&self) -> &str { self.spec.main_application_file.as_ref() } @@ -480,55 +514,6 @@ impl v1alpha1::SparkApplication { Ok(result.into_values().collect()) } - /// Return the volume mounts for the spark-submit pod. - /// - /// These volume mounts are assembled from: - /// * two pod template CMs for the driver and executors - /// * volume mounts for accessing applications stored in S3 buckets - /// * S3 credentials - /// * S3 verification certificates - /// * python packages (razvan: this was also a mistake since these packages are not used here.) - /// * volume mounts additional java packages - /// * finally user specified volume maps in `spec.job`. - /// - pub fn spark_job_volume_mounts( - &self, - s3conn: &Option, - logdir: &Option, - ) -> Vec { - let mut tmpl_mounts = vec![ - VolumeMount { - name: VOLUME_MOUNT_NAME_DRIVER_POD_TEMPLATES.into(), - mount_path: VOLUME_MOUNT_PATH_DRIVER_POD_TEMPLATES.into(), - ..VolumeMount::default() - }, - VolumeMount { - name: VOLUME_MOUNT_NAME_EXECUTOR_POD_TEMPLATES.into(), - mount_path: VOLUME_MOUNT_PATH_EXECUTOR_POD_TEMPLATES.into(), - ..VolumeMount::default() - }, - ]; - - tmpl_mounts = self.add_common_volume_mounts(tmpl_mounts, s3conn, logdir, false); - - if let Some(CommonConfiguration { - config: - SubmitConfigFragment { - volume_mounts: - Some(VolumeMounts { - volume_mounts: job_vm, - }), - .. - }, - .. - }) = &self.spec.job - { - tmpl_mounts.extend(job_vm.clone()); - } - - tmpl_mounts - } - fn add_common_volume_mounts( &self, mut mounts: Vec, @@ -616,7 +601,7 @@ impl v1alpha1::SparkApplication { &'a self, app_version: &'a str, role: &'a str, - ) -> ObjectLabels<'a, v1alpha1::SparkApplication> { + ) -> ObjectLabels<'a, v1alpha2::SparkApplication> { ObjectLabels { owner: self, app_name: APP_NAME, @@ -635,8 +620,8 @@ impl v1alpha1::SparkApplication { spark_image: &str, ) -> Result, Error> { // mandatory properties - let mode = &self.spec.mode; let name = self.metadata.name.clone().context(ObjectHasNoNameSnafu)?; + let driver_host = self.driver_host()?; // Commands needed to build the p12 trust store from the secret class certs configured for // S3 connections. @@ -660,18 +645,20 @@ impl v1alpha1::SparkApplication { "--verbose".to_string(), "--master k8s://https://${KUBERNETES_SERVICE_HOST}:${KUBERNETES_SERVICE_PORT_HTTPS}" .to_string(), - format!("--deploy-mode {mode}"), + // The operator launches the driver directly as a Kubernetes Job. Running spark-submit in + // client mode makes this very process the driver (no separate driver pod is created by + // Spark). The driver then requests executor pods from the Kubernetes backend. + "--deploy-mode client".to_string(), format!("--name {name}"), - format!( - "--conf spark.kubernetes.driver.podTemplateFile={VOLUME_MOUNT_PATH_DRIVER_POD_TEMPLATES}/{POD_TEMPLATE_FILE}" - ), + // Executors connect back to the driver via the headless driver Service. The driver binds + // to all interfaces inside its pod and advertises the Service address on fixed ports. + format!("--conf spark.driver.host={driver_host}"), + "--conf spark.driver.bindAddress=0.0.0.0".to_string(), + format!("--conf spark.driver.port={DRIVER_PORT}"), + format!("--conf spark.driver.blockManager.port={DRIVER_BLOCK_MANAGER_PORT}"), format!( "--conf spark.kubernetes.executor.podTemplateFile={VOLUME_MOUNT_PATH_EXECUTOR_POD_TEMPLATES}/{POD_TEMPLATE_FILE}" ), - format!( - "--conf spark.kubernetes.driver.podTemplateContainerName={container_name}", - container_name = SparkContainer::Spark - ), format!( "--conf spark.kubernetes.executor.podTemplateContainerName={container_name}", container_name = SparkContainer::Spark @@ -680,10 +667,6 @@ impl v1alpha1::SparkApplication { "--conf spark.kubernetes.namespace={}", self.metadata.namespace.as_ref().context(NoNamespaceSnafu)? ), - format!( - "--conf spark.kubernetes.driver.container.image={}", - spark_image.to_string() - ), format!( "--conf spark.kubernetes.executor.container.image={}", spark_image.to_string() @@ -875,16 +858,6 @@ impl v1alpha1::SparkApplication { e } - pub fn submit_config(&self) -> Result { - if let Some(CommonConfiguration { mut config, .. }) = self.spec.job.clone() { - config.merge(&SubmitConfig::default_config()); - fragment::validate(config).context(FragmentValidationFailureSnafu) - } else { - fragment::validate(SubmitConfig::default_config()) - .context(FragmentValidationFailureSnafu) - } - } - pub fn driver_config(&self) -> Result { if let Some(CommonConfiguration { mut config, .. }) = self.spec.driver.clone() { config.merge(&RoleConfig::default_config()); @@ -909,7 +882,6 @@ impl v1alpha1::SparkApplication { pub fn pod_overrides(&self, role: SparkApplicationRole) -> Option { match role { - SparkApplicationRole::Submit => self.spec.job.clone().map(|j| j.pod_overrides), SparkApplicationRole::Driver => self.spec.driver.clone().map(|d| d.pod_overrides), SparkApplicationRole::Executor => { self.spec.executor.clone().map(|r| r.config.pod_overrides) @@ -926,7 +898,6 @@ impl v1alpha1::SparkApplication { // Merge the role-specific envOverrides on top let role_envs = match role { - SparkApplicationRole::Submit => self.spec.job.as_ref().map(|j| &j.env_overrides), SparkApplicationRole::Driver => self.spec.driver.as_ref().map(|d| &d.env_overrides), SparkApplicationRole::Executor => { self.spec.executor.as_ref().map(|e| &e.config.env_overrides) @@ -953,14 +924,6 @@ impl v1alpha1::SparkApplication { resolved_product_image: &ResolvedProductImage, product_config: &ProductConfigManager, ) -> Result { - let submit_conf = match self.spec.job.as_ref() { - Some(job) => job.clone(), - None => CommonConfiguration { - config: SubmitConfig::default_config(), - ..CommonConfiguration::default() - }, - }; - let driver_conf = match self.spec.driver.as_ref() { Some(driver) => driver.clone(), None => CommonConfiguration { @@ -981,29 +944,6 @@ impl v1alpha1::SparkApplication { }; let mut roles_to_validate = HashMap::new(); - roles_to_validate.insert( - SparkApplicationRole::Submit.to_string(), - ( - vec![ - PropertyNameKind::Env, - PropertyNameKind::File(SPARK_ENV_SH_FILE_NAME.to_string()), - PropertyNameKind::File(JVM_SECURITY_PROPERTIES_FILE.to_string()), - ], - Role { - config: submit_conf.clone(), - role_config: GenericRoleConfig::default(), - role_groups: [( - "default".to_string(), - RoleGroup { - config: submit_conf, - replicas: Some(1), - }, - )] - .into(), - } - .erase(), - ), - ); roles_to_validate.insert( SparkApplicationRole::Driver.to_string(), ( @@ -1055,17 +995,6 @@ impl v1alpha1::SparkApplication { ) .context(InvalidProductConfigSnafu) } - - pub fn retry_on_failure_count(&self) -> i32 { - let effective_retry_on_failure_count = self - .spec - .job - .as_ref() - .map(|common_config| common_config.config.clone()) - .and_then(|config| config.retry_on_failure_count) - .unwrap_or(DEFAULT_SUBMIT_JOB_RETRY_ON_FAILURE_COUNT); - effective_retry_on_failure_count as i32 - } } /// CPU Limits can be defined as integer, decimal, or unitised values (see @@ -1259,7 +1188,7 @@ mod tests { #[test] fn test_default_resource_limits() { - let spark_application = serde_yaml::from_str::(indoc! {" + let spark_application = serde_yaml::from_str::(indoc! {" --- apiVersion: spark.stackable.tech/v1alpha1 kind: SparkApplication @@ -1273,10 +1202,6 @@ mod tests { "}) .unwrap(); - let job_resources = &spark_application.submit_config().unwrap().resources; - assert_eq!("100m", job_resources.cpu.min.as_ref().unwrap().0); - assert_eq!("400m", job_resources.cpu.max.as_ref().unwrap().0); - let driver_resources = &spark_application.driver_config().unwrap().resources; assert_eq!("250m", driver_resources.cpu.min.as_ref().unwrap().0); assert_eq!("1", driver_resources.cpu.max.as_ref().unwrap().0); @@ -1288,7 +1213,7 @@ mod tests { #[test] fn test_merged_resource_limits() { - let spark_application = serde_yaml::from_str::(indoc! {r#" + let spark_application = serde_yaml::from_str::(indoc! {r#" --- apiVersion: spark.stackable.tech/v1alpha1 kind: SparkApplication @@ -1299,7 +1224,7 @@ mod tests { mainApplicationFile: test.py sparkImage: productVersion: 1.2.3 - job: + deprecatedJob: config: resources: cpu: @@ -1327,17 +1252,6 @@ mod tests { "# }) .unwrap(); - assert_eq!( - "200m", - &spark_application - .submit_config() - .unwrap() - .resources - .cpu - .max - .unwrap() - .0 - ); assert_eq!( "1300m", &spark_application @@ -1364,7 +1278,7 @@ mod tests { #[test] fn test_role_affinities() { - let spark_application = serde_yaml::from_str::(indoc! {r#" + let spark_application = serde_yaml::from_str::(indoc! {r#" apiVersion: spark.stackable.tech/v1alpha1 kind: SparkApplication metadata: @@ -1374,7 +1288,7 @@ spec: mainApplicationFile: test.py sparkImage: productVersion: 1.2.3 - job: + deprecatedJob: config: affinity: nodeSelector: @@ -1469,49 +1383,6 @@ spec: "#}) .unwrap(); - let job_affinity = spark_application.submit_config().unwrap().affinity; - assert_eq!( - Some("job"), - job_affinity - .node_selector - .as_ref() - .and_then(|selectors| selectors.node_selector.get("affinity-role")) - .map(String::as_str) - ); - assert_eq!( - Some(11), - job_affinity - .node_affinity - .as_ref() - .and_then(|a| a - .preferred_during_scheduling_ignored_during_execution - .as_ref()) - .and_then(|terms| terms.first()) - .map(|term| term.weight) - ); - assert_eq!( - Some(21), - job_affinity - .pod_affinity - .as_ref() - .and_then(|a| a - .preferred_during_scheduling_ignored_during_execution - .as_ref()) - .and_then(|terms| terms.first()) - .map(|term| term.weight) - ); - assert_eq!( - Some(31), - job_affinity - .pod_anti_affinity - .as_ref() - .and_then(|a| a - .preferred_during_scheduling_ignored_during_execution - .as_ref()) - .and_then(|terms| terms.first()) - .map(|term| term.weight) - ); - let driver_affinity = spark_application.driver_config().unwrap().affinity; assert_eq!( Some("driver"), @@ -1702,7 +1573,7 @@ spec: #[test] fn test_validated_config() { - let spark_application = serde_yaml::from_str::(indoc! {r#" + let spark_application = serde_yaml::from_str::(indoc! {r#" --- apiVersion: spark.stackable.tech/v1alpha1 kind: SparkApplication @@ -1756,7 +1627,6 @@ spec: .into_iter() .collect(); let expected: ValidatedRoleConfigByPropertyKind = vec![ - ("submit".into(), expected_role_groups.clone()), ("driver".into(), expected_role_groups.clone()), ("executor".into(), expected_role_groups), ] @@ -1766,64 +1636,7 @@ spec: assert_eq!(expected, validated_config); } - #[test] - fn test_job_volume_mounts() { - let spark_application = serde_yaml::from_str::(indoc! {r#" - --- - apiVersion: spark.stackable.tech/v1alpha1 - kind: SparkApplication - metadata: - name: spark-examples - spec: - mode: cluster - mainApplicationFile: test.py - sparkImage: - productVersion: 1.2.3 - job: - config: - volumeMounts: - - name: keytab - mountPath: /kerberos - volumes: - - name: keytab - configMap: - name: keytab - "#}) - .unwrap(); - - let got = spark_application.spark_job_volume_mounts(&None, &None); - - let expected = vec![ - VolumeMount { - mount_path: "/stackable/spark/driver-pod-templates".into(), - mount_propagation: None, - name: "driver-pod-template".into(), - ..VolumeMount::default() - }, - VolumeMount { - mount_path: "/stackable/spark/executor-pod-templates".into(), - mount_propagation: None, - name: "executor-pod-template".into(), - ..VolumeMount::default() - }, - VolumeMount { - mount_path: "/stackable/log".into(), - mount_propagation: None, - name: "log".into(), - ..VolumeMount::default() - }, - VolumeMount { - mount_path: "/kerberos".into(), - mount_propagation: None, - name: "keytab".into(), - ..VolumeMount::default() - }, - ]; - - assert_eq!(got, expected); - } - - impl RoundtripTestData for v1alpha1::SparkApplicationSpec { + impl RoundtripTestData for v1alpha2::SparkApplicationSpec { fn roundtrip_test_data() -> Vec { stackable_operator::utils::yaml_from_str_singleton_map(indoc! {r#" - sparkImage: @@ -1852,7 +1665,7 @@ spec: value: ORIGINAL - name: TEST_SPARK_VAR_1 value: DONOTREPLACE - job: + deprecatedJob: envOverrides: TEST_SPARK_VAR_0: REPLACED configOverrides: @@ -1972,4 +1785,15 @@ spec: .expect("Failed to parse SparkApplicationSpec YAML") } } + + // v1alpha1 round-trip data is derived from the v1alpha2 data via the auto-generated downgrade + // conversion, so we don't duplicate the (large) YAML fixture. + impl RoundtripTestData for v1alpha1::SparkApplicationSpec { + fn roundtrip_test_data() -> Vec { + v1alpha2::SparkApplicationSpec::roundtrip_test_data() + .into_iter() + .map(Into::into) + .collect() + } + } } diff --git a/rust/operator-binary/src/crd/roles.rs b/rust/operator-binary/src/crd/roles.rs index 52e95af8..8f15ada9 100644 --- a/rust/operator-binary/src/crd/roles.rs +++ b/rust/operator-binary/src/crd/roles.rs @@ -7,7 +7,7 @@ //! this responsibility to the Submit job. //! //! The submit job only supports one group per role. For this reason, the -//! [`v1alpha1::SparkApplication`] spec doesn't declare Role objects directly. Instead it +//! [`v1alpha2::SparkApplication`] spec doesn't declare Role objects directly. Instead it //! only declares [`stackable_operator::role_utils::CommonConfiguration`] objects for job, //! driver and executor and constructs the Roles dynamically when needed. The only group under //! each role is named "default". These roles are transparent to the user. @@ -38,12 +38,11 @@ use stackable_operator::{ }; use strum::{Display, EnumIter}; -use crate::crd::{ResolvedLogDir, constants::DEFAULT_SUBMIT_JOB_RETRY_ON_FAILURE_COUNT, v1alpha1}; +use crate::crd::{ResolvedLogDir, v1alpha2}; #[derive(Clone, Debug, Deserialize, Display, Eq, PartialEq, Serialize, JsonSchema)] #[strum(serialize_all = "kebab-case")] pub enum SparkApplicationRole { - Submit, Driver, Executor, } @@ -156,7 +155,7 @@ impl RoleConfig { pub fn volume_mounts( &self, - spark_application: &v1alpha1::SparkApplication, + spark_application: &v1alpha2::SparkApplication, s3conn: &Option, logdir: &Option, ) -> Vec { @@ -166,7 +165,7 @@ impl RoleConfig { } impl Configuration for RoleConfigFragment { - type Configurable = v1alpha1::SparkApplication; + type Configurable = v1alpha2::SparkApplication; fn compute_env( &self, @@ -211,6 +210,9 @@ impl Configuration for RoleConfigFragment { ), serde(rename_all = "camelCase") )] +// Deprecated since v1alpha2: this type only exists so the deprecated `spec.job` field keeps its +// schema. It is no longer constructed or used at runtime. +#[allow(dead_code)] pub struct SubmitConfig { #[fragment_attrs(serde(default))] pub resources: Resources, @@ -231,33 +233,8 @@ pub struct SubmitConfig { pub retry_on_failure_count: u16, } -impl SubmitConfig { - // Auto TLS certificate lifetime - const DEFAULT_SECRET_LIFETIME: Duration = Duration::from_days_unchecked(1); - - pub fn default_config() -> SubmitConfigFragment { - SubmitConfigFragment { - resources: ResourcesFragment { - cpu: CpuLimitsFragment { - min: Some(Quantity("100m".to_owned())), - max: Some(Quantity("400m".to_owned())), - }, - memory: MemoryLimitsFragment { - limit: Some(Quantity("512Mi".to_owned())), - runtime_limits: NoRuntimeLimitsFragment {}, - }, - storage: SparkStorageConfigFragment {}, - }, - affinity: Default::default(), - volume_mounts: Some(VolumeMounts::default()), - requested_secret_lifetime: Some(Self::DEFAULT_SECRET_LIFETIME), - retry_on_failure_count: Some(DEFAULT_SUBMIT_JOB_RETRY_ON_FAILURE_COUNT), - } - } -} - impl Configuration for SubmitConfigFragment { - type Configurable = v1alpha1::SparkApplication; + type Configurable = v1alpha2::SparkApplication; fn compute_env( &self, diff --git a/rust/operator-binary/src/crd/template_merger.rs b/rust/operator-binary/src/crd/template_merger.rs index 527526d0..5ccee4f4 100644 --- a/rust/operator-binary/src/crd/template_merger.rs +++ b/rust/operator-binary/src/crd/template_merger.rs @@ -10,7 +10,7 @@ use stackable_operator::{ role_utils::{CommonConfiguration, RoleGroup}, }; -use super::v1alpha1::SparkApplication; +use super::v1alpha2::SparkApplication; /// Deep merge two SparkApplication instances. /// @@ -30,11 +30,12 @@ use super::v1alpha1::SparkApplication; /// # Returns /// /// A new SparkApplication containing the merged result +#[allow(deprecated)] // the deprecated `job` field is still merged for backwards compatibility pub fn deep_merge(base: &SparkApplication, overlay: &SparkApplication) -> SparkApplication { let metadata = merge_metadata(&base.metadata, &overlay.metadata); // Merge spec fields - let spec = super::v1alpha1::SparkApplicationSpec { + let spec = super::v1alpha2::SparkApplicationSpec { // Scalar fields: overlay takes precedence mode: overlay.spec.mode.clone(), main_application_file: overlay.spec.main_application_file.clone(), @@ -69,8 +70,11 @@ pub fn deep_merge(base: &SparkApplication, overlay: &SparkApplication) -> SparkA // Product image: overlay takes precedence spark_image: overlay.spec.spark_image.clone(), - // Merge job configuration - job: merge_common_config(base.spec.job.as_ref(), overlay.spec.job.as_ref()), + // Merge the deprecated job configuration (kept for backwards compatibility). + deprecated_job: merge_common_config( + base.spec.deprecated_job.as_ref(), + overlay.spec.deprecated_job.as_ref(), + ), // Merge driver configuration driver: merge_common_config(base.spec.driver.as_ref(), overlay.spec.driver.as_ref()), @@ -249,6 +253,7 @@ fn merge_deps( } #[cfg(test)] +#[allow(deprecated)] // tests exercise the deprecated `job` field merge for backwards compatibility mod tests { use indoc::indoc; @@ -256,7 +261,7 @@ mod tests { #[test] fn test_deep_merge_basic() { - let base = serde_yaml::from_str::(indoc! {r#" + let base = serde_yaml::from_str::(indoc! {r#" --- apiVersion: spark.stackable.tech/v1alpha1 kind: SparkApplication @@ -274,7 +279,7 @@ mod tests { "#}) .unwrap(); - let overlay = serde_yaml::from_str::(indoc! {r#" + let overlay = serde_yaml::from_str::(indoc! {r#" --- apiVersion: spark.stackable.tech/v1alpha1 kind: SparkApplication @@ -314,7 +319,7 @@ mod tests { #[test] fn test_deep_merge_metadata_namespace_overlay_wins() { - let base = serde_yaml::from_str::(indoc! {r#" + let base = serde_yaml::from_str::(indoc! {r#" --- apiVersion: spark.stackable.tech/v1alpha1 kind: SparkApplication @@ -329,7 +334,7 @@ mod tests { "#}) .unwrap(); - let overlay = serde_yaml::from_str::(indoc! {r#" + let overlay = serde_yaml::from_str::(indoc! {r#" --- apiVersion: spark.stackable.tech/v1alpha1 kind: SparkApplication @@ -355,7 +360,7 @@ mod tests { #[test] fn test_deep_merge_spark_conf() { - let base = serde_yaml::from_str::(indoc! {r#" + let base = serde_yaml::from_str::(indoc! {r#" --- apiVersion: spark.stackable.tech/v1alpha1 kind: SparkApplication @@ -372,7 +377,7 @@ mod tests { "#}) .unwrap(); - let overlay = serde_yaml::from_str::(indoc! {r#" + let overlay = serde_yaml::from_str::(indoc! {r#" --- apiVersion: spark.stackable.tech/v1alpha1 kind: SparkApplication @@ -408,7 +413,7 @@ mod tests { #[test] fn test_deep_merge_job() { - let base = serde_yaml::from_str::(indoc! {r#" + let base = serde_yaml::from_str::(indoc! {r#" --- apiVersion: spark.stackable.tech/v1alpha1 kind: SparkApplication @@ -422,7 +427,7 @@ mod tests { "#}) .unwrap(); - let overlay = serde_yaml::from_str::(indoc! {r#" + let overlay = serde_yaml::from_str::(indoc! {r#" --- apiVersion: spark.stackable.tech/v1alpha1 kind: SparkApplication @@ -433,7 +438,7 @@ mod tests { mainApplicationFile: overlay.py sparkImage: productVersion: "4.1.0" - job: + deprecatedJob: config: retryOnFailureCount: 2 "#}) @@ -445,7 +450,7 @@ mod tests { assert_eq!( merged .spec - .job + .deprecated_job .as_ref() .unwrap() .config @@ -456,7 +461,7 @@ mod tests { #[test] fn test_deep_merge_config_overrides() { - let base = serde_yaml::from_str::(indoc! {r#" + let base = serde_yaml::from_str::(indoc! {r#" --- apiVersion: spark.stackable.tech/v1alpha1 kind: SparkApplication @@ -467,7 +472,7 @@ mod tests { mainApplicationFile: base.py sparkImage: productVersion: "3.5.8" - job: + deprecatedJob: configOverrides: security.properties: test.base.only: base @@ -486,7 +491,7 @@ mod tests { "#}) .unwrap(); - let overlay = serde_yaml::from_str::(indoc! {r#" + let overlay = serde_yaml::from_str::(indoc! {r#" --- apiVersion: spark.stackable.tech/v1alpha1 kind: SparkApplication @@ -497,7 +502,7 @@ mod tests { mainApplicationFile: overlay.py sparkImage: productVersion: "4.1.0" - job: + deprecatedJob: configOverrides: security.properties: test.overridden: overlay @@ -522,7 +527,7 @@ mod tests { // conflicting keys are overridden by overlay values, and new overlay keys are added. let submit_security_props = merged .spec - .job + .deprecated_job .and_then(|config| config.config_overrides.security_properties) .map(|security_properties| security_properties.overrides) .unwrap(); @@ -580,7 +585,7 @@ mod tests { #[test] fn test_deep_merge_config_overrides_base_only() { - let base = serde_yaml::from_str::(indoc! {r#" + let base = serde_yaml::from_str::(indoc! {r#" --- apiVersion: spark.stackable.tech/v1alpha1 kind: SparkApplication @@ -591,7 +596,7 @@ mod tests { mainApplicationFile: base.py sparkImage: productVersion: "3.5.8" - job: + deprecatedJob: configOverrides: security.properties: test.base.only: base @@ -600,7 +605,7 @@ mod tests { "#}) .unwrap(); - let overlay = serde_yaml::from_str::(indoc! {r#" + let overlay = serde_yaml::from_str::(indoc! {r#" --- apiVersion: spark.stackable.tech/v1alpha1 kind: SparkApplication @@ -611,7 +616,7 @@ mod tests { mainApplicationFile: overlay.py sparkImage: productVersion: "4.1.0" - job: + deprecatedJob: config: retryOnFailureCount: 2 "#}) @@ -620,7 +625,7 @@ mod tests { let merged = deep_merge(&base, &overlay); let submit_security_props = merged .spec - .job + .deprecated_job .and_then(|config| config.config_overrides.security_properties) .map(|security_properties| security_properties.overrides) .unwrap(); @@ -633,7 +638,7 @@ mod tests { #[test] fn test_deep_merge_config_overrides_overlay_only() { - let base = serde_yaml::from_str::(indoc! {r#" + let base = serde_yaml::from_str::(indoc! {r#" --- apiVersion: spark.stackable.tech/v1alpha1 kind: SparkApplication @@ -644,13 +649,13 @@ mod tests { mainApplicationFile: base.py sparkImage: productVersion: "3.5.8" - job: + deprecatedJob: config: retryOnFailureCount: 1 "#}) .unwrap(); - let overlay = serde_yaml::from_str::(indoc! {r#" + let overlay = serde_yaml::from_str::(indoc! {r#" --- apiVersion: spark.stackable.tech/v1alpha1 kind: SparkApplication @@ -661,7 +666,7 @@ mod tests { mainApplicationFile: overlay.py sparkImage: productVersion: "4.1.0" - job: + deprecatedJob: configOverrides: security.properties: test.overlay.only: overlay @@ -673,7 +678,7 @@ mod tests { let merged = deep_merge(&base, &overlay); let submit_security_props = merged .spec - .job + .deprecated_job .and_then(|config| config.config_overrides.security_properties) .map(|security_properties| security_properties.overrides) .unwrap(); @@ -686,7 +691,7 @@ mod tests { #[test] fn test_deep_merge_env_overrides() { - let base = serde_yaml::from_str::(indoc! {r#" + let base = serde_yaml::from_str::(indoc! {r#" --- apiVersion: spark.stackable.tech/v1alpha1 kind: SparkApplication @@ -697,7 +702,7 @@ mod tests { mainApplicationFile: base.py sparkImage: productVersion: "3.5.8" - job: + deprecatedJob: envOverrides: TEST_BASE_ONLY: base TEST_OVERRIDDEN: base @@ -713,7 +718,7 @@ mod tests { "#}) .unwrap(); - let overlay = serde_yaml::from_str::(indoc! {r#" + let overlay = serde_yaml::from_str::(indoc! {r#" --- apiVersion: spark.stackable.tech/v1alpha1 kind: SparkApplication @@ -724,7 +729,7 @@ mod tests { mainApplicationFile: overlay.py sparkImage: productVersion: "4.1.0" - job: + deprecatedJob: envOverrides: TEST_OVERRIDDEN: overlay TEST_OVERLAY_ONLY: overlay @@ -742,7 +747,7 @@ mod tests { let merged = deep_merge(&base, &overlay); - let submit_env = &merged.spec.job.as_ref().unwrap().env_overrides; + let submit_env = &merged.spec.deprecated_job.as_ref().unwrap().env_overrides; assert_eq!(submit_env.get("TEST_BASE_ONLY"), Some(&"base".to_string())); assert_eq!( submit_env.get("TEST_OVERRIDDEN"), @@ -781,7 +786,7 @@ mod tests { #[test] fn test_deep_merge_pod_overrides() { - let base = serde_yaml::from_str::(indoc! {r#" + let base = serde_yaml::from_str::(indoc! {r#" --- apiVersion: spark.stackable.tech/v1alpha1 kind: SparkApplication @@ -792,7 +797,7 @@ mod tests { mainApplicationFile: base.py sparkImage: productVersion: "3.5.8" - job: + deprecatedJob: podOverrides: spec: serviceAccountName: base-sa @@ -817,7 +822,7 @@ mod tests { "#}) .unwrap(); - let overlay = serde_yaml::from_str::(indoc! {r#" + let overlay = serde_yaml::from_str::(indoc! {r#" --- apiVersion: spark.stackable.tech/v1alpha1 kind: SparkApplication @@ -828,7 +833,7 @@ mod tests { mainApplicationFile: overlay.py sparkImage: productVersion: "4.1.0" - job: + deprecatedJob: podOverrides: spec: serviceAccountName: overlay-sa @@ -857,7 +862,7 @@ mod tests { let submit_spec = merged .spec - .job + .deprecated_job .as_ref() .unwrap() .pod_overrides @@ -958,7 +963,7 @@ mod tests { #[test] fn test_deep_merge_pod_overrides_container_resources() { - let base = serde_yaml::from_str::(indoc! {r#" + let base = serde_yaml::from_str::(indoc! {r#" --- apiVersion: spark.stackable.tech/v1alpha1 kind: SparkApplication @@ -969,7 +974,7 @@ mod tests { mainApplicationFile: base.py sparkImage: productVersion: "3.5.8" - job: + deprecatedJob: podOverrides: spec: containers: @@ -981,7 +986,7 @@ mod tests { "#}) .unwrap(); - let overlay = serde_yaml::from_str::(indoc! {r#" + let overlay = serde_yaml::from_str::(indoc! {r#" --- apiVersion: spark.stackable.tech/v1alpha1 kind: SparkApplication @@ -992,7 +997,7 @@ mod tests { mainApplicationFile: overlay.py sparkImage: productVersion: "4.1.0" - job: + deprecatedJob: podOverrides: spec: containers: @@ -1009,7 +1014,7 @@ mod tests { let submit_spec = merged .spec - .job + .deprecated_job .as_ref() .unwrap() .pod_overrides @@ -1047,7 +1052,7 @@ mod tests { #[test] fn test_deep_merge_pod_overrides_base_only() { - let base = serde_yaml::from_str::(indoc! {r#" + let base = serde_yaml::from_str::(indoc! {r#" --- apiVersion: spark.stackable.tech/v1alpha1 kind: SparkApplication @@ -1058,7 +1063,7 @@ mod tests { mainApplicationFile: base.py sparkImage: productVersion: "3.5.8" - job: + deprecatedJob: podOverrides: spec: serviceAccountName: base-sa @@ -1069,7 +1074,7 @@ mod tests { "#}) .unwrap(); - let overlay = serde_yaml::from_str::(indoc! {r#" + let overlay = serde_yaml::from_str::(indoc! {r#" --- apiVersion: spark.stackable.tech/v1alpha1 kind: SparkApplication @@ -1080,7 +1085,7 @@ mod tests { mainApplicationFile: overlay.py sparkImage: productVersion: "4.1.0" - job: + deprecatedJob: config: retryOnFailureCount: 2 "#}) @@ -1089,7 +1094,7 @@ mod tests { let merged = deep_merge(&base, &overlay); let submit_spec = merged .spec - .job + .deprecated_job .as_ref() .unwrap() .pod_overrides @@ -1110,7 +1115,7 @@ mod tests { #[test] fn test_deep_merge_pod_overrides_overlay_only() { - let base = serde_yaml::from_str::(indoc! {r#" + let base = serde_yaml::from_str::(indoc! {r#" --- apiVersion: spark.stackable.tech/v1alpha1 kind: SparkApplication @@ -1121,13 +1126,13 @@ mod tests { mainApplicationFile: base.py sparkImage: productVersion: "3.5.8" - job: + deprecatedJob: config: retryOnFailureCount: 1 "#}) .unwrap(); - let overlay = serde_yaml::from_str::(indoc! {r#" + let overlay = serde_yaml::from_str::(indoc! {r#" --- apiVersion: spark.stackable.tech/v1alpha1 kind: SparkApplication @@ -1138,7 +1143,7 @@ mod tests { mainApplicationFile: overlay.py sparkImage: productVersion: "4.1.0" - job: + deprecatedJob: podOverrides: spec: serviceAccountName: overlay-sa @@ -1152,7 +1157,7 @@ mod tests { let merged = deep_merge(&base, &overlay); let submit_spec = merged .spec - .job + .deprecated_job .as_ref() .unwrap() .pod_overrides @@ -1177,7 +1182,7 @@ mod tests { #[test] fn test_merge_two_templates_into_spark_application() { let template_a = serde_yaml::from_str::< - crate::crd::template_spec::v1alpha1::SparkApplicationTemplate, + crate::crd::template_spec::v1alpha2::SparkApplicationTemplate, >(indoc! {r#" --- apiVersion: spark.stackable.tech/v1alpha1 @@ -1201,7 +1206,7 @@ mod tests { .unwrap(); let template_b = serde_yaml::from_str::< - crate::crd::template_spec::v1alpha1::SparkApplicationTemplate, + crate::crd::template_spec::v1alpha2::SparkApplicationTemplate, >(indoc! {r#" --- apiVersion: spark.stackable.tech/v1alpha1 @@ -1225,7 +1230,7 @@ mod tests { "#}) .unwrap(); - let spark_app = serde_yaml::from_str::(indoc! {r#" + let spark_app = serde_yaml::from_str::(indoc! {r#" --- apiVersion: spark.stackable.tech/v1alpha1 kind: SparkApplication @@ -1249,8 +1254,8 @@ mod tests { // Convert templates to SparkApplication and merge left-to-right: // template-a has the lowest priority, the spark application has the highest. - let app_from_a = crate::crd::v1alpha1::SparkApplication::from(template_a); - let app_from_b = crate::crd::v1alpha1::SparkApplication::from(template_b); + let app_from_a = crate::crd::v1alpha2::SparkApplication::from(template_a); + let app_from_b = crate::crd::v1alpha2::SparkApplication::from(template_b); // This how `merge_application_templates()` does it let template_apps = [app_from_a, app_from_b, spark_app]; let merged = template_apps @@ -1310,7 +1315,7 @@ mod tests { #[test] fn test_merge_template_role_affinities_into_spark_application() { let template = serde_yaml::from_str::< - crate::crd::template_spec::v1alpha1::SparkApplicationTemplate, + crate::crd::template_spec::v1alpha2::SparkApplicationTemplate, >(indoc! {r#" --- apiVersion: spark.stackable.tech/v1alpha1 @@ -1322,7 +1327,7 @@ mod tests { mainApplicationFile: local:///template.jar sparkImage: productVersion: "3.5.8" - job: + deprecatedJob: config: affinity: nodeSelector: @@ -1416,7 +1421,7 @@ mod tests { "#}) .unwrap(); - let spark_app = serde_yaml::from_str::(indoc! {r#" + let spark_app = serde_yaml::from_str::(indoc! {r#" --- apiVersion: spark.stackable.tech/v1alpha1 kind: SparkApplication @@ -1431,52 +1436,9 @@ mod tests { "#}) .unwrap(); - let app_from_template = crate::crd::v1alpha1::SparkApplication::from(template); + let app_from_template = crate::crd::v1alpha2::SparkApplication::from(template); let merged = deep_merge(&app_from_template, &spark_app); - let submit_affinity = merged.submit_config().unwrap().affinity; - assert_eq!( - Some("template-job"), - submit_affinity - .node_selector - .as_ref() - .and_then(|selectors| selectors.node_selector.get("affinity-role")) - .map(String::as_str) - ); - assert_eq!( - Some(11), - submit_affinity - .node_affinity - .as_ref() - .and_then(|a| a - .preferred_during_scheduling_ignored_during_execution - .as_ref()) - .and_then(|terms| terms.first()) - .map(|term| term.weight) - ); - assert_eq!( - Some(21), - submit_affinity - .pod_affinity - .as_ref() - .and_then(|a| a - .preferred_during_scheduling_ignored_during_execution - .as_ref()) - .and_then(|terms| terms.first()) - .map(|term| term.weight) - ); - assert_eq!( - Some(31), - submit_affinity - .pod_anti_affinity - .as_ref() - .and_then(|a| a - .preferred_during_scheduling_ignored_during_execution - .as_ref()) - .and_then(|terms| terms.first()) - .map(|term| term.weight) - ); - let driver_affinity = merged.driver_config().unwrap().affinity; assert_eq!( Some("template-driver"), diff --git a/rust/operator-binary/src/crd/template_spec.rs b/rust/operator-binary/src/crd/template_spec.rs index 073e526e..81582f67 100644 --- a/rust/operator-binary/src/crd/template_spec.rs +++ b/rust/operator-binary/src/crd/template_spec.rs @@ -51,6 +51,7 @@ pub enum Error { #[versioned( version(name = "v1alpha1"), + version(name = "v1alpha2"), crates( kube_core = "stackable_operator::kube::core", kube_client = "stackable_operator::kube::client", @@ -73,18 +74,33 @@ pub mod versioned { #[derive(Clone, CustomResource, Debug, Deserialize, JsonSchema, Serialize)] #[serde(rename_all = "camelCase")] pub struct SparkApplicationTemplateSpec { - /// The template body mirrors a [`SparkApplication`](super::super::v1alpha1::SparkApplication) + /// The template body mirrors a [`SparkApplication`](super::super::v1alpha2::SparkApplication) /// spec exactly — it is merged into a `SparkApplication` at reconcile time, see /// [`merge_application_templates`]. + /// + /// The embedded spec tracks the SparkApplication CRD version: a v1alpha1 template embeds a + /// v1alpha1 spec and a v1alpha2 template embeds a v1alpha2 spec. Conversion reuses the + /// auto-generated SparkApplicationSpec version conversions. + #[versioned(changed( + since = "v1alpha2", + from_type = "crate::crd::v1alpha1::SparkApplicationSpec", + downgrade_with = downgrade_template_spec + ))] #[serde(flatten)] - pub spec: crate::crd::v1alpha1::SparkApplicationSpec, + pub spec: crate::crd::v1alpha2::SparkApplicationSpec, } } -impl From<&v1alpha1::SparkApplicationTemplate> - for super::v1alpha1::ResolvedSparkApplicationTemplate -{ - fn from(value: &v1alpha1::SparkApplicationTemplate) -> Self { +/// Downgrades the embedded SparkApplicationSpec when converting a v1alpha2 template back to +/// v1alpha1. Delegates to the auto-generated SparkApplicationSpec conversion. +fn downgrade_template_spec( + spec: crate::crd::v1alpha2::SparkApplicationSpec, +) -> crate::crd::v1alpha1::SparkApplicationSpec { + spec.into() +} + +impl From<&v1alpha2::SparkApplicationTemplate> for super::ResolvedSparkApplicationTemplate { + fn from(value: &v1alpha2::SparkApplicationTemplate) -> Self { Self { name: value.name_any(), uid: value.metadata.uid.clone(), @@ -92,9 +108,9 @@ impl From<&v1alpha1::SparkApplicationTemplate> } } -impl From for super::v1alpha1::SparkApplication { - fn from(template: v1alpha1::SparkApplicationTemplate) -> super::v1alpha1::SparkApplication { - super::v1alpha1::SparkApplication { +impl From for super::v1alpha2::SparkApplication { + fn from(template: v1alpha2::SparkApplicationTemplate) -> super::v1alpha2::SparkApplication { + super::v1alpha2::SparkApplication { metadata: template.metadata, spec: template.spec.spec, status: None, @@ -137,12 +153,12 @@ const ANNO_TEMPLATE_UPDATE_STRATEGY: &str = "spark-application.template.updateSt // Currently only "enforce" is supported, meaning: fail in case of errors. const ANNO_TEMPLATE_APPLY_STRATEGY: &str = "spark-application.template.applyStrategy"; -impl TryFrom<&super::v1alpha1::SparkApplication> for MergeTemplateOptions { +impl TryFrom<&super::v1alpha2::SparkApplication> for MergeTemplateOptions { type Error = Error; // Build a `MergeTemplateOptions` value from the metadata annotations of a `SparkApplication`. // The `template_names` are sorted in the order in which they are applied as specified by the `index`. - fn try_from(app: &super::v1alpha1::SparkApplication) -> Result { + fn try_from(app: &super::v1alpha2::SparkApplication) -> Result { if let Some(annos) = app.metadata.annotations.as_ref() { let merge: bool = match annos.get(ANNO_TEMPLATE_MERGE) { Some(v) => { @@ -198,12 +214,12 @@ impl TryFrom<&super::v1alpha1::SparkApplication> for MergeTemplateOptions { } pub(crate) struct MergeTemplateResult { - pub app: Option, - pub resolved_template_ref: Vec, + pub app: Option, + pub resolved_template_ref: Vec, } -// Merges one or more [`SparkApplicationTemplate`](v1alpha1::SparkApplicationTemplate) resources -// into the given [`SparkApplication`](super::v1alpha1::SparkApplication). +// Merges one or more [`SparkApplicationTemplate`](v1alpha2::SparkApplicationTemplate) resources +// into the given [`SparkApplication`](super::v1alpha2::SparkApplication). // // Template merging is controlled by annotations on the `SparkApplication`. // @@ -234,7 +250,7 @@ pub(crate) struct MergeTemplateResult { // - `Err(Error)` if annotation parsing fails or a Kubernetes API call returns an error. pub(crate) async fn merge_application_templates( client: &stackable_operator::client::Client, - spark_application: &super::v1alpha1::SparkApplication, + spark_application: &super::v1alpha2::SparkApplication, ) -> Result { let app_name = spark_application.name_any(); @@ -279,11 +295,11 @@ pub(crate) async fn merge_application_templates( // The list of apps from templates in the correct order. // The final element is the actual Spark application being reconciled // which has the highest priority during merging. - let mut template_apps: Vec = templates + let mut template_apps: Vec = templates .iter() .cloned() - .map(super::v1alpha1::SparkApplication::from) - .collect::>(); + .map(super::v1alpha2::SparkApplication::from) + .collect::>(); template_apps.push(spark_application.clone()); // Deep merge app templates from left to right @@ -295,8 +311,8 @@ pub(crate) async fn merge_application_templates( // list might differ from what is in the app annotations so make sure we return the correct one. let effective_template_list = templates .iter() - .map(super::v1alpha1::ResolvedSparkApplicationTemplate::from) - .collect::>(); + .map(super::ResolvedSparkApplicationTemplate::from) + .collect::>(); tracing::info!( "app [{app_name}] : successfully merged templates [{tnames}]", @@ -323,7 +339,7 @@ pub(crate) async fn merge_application_templates( } fn spark_application_namespace( - spark_application: &super::v1alpha1::SparkApplication, + spark_application: &super::v1alpha2::SparkApplication, ) -> Result<&str, Error> { spark_application .metadata @@ -337,13 +353,13 @@ async fn resolve( namespace: &str, template_names: &[String], apply_strategy: TemplateApplyStrategy, -) -> Result, Error> { +) -> Result, Error> { if template_names.is_empty() { return Ok(vec![]); } let templates_api = - Api::::namespaced(client.as_kube_client(), namespace); + Api::::namespaced(client.as_kube_client(), namespace); let mut resolved_templates = Vec::new(); for template_name in template_names { let template_res = templates_api @@ -378,7 +394,7 @@ mod tests { #[test] fn try_from_parses_annotations_and_sorts_template_names() { let spark_application = - serde_yaml::from_str::(indoc! {r#" + serde_yaml::from_str::(indoc! {r#" --- apiVersion: spark.stackable.tech/v1alpha1 kind: SparkApplication @@ -423,7 +439,7 @@ mod tests { #[test] fn try_from_without_annotations_returns_default_options() { let spark_application = - serde_yaml::from_str::(indoc! {r#" + serde_yaml::from_str::(indoc! {r#" --- apiVersion: spark.stackable.tech/v1alpha1 kind: SparkApplication @@ -454,7 +470,7 @@ mod tests { #[test] fn spark_application_namespace_returns_namespace() { let spark_application = - serde_yaml::from_str::(indoc! {r#" + serde_yaml::from_str::(indoc! {r#" --- apiVersion: spark.stackable.tech/v1alpha1 kind: SparkApplication @@ -478,7 +494,7 @@ mod tests { #[test] fn spark_application_namespace_returns_error_when_missing() { let spark_application = - serde_yaml::from_str::(indoc! {r#" + serde_yaml::from_str::(indoc! {r#" --- apiVersion: spark.stackable.tech/v1alpha1 kind: SparkApplication @@ -498,9 +514,19 @@ mod tests { )); } - impl RoundtripTestData for v1alpha1::SparkApplicationTemplateSpec { + impl RoundtripTestData for v1alpha2::SparkApplicationTemplateSpec { fn roundtrip_test_data() -> Vec { // SparkApplicationTemplateSpec is just a wrapper around SparkApplicationSpec + let test_data = crate::crd::v1alpha2::SparkApplicationSpec::roundtrip_test_data(); + test_data + .into_iter() + .map(|spec| v1alpha2::SparkApplicationTemplateSpec { spec }) + .collect() + } + } + + impl RoundtripTestData for v1alpha1::SparkApplicationTemplateSpec { + fn roundtrip_test_data() -> Vec { let test_data = crate::crd::v1alpha1::SparkApplicationSpec::roundtrip_test_data(); test_data .into_iter() diff --git a/rust/operator-binary/src/main.rs b/rust/operator-binary/src/main.rs index 6ae07183..060c7b81 100644 --- a/rust/operator-binary/src/main.rs +++ b/rust/operator-binary/src/main.rs @@ -15,6 +15,7 @@ use stackable_operator::{ eos::EndOfSupportChecker, k8s_openapi::api::{ apps::v1::StatefulSet, + batch::v1::Job, core::v1::{ConfigMap, Pod, Service}, }, kube::{ @@ -83,13 +84,13 @@ async fn main() -> anyhow::Result<()> { let opts = Opts::parse(); match opts.cmd { Command::Crd => { - SparkApplication::merged_crd(crd::SparkApplicationVersion::V1Alpha1)? + SparkApplication::merged_crd(crd::SparkApplicationVersion::V1Alpha2)? .print_yaml_schema(built_info::PKG_VERSION, &SerializeOptions::default())?; SparkHistoryServer::merged_crd(crd::history::SparkHistoryServerVersion::V1Alpha1)? .print_yaml_schema(built_info::PKG_VERSION, &SerializeOptions::default())?; SparkConnectServer::merged_crd(SparkConnectServerVersion::V1Alpha1)? .print_yaml_schema(built_info::PKG_VERSION, &SerializeOptions::default())?; - SparkApplicationTemplate::merged_crd(SparkApplicationTemplateVersion::V1Alpha1)? + SparkApplicationTemplate::merged_crd(SparkApplicationTemplateVersion::V1Alpha2)? .print_yaml_schema(built_info::PKG_VERSION, &SerializeOptions::default())?; } Command::Run(RunArguments { @@ -157,13 +158,21 @@ async fn main() -> anyhow::Result<()> { )); let app_controller = Controller::new( watch_namespace - .get_api::>(&client), + .get_api::>(&client), watcher::Config::default(), ) .owns( watch_namespace.get_api::>(&client), watcher::Config::default(), ) + .owns( + watch_namespace.get_api::>(&client), + watcher::Config::default(), + ) + .owns( + watch_namespace.get_api::>(&client), + watcher::Config::default(), + ) .graceful_shutdown_on(sigterm_watcher.handle()) .run( spark_k8s_controller::reconcile, @@ -383,7 +392,7 @@ async fn main() -> anyhow::Result<()> { }; let delayed_app_controller = async { - signal::crd_established(&client, crd::v1alpha1::SparkApplication::crd_name(), None) + signal::crd_established(&client, crd::v1alpha2::SparkApplication::crd_name(), None) .await?; app_controller.await }; diff --git a/rust/operator-binary/src/pod_driver_controller.rs b/rust/operator-binary/src/pod_driver_controller.rs index 2587ed86..02940ea3 100644 --- a/rust/operator-binary/src/pod_driver_controller.rs +++ b/rust/operator-binary/src/pod_driver_controller.rs @@ -13,7 +13,7 @@ use stackable_operator::{ }; use strum::{EnumDiscriminants, IntoStaticStr}; -use crate::crd::{constants::POD_DRIVER_CONTROLLER_NAME, v1alpha1}; +use crate::crd::{constants::POD_DRIVER_CONTROLLER_NAME, v1alpha2}; const LABEL_NAME_INSTANCE: &str = "app.kubernetes.io/instance"; @@ -49,12 +49,6 @@ pub enum Error { InvalidPod { source: error_boundary::InvalidObject, }, - - #[snafu(display("cannot delete Spark driver pod {pod_name:?}"))] - DeleteDriverPod { - source: stackable_operator::client::Error, - pod_name: String, - }, } type Result = std::result::Result; @@ -93,7 +87,7 @@ pub async fn reconcile(pod: Arc>, client: Arc) -> )?; let app = client - .get::( + .get::( app_name.as_ref(), pod.metadata .namespace @@ -111,7 +105,7 @@ pub async fn reconcile(pod: Arc>, client: Arc) -> .apply_patch_status( POD_DRIVER_CONTROLLER_NAME, &app, - &v1alpha1::SparkApplicationStatus { + &crate::crd::SparkApplicationStatus { phase: phase.clone(), resolved_template_ref: app .status @@ -125,19 +119,8 @@ pub async fn reconcile(pod: Arc>, client: Arc) -> name: app_name.clone(), })?; - // We must manually delete the driver pod when the application reached a terminal state - // otherwise they are left hanging forever. - if phase == "Succeeded" || phase == "Failed" { - tracing::info!( - "Spark application {app_name:?} completed with phase {phase:?}, deleting driver pod {pod_name:?}" - ); - client - .delete(pod) - .await - .with_context(|_| DeleteDriverPodSnafu { - pod_name: pod_name.clone(), - })?; - } + // The driver pod is now owned by the driver Job. We must not delete it ourselves: the Job's + // `ttlSecondsAfterFinished` cleans up the Job and its pod once the application has finished. Ok(Action::await_change()) } diff --git a/rust/operator-binary/src/spark_k8s_controller.rs b/rust/operator-binary/src/spark_k8s_controller.rs index 3b4a70be..3928c2d7 100644 --- a/rust/operator-binary/src/spark_k8s_controller.rs +++ b/rust/operator-binary/src/spark_k8s_controller.rs @@ -23,8 +23,8 @@ use stackable_operator::{ api::{ batch::v1::{Job, JobSpec}, core::v1::{ - Affinity, ConfigMap, Container, EnvVar, PodSecurityContext, PodSpec, - PodTemplateSpec, ServiceAccount, Volume, + ConfigMap, Container, EnvVar, PodSecurityContext, PodTemplateSpec, Service, + ServiceAccount, ServicePort, ServiceSpec, Volume, }, rbac::v1::{ClusterRole, RoleBinding, RoleRef, Subject}, }, @@ -56,8 +56,8 @@ use crate::{ crd::{ constants::*, logdir::ResolvedLogDir, - roles::{RoleConfig, SparkApplicationRole, SparkContainer, SubmitConfig}, - tlscerts, to_spark_env_sh_string, v1alpha1, + roles::{RoleConfig, SparkApplicationRole, SparkContainer}, + tlscerts, to_spark_env_sh_string, v1alpha2, }, product_logging::{self}, }; @@ -135,9 +135,6 @@ pub enum Error { role: SparkApplicationRole, }, - #[snafu(display("invalid submit config"))] - SubmitConfig { source: crate::crd::Error }, - #[snafu(display("failed to build Labels"))] LabelBuild { source: stackable_operator::kvp::LabelError, @@ -180,7 +177,7 @@ impl ReconcilerError for Error { } pub async fn reconcile( - spark_application: Arc>, + spark_application: Arc>, ctx: Arc, ) -> Result { tracing::info!("Starting reconcile"); @@ -296,38 +293,25 @@ pub async fn reconcile( .build_command(opt_s3conn, logdir, &resolved_product_image.image) .context(BuildCommandSnafu)?; - let submit_config = spark_application - .submit_config() - .context(SubmitConfigSnafu)?; - - let submit_product_config: Option<&HashMap>> = - validated_product_config - .get(&SparkApplicationRole::Submit.to_string()) - .and_then(|r| r.get(&"default".to_string())); - - let submit_job_config_map = submit_job_config_map( - spark_application, - submit_product_config, - resolved_product_image, - )?; + // The driver runs in client mode, so executors connect back to it via a headless Service that + // selects the driver pod of this application. + let driver_service = driver_service(spark_application, resolved_product_image)?; client - .apply_patch( - SPARK_CONTROLLER_NAME, - &submit_job_config_map, - &submit_job_config_map, - ) + .apply_patch(SPARK_CONTROLLER_NAME, &driver_service, &driver_service) .await .context(ApplyApplicationSnafu)?; - let job = spark_job( + // The driver itself now runs directly as a Kubernetes Job (no separate spark-submit process). + // Its pod is built from `spec.driver`. + let job = driver_job( spark_application, - resolved_product_image, - &serviceaccount, + &driver_config, &env_vars, &job_commands, opt_s3conn, logdir, - &submit_config, + resolved_product_image, + &serviceaccount, )?; client .apply_patch(SPARK_CONTROLLER_NAME, &job, &job) @@ -341,7 +325,7 @@ pub async fn reconcile( .apply_patch_status( SPARK_CONTROLLER_NAME, spark_application, - &v1alpha1::SparkApplicationStatus { + &crate::crd::SparkApplicationStatus { phase: "Unknown".to_string(), resolved_template_ref: validated.resolved_template_refs.clone(), }, @@ -355,7 +339,7 @@ pub async fn reconcile( } fn init_containers( - spark_application: &v1alpha1::SparkApplication, + spark_application: &v1alpha2::SparkApplication, logging: &Logging, s3conn: &Option, logdir: &Option, @@ -510,7 +494,7 @@ fn init_containers( #[allow(clippy::too_many_arguments)] fn pod_template( - spark_application: &v1alpha1::SparkApplication, + spark_application: &v1alpha2::SparkApplication, role: SparkApplicationRole, config: &RoleConfig, volumes: &[Volume], @@ -519,6 +503,11 @@ fn pod_template( logdir: &Option, spark_image: &ResolvedProductImage, service_account: &ServiceAccount, + // When set, the spark container runs this command (the spark-submit client invocation). This is + // used for the driver pod, which the operator now launches directly as a Kubernetes Job. When + // `None` (executor pod template), the container relies on the image entrypoint, since Spark sets + // the command on the executor pods it creates. + command: Option<&[String]>, ) -> Result { let container_name = SparkContainer::Spark.to_string(); let mut cb = ContainerBuilder::new(&container_name).context(IllegalContainerNameSnafu)?; @@ -530,6 +519,43 @@ fn pod_template( .resources(config.resources.clone().into()) .image_from_product_image(spark_image); + if let Some(command) = command { + // The driver mounts the executor pod template so that Spark's Kubernetes backend can create + // executor pods from it. + cb.add_volume_mount( + VOLUME_MOUNT_NAME_EXECUTOR_POD_TEMPLATES, + VOLUME_MOUNT_PATH_EXECUTOR_POD_TEMPLATES, + ) + .context(AddVolumeMountSnafu)?; + + // The SPARK_SUBMIT_OPTS env var configures the JVM settings of the spark-submit/driver + // process: it points the JVM to our logging configuration and, if S3 (data or Spark + // History) is used, to the trust store. + let mut spark_submit_opts = vec![format!( + "-Dlog4j.configurationFile={VOLUME_MOUNT_PATH_LOG_CONFIG}/{LOG4J2_CONFIG_FILE}" + )]; + if tlscerts::tls_secret_names(s3conn, logdir).is_some() { + spark_submit_opts.push(format!( + "-Djavax.net.ssl.trustStore={STACKABLE_TRUST_STORE}/truststore.p12" + )); + spark_submit_opts.push(format!( + "-Djavax.net.ssl.trustStorePassword={STACKABLE_TLS_STORE_PASSWORD}" + )); + } + + cb.command(vec![ + "/bin/bash".to_string(), + "-x".to_string(), + "-euo".to_string(), + "pipefail".to_string(), + "-c".to_string(), + ]) + .args(vec![command.join("\n")]) + .add_env_var("SPARK_SUBMIT_OPTS", spark_submit_opts.join(" ")) + // TODO: move this to the image + .add_env_var("SPARK_CONF_DIR", "/stackable/spark/conf"); + } + if config.logging.enable_vector_agent { cb.add_env_var( "_STACKABLE_POST_HOOK", @@ -560,6 +586,14 @@ fn pod_template( omb.with_label(Label::try_from(("prometheus.io/scrape", "true")).context(LabelBuildSnafu)?); } + // The actual driver pod (the Job pod, identified by `command` being set) needs the `spark-role` + // label so the headless driver Service can select it and the operator can identify it. + if command.is_some() { + omb.with_label( + Label::try_from((SPARK_ROLE_LABEL, SPARK_ROLE_DRIVER)).context(LabelBuildSnafu)?, + ); + } + let mut metadata = omb.build(); // We explicitly remove the application owner reference from driver and executor pods. @@ -633,7 +667,7 @@ fn pod_template( #[allow(clippy::too_many_arguments)] fn pod_template_config_map( - spark_application: &v1alpha1::SparkApplication, + spark_application: &v1alpha2::SparkApplication, role: SparkApplicationRole, merged_config: &RoleConfig, product_config: Option<&HashMap>>, @@ -684,6 +718,7 @@ fn pod_template_config_map( logdir, spark_image, service_account, + None, )?; let mut cm_builder = ConfigMapBuilder::new(); @@ -751,169 +786,129 @@ fn pod_template_config_map( cm_builder.build().context(PodTemplateConfigMapSnafu) } -fn submit_job_config_map( - spark_application: &v1alpha1::SparkApplication, - product_config: Option<&HashMap>>, +/// Headless Service that exposes the driver pod so executors can connect back to it in client mode. +fn driver_service( + spark_application: &v1alpha2::SparkApplication, spark_image: &ResolvedProductImage, -) -> Result { - let cm_name = spark_application.submit_job_config_map_name(); - - let mut cm_builder = ConfigMapBuilder::new(); +) -> Result { + // Select exactly the driver pod of this application. + let selector = BTreeMap::from([ + ( + "app.kubernetes.io/instance".to_string(), + spark_application.name_any(), + ), + (SPARK_ROLE_LABEL.to_string(), SPARK_ROLE_DRIVER.to_string()), + ]); - cm_builder.metadata( - ObjectMetaBuilder::new() + let service = Service { + metadata: ObjectMetaBuilder::new() .name_and_namespace(spark_application) - .name(&cm_name) + .name(spark_application.driver_service_name()) .ownerreference_from_resource(spark_application, None, Some(true)) .context(ObjectMissingMetadataForOwnerRefSnafu)? .with_recommended_labels( &spark_application - .build_recommended_labels(&spark_image.app_version_label_value, "spark-submit"), + .build_recommended_labels(&spark_image.app_version_label_value, "driver"), ) .context(MetadataBuildSnafu)? .build(), - ); - - if let Some(product_config) = product_config { - cm_builder.add_data( - SPARK_ENV_SH_FILE_NAME, - to_spark_env_sh_string( - product_config - .get(&PropertyNameKind::File(SPARK_ENV_SH_FILE_NAME.to_string())) - .cloned() - .unwrap_or_default() - .iter(), - ), - ); - - let jvm_sec_props: BTreeMap> = product_config - .get(&PropertyNameKind::File( - JVM_SECURITY_PROPERTIES_FILE.to_string(), - )) - .cloned() - .unwrap_or_default() - .into_iter() - .map(|(k, v)| (k, Some(v))) - .collect(); - - cm_builder.add_data( - JVM_SECURITY_PROPERTIES_FILE, - to_java_properties_string(jvm_sec_props.iter()).with_context(|_| { - JvmSecurityPropertiesSnafu { - role: SparkApplicationRole::Submit, - } - })?, - ); - } + spec: Some(ServiceSpec { + // Headless: executors resolve the driver pod directly. + cluster_ip: Some("None".to_string()), + selector: Some(selector), + // The driver must be reachable as soon as its pod has an IP, even before it is "ready". + publish_not_ready_addresses: Some(true), + ports: Some(vec![ + ServicePort { + name: Some("driver".to_string()), + port: i32::from(DRIVER_PORT), + ..ServicePort::default() + }, + ServicePort { + name: Some("block-manager".to_string()), + port: i32::from(DRIVER_BLOCK_MANAGER_PORT), + ..ServicePort::default() + }, + ]), + ..ServiceSpec::default() + }), + status: None, + }; - cm_builder.build().context(PodTemplateConfigMapSnafu) + Ok(service) } +/// The driver Job. Its pod runs `spark-submit` in client mode and therefore *is* the Spark driver. +/// The pod spec is built from `spec.driver`. Executors are created by the driver via Spark's +/// Kubernetes backend. #[allow(clippy::too_many_arguments)] -fn spark_job( - spark_application: &v1alpha1::SparkApplication, - spark_image: &ResolvedProductImage, - serviceaccount: &ServiceAccount, +fn driver_job( + spark_application: &v1alpha2::SparkApplication, + driver_config: &RoleConfig, env: &[EnvVar], job_commands: &[String], s3conn: &Option, logdir: &Option, - job_config: &SubmitConfig, + spark_image: &ResolvedProductImage, + service_account: &ServiceAccount, ) -> Result { - let mut cb = ContainerBuilder::new(&SparkContainer::SparkSubmit.to_string()) - .context(IllegalContainerNameSnafu)?; + let cm_name = spark_application.pod_template_config_map_name(SparkApplicationRole::Driver); - let merged_env = spark_application.merged_env(SparkApplicationRole::Submit, env); - - // The SPARK_SUBMIT_OPTS env var is used to configure the JVM settings of the spark-submit job. - // Here we need to point the JVM to our logging configuration and if S3 is used for data or Spark History, - // we also need to tell the JVM where the trust store is located. - // The same properties are also set for the driver and executor pods via the pod template config maps. - let mut spark_submit_opts_env = vec![format!( - "-Dlog4j.configurationFile={VOLUME_MOUNT_PATH_LOG_CONFIG}/{LOG4J2_CONFIG_FILE}" - )]; - if tlscerts::tls_secret_names(s3conn, logdir).is_some() { - spark_submit_opts_env.push(format!( - "-Djavax.net.ssl.trustStore={STACKABLE_TRUST_STORE}/truststore.p12" - )); - spark_submit_opts_env.push(format!( - "-Djavax.net.ssl.trustStorePassword={STACKABLE_TLS_STORE_PASSWORD}" - )); - } - cb.image_from_product_image(spark_image) - .command(vec![ - "/bin/bash".to_string(), - "-x".to_string(), - "-euo".to_string(), - "pipefail".to_string(), - "-c".to_string(), - ]) - .args(vec![job_commands.join("\n")]) - .resources(job_config.resources.clone().into()) - .add_volume_mounts(spark_application.spark_job_volume_mounts(s3conn, logdir)) - .context(AddVolumeMountSnafu)? - .add_env_vars(merged_env) - .add_env_var("SPARK_SUBMIT_OPTS", spark_submit_opts_env.join(" ")) - // TODO: move this to the image - .add_env_var("SPARK_CONF_DIR", "/stackable/spark/conf"); + let log_config_map = if let Some(ContainerLogConfig { + choice: + Some(ContainerLogConfigChoice::Custom(CustomContainerLogConfig { + custom: ConfigMapLogConfig { config_map }, + })), + }) = driver_config.logging.containers.get(&SparkContainer::Spark) + { + config_map.into() + } else { + cm_name.clone() + }; + + let requested_secret_lifetime = driver_config + .requested_secret_lifetime + .context(MissingSecretLifetimeSnafu)?; - let mut volumes = vec![ + let mut volumes = spark_application + .volumes( + s3conn, + logdir, + Some(&log_config_map), + &requested_secret_lifetime, + ) + .context(CreateVolumesSnafu)?; + // The driver's own config (spark-env.sh, security.properties, log4j) ConfigMap. + volumes.push( VolumeBuilder::new(VOLUME_MOUNT_NAME_CONFIG) - .with_config_map(spark_application.submit_job_config_map_name()) - .build(), - VolumeBuilder::new(VOLUME_MOUNT_NAME_DRIVER_POD_TEMPLATES) - .with_config_map( - spark_application.pod_template_config_map_name(SparkApplicationRole::Driver), - ) + .with_config_map(&cm_name) .build(), + ); + // The executor pod template ConfigMap, read by the driver's Spark Kubernetes backend. + volumes.push( VolumeBuilder::new(VOLUME_MOUNT_NAME_EXECUTOR_POD_TEMPLATES) .with_config_map( spark_application.pod_template_config_map_name(SparkApplicationRole::Executor), ) .build(), - ]; - let requested_secret_lifetime = job_config - .requested_secret_lifetime - .context(MissingSecretLifetimeSnafu)?; - volumes.extend( - spark_application - .volumes(s3conn, logdir, None, &requested_secret_lifetime) - .context(CreateVolumesSnafu)?, ); - let containers = vec![cb.build()]; - - let mut pod = PodTemplateSpec { - metadata: Some( - ObjectMetaBuilder::new() - .name("spark-submit") - .with_recommended_labels(&spark_application.build_recommended_labels( - &spark_image.app_version_label_value, - "spark-job-template", - )) - .context(MetadataBuildSnafu)? - .build(), - ), - spec: Some(PodSpec { - containers, - restart_policy: Some("Never".to_string()), - service_account_name: serviceaccount.metadata.name.clone(), - volumes: Some(volumes), - affinity: Some(Affinity { - node_affinity: job_config.affinity.node_affinity.clone(), - pod_affinity: job_config.affinity.pod_affinity.clone(), - pod_anti_affinity: job_config.affinity.pod_anti_affinity.clone(), - }), - image_pull_secrets: spark_image.pull_secrets.clone(), - security_context: Some(security_context()), - ..PodSpec::default() - }), - }; + let mut template = pod_template( + spark_application, + SparkApplicationRole::Driver, + driver_config, + &volumes, + env, + s3conn, + logdir, + spark_image, + service_account, + Some(job_commands), + )?; - if let Some(submit_pod_overrides) = - spark_application.pod_overrides(SparkApplicationRole::Submit) - { - pod.merge_from(submit_pod_overrides); + // A Job's pod must declare a restart policy. + if let Some(spec) = template.spec.as_mut() { + spec.restart_policy = Some("Never".to_string()); } let job = Job { @@ -928,9 +923,11 @@ fn spark_job( .context(MetadataBuildSnafu)? .build(), spec: Some(JobSpec { - template: pod, + template, ttl_seconds_after_finished: Some(600), - backoff_limit: Some(spark_application.retry_on_failure_count()), + // The driver Job is not retried by default. `spec.job.retryOnFailureCount` configured the + // old submit Job and is deprecated since v1alpha2. + backoff_limit: Some(0), ..Default::default() }), status: None, @@ -944,7 +941,7 @@ fn spark_job( /// Both objects have an owner reference to the SparkApplication, as well as the same name as the app. /// They are deleted when the job is deleted. fn build_spark_role_serviceaccount( - spark_app: &v1alpha1::SparkApplication, + spark_app: &v1alpha2::SparkApplication, spark_image: &ResolvedProductImage, ) -> Result<(ServiceAccount, RoleBinding)> { // TODO (@NickLarsenNZ): Explain this unwrap. Either convert to expect, or gracefully handle the error. @@ -1000,7 +997,7 @@ fn security_context() -> PodSecurityContext { } pub fn error_policy( - _obj: Arc>, + _obj: Arc>, error: &Error, _ctx: Arc, ) -> Action { diff --git a/rust/operator-binary/src/spark_k8s_controller/dereference.rs b/rust/operator-binary/src/spark_k8s_controller/dereference.rs index aa68ea14..3bce2027 100644 --- a/rust/operator-binary/src/spark_k8s_controller/dereference.rs +++ b/rust/operator-binary/src/spark_k8s_controller/dereference.rs @@ -8,9 +8,10 @@ use snafu::{ResultExt, Snafu}; use stackable_operator::{client::Client, crd::s3}; use crate::crd::{ + ResolvedSparkApplicationTemplate, logdir::ResolvedLogDir, template_spec::{self}, - v1alpha1, + v1alpha2, }; #[derive(Snafu, Debug)] @@ -35,9 +36,9 @@ type Result = std::result::Result; /// Kubernetes objects referenced from a SparkApplication, already fetched. pub struct DereferencedSparkApplication { /// SparkApplication after merging any referenced templates. - pub spark_application: v1alpha1::SparkApplication, + pub spark_application: v1alpha2::SparkApplication, /// Resolved template references for status reporting. - pub resolved_template_refs: Vec, + pub resolved_template_refs: Vec, /// Resolved S3 connection, if `spec.s3connection` is set. pub s3_connection: Option, /// Resolved log directory, if `spec.log_file_directory` is set. @@ -47,7 +48,7 @@ pub struct DereferencedSparkApplication { /// Fetches all Kubernetes objects referenced from the given SparkApplication. pub async fn dereference( client: &Client, - spark_application: &v1alpha1::SparkApplication, + spark_application: &v1alpha2::SparkApplication, ) -> Result { // 1. Template merging — must happen first so subsequent lookups see the merged spec. let merged = template_spec::merge_application_templates(client, spark_application) diff --git a/rust/operator-binary/src/spark_k8s_controller/validate.rs b/rust/operator-binary/src/spark_k8s_controller/validate.rs index ae9b6682..b3cf26a2 100644 --- a/rust/operator-binary/src/spark_k8s_controller/validate.rs +++ b/rust/operator-binary/src/spark_k8s_controller/validate.rs @@ -16,7 +16,10 @@ use stackable_operator::{ }; use crate::{ - crd::{constants::CONTAINER_IMAGE_BASE_NAME, logdir::ResolvedLogDir, v1alpha1}, + crd::{ + ResolvedSparkApplicationTemplate, constants::CONTAINER_IMAGE_BASE_NAME, + logdir::ResolvedLogDir, v1alpha2, + }, spark_k8s_controller::dereference::DereferencedSparkApplication, }; @@ -38,8 +41,8 @@ type Result = std::result::Result; /// Inputs the rest of `reconcile` needs after dereferencing. pub struct ValidatedSparkApplication { - pub spark_application: v1alpha1::SparkApplication, - pub resolved_template_refs: Vec, + pub spark_application: v1alpha2::SparkApplication, + pub resolved_template_refs: Vec, pub s3_connection: Option, pub log_dir: Option, pub resolved_product_image: ResolvedProductImage, diff --git a/rust/operator-binary/src/webhooks/conversion.rs b/rust/operator-binary/src/webhooks/conversion.rs index 2bd9d683..468f7abd 100644 --- a/rust/operator-binary/src/webhooks/conversion.rs +++ b/rust/operator-binary/src/webhooks/conversion.rs @@ -47,12 +47,12 @@ pub async fn create_webhook_server( SparkConnectServer::try_convert as fn(_) -> _, ), ( - SparkApplication::merged_crd(SparkApplicationVersion::V1Alpha1) + SparkApplication::merged_crd(SparkApplicationVersion::V1Alpha2) .context(MergeCrdSnafu)?, SparkApplication::try_convert as fn(_) -> _, ), ( - SparkApplicationTemplate::merged_crd(SparkApplicationTemplateVersion::V1Alpha1) + SparkApplicationTemplate::merged_crd(SparkApplicationTemplateVersion::V1Alpha2) .context(MergeCrdSnafu)?, SparkApplicationTemplate::try_convert as fn(_) -> _, ), diff --git a/tests/templates/kuttl/overrides/10-assert.yaml b/tests/templates/kuttl/overrides/10-assert.yaml index 785637fb..a40e3fb6 100644 --- a/tests/templates/kuttl/overrides/10-assert.yaml +++ b/tests/templates/kuttl/overrides/10-assert.yaml @@ -18,7 +18,9 @@ spec: template: spec: containers: - - name: spark-submit + # The driver Job's container is named "spark"; its resources come from the driver + # podOverrides. + - name: spark resources: limits: cpu: 1500m diff --git a/tests/templates/kuttl/overrides/11-assert.yaml b/tests/templates/kuttl/overrides/11-assert.yaml index af5d33b4..390299dc 100644 --- a/tests/templates/kuttl/overrides/11-assert.yaml +++ b/tests/templates/kuttl/overrides/11-assert.yaml @@ -23,17 +23,6 @@ commands: --- apiVersion: v1 kind: ConfigMap -metadata: - name: spark-pi-s3-1-submit-job -data: - security.properties: | - networkaddress.cache.negative.ttl=0 - networkaddress.cache.ttl=30 - test.job.securityProperties=test - spark-env.sh: export TEST_JOB_SPARK-ENV-SH="TEST" ---- -apiVersion: v1 -kind: ConfigMap metadata: name: spark-pi-s3-1-driver-pod-template data: diff --git a/tests/templates/kuttl/resources/10-assert.yaml.j2 b/tests/templates/kuttl/resources/10-assert.yaml.j2 index 4afdc9c5..413daf5c 100644 --- a/tests/templates/kuttl/resources/10-assert.yaml.j2 +++ b/tests/templates/kuttl/resources/10-assert.yaml.j2 @@ -3,27 +3,14 @@ apiVersion: kuttl.dev/v1beta1 kind: TestAssert timeout: 600 --- +# The driver now runs directly as a Kubernetes Job (no separate spark-submit pod). Its pod has a +# generated name, so it is selected via labels. apiVersion: v1 kind: Pod metadata: labels: - job-name: resources-crd - app.kubernetes.io/managed-by: spark.stackable.tech_sparkapplication -spec: - containers: - - name: spark-submit - resources: - limits: - cpu: 500m - memory: 512Mi - requests: - cpu: 250m - memory: 512Mi ---- -apiVersion: v1 -kind: Pod -metadata: - name: resources-crd-driver + app.kubernetes.io/instance: resources-crd + spark-role: driver spec: containers: {% if lookup('env', 'VECTOR_AGGREGATOR') %} diff --git a/tests/templates/kuttl/resources/12-assert.yaml.j2 b/tests/templates/kuttl/resources/12-assert.yaml.j2 index 39d11548..f5380433 100644 --- a/tests/templates/kuttl/resources/12-assert.yaml.j2 +++ b/tests/templates/kuttl/resources/12-assert.yaml.j2 @@ -3,27 +3,14 @@ apiVersion: kuttl.dev/v1beta1 kind: TestAssert timeout: 900 --- +# The driver now runs directly as a Kubernetes Job (no separate spark-submit pod). Its pod has a +# generated name, so it is selected via labels. apiVersion: v1 kind: Pod metadata: labels: - job-name: resources-sparkconf - app.kubernetes.io/managed-by: spark.stackable.tech_sparkapplication -spec: - containers: - - name: spark-submit - resources: - limits: - cpu: 400m - memory: 512Mi - requests: - cpu: 100m - memory: 512Mi ---- -apiVersion: v1 -kind: Pod -metadata: - name: resources-sparkconf-driver + app.kubernetes.io/instance: resources-sparkconf + spark-role: driver spec: containers: {% if lookup('env', 'VECTOR_AGGREGATOR') %} diff --git a/tests/templates/kuttl/smoke/50-assert.yaml b/tests/templates/kuttl/smoke/50-assert.yaml index 1319f33a..b793a4db 100644 --- a/tests/templates/kuttl/smoke/50-assert.yaml +++ b/tests/templates/kuttl/smoke/50-assert.yaml @@ -11,5 +11,5 @@ apiVersion: kuttl.dev/v1beta1 kind: TestAssert commands: - script: | - SPARK_SUBMIT_POD=$(kubectl get -n $NAMESPACE pods --field-selector=status.phase=Running --selector batch.kubernetes.io/job-name=spark-pi-s3-1 -o jsonpath='{.items[0].metadata.name}') - kubectl exec -n $NAMESPACE --container spark-submit $SPARK_SUBMIT_POD -- cat /stackable/log/containerdebug-state.json | jq --exit-status '"valid JSON"' + DRIVER_POD=$(kubectl get -n $NAMESPACE pods --field-selector=status.phase=Running --selector batch.kubernetes.io/job-name=spark-pi-s3-1 -o jsonpath='{.items[0].metadata.name}') + kubectl exec -n $NAMESPACE --container spark $DRIVER_POD -- cat /stackable/log/containerdebug-state.json | jq --exit-status '"valid JSON"' diff --git a/tests/templates/kuttl/smoke/52-assert.yaml b/tests/templates/kuttl/smoke/52-assert.yaml index 8404522b..dd15476d 100644 --- a/tests/templates/kuttl/smoke/52-assert.yaml +++ b/tests/templates/kuttl/smoke/52-assert.yaml @@ -15,7 +15,7 @@ metadata: app.kubernetes.io/role-group: sparkapplication stackable.tech/vendor: Stackable ownerReferences: - - apiVersion: spark.stackable.tech/v1alpha1 + - apiVersion: spark.stackable.tech/v1alpha2 controller: true kind: SparkApplication name: spark-pi-s3-1 @@ -32,24 +32,7 @@ metadata: app.kubernetes.io/role-group: sparkapplication stackable.tech/vendor: Stackable ownerReferences: - - apiVersion: spark.stackable.tech/v1alpha1 - controller: true - kind: SparkApplication - name: spark-pi-s3-1 ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: spark-pi-s3-1-submit-job - labels: - app.kubernetes.io/component: spark-submit - app.kubernetes.io/instance: spark-pi-s3-1 - app.kubernetes.io/managed-by: spark.stackable.tech_sparkapplication - app.kubernetes.io/name: spark-k8s - app.kubernetes.io/role-group: sparkapplication - stackable.tech/vendor: Stackable - ownerReferences: - - apiVersion: spark.stackable.tech/v1alpha1 + - apiVersion: spark.stackable.tech/v1alpha2 controller: true kind: SparkApplication name: spark-pi-s3-1 @@ -66,7 +49,7 @@ metadata: app.kubernetes.io/role-group: sparkapplication stackable.tech/vendor: Stackable ownerReferences: - - apiVersion: spark.stackable.tech/v1alpha1 + - apiVersion: spark.stackable.tech/v1alpha2 controller: true kind: SparkApplication name: spark-pi-s3-1 @@ -83,7 +66,7 @@ metadata: app.kubernetes.io/role-group: sparkapplication stackable.tech/vendor: Stackable ownerReferences: - - apiVersion: spark.stackable.tech/v1alpha1 + - apiVersion: spark.stackable.tech/v1alpha2 controller: true kind: SparkApplication name: spark-pi-s3-1 @@ -94,3 +77,23 @@ roleRef: subjects: - kind: ServiceAccount name: spark-pi-s3-1 +--- +# Headless Service that executors use to reach the driver in client mode. +apiVersion: v1 +kind: Service +metadata: + name: spark-pi-s3-1-driver + labels: + app.kubernetes.io/instance: spark-pi-s3-1 + app.kubernetes.io/managed-by: spark.stackable.tech_sparkapplication + app.kubernetes.io/name: spark-k8s + ownerReferences: + - apiVersion: spark.stackable.tech/v1alpha2 + controller: true + kind: SparkApplication + name: spark-pi-s3-1 +spec: + clusterIP: None + selector: + app.kubernetes.io/instance: spark-pi-s3-1 + spark-role: driver diff --git a/tests/templates/kuttl/smoke/53-assert.yaml.j2 b/tests/templates/kuttl/smoke/53-assert.yaml.j2 index 1d161cd5..673a7e79 100644 --- a/tests/templates/kuttl/smoke/53-assert.yaml.j2 +++ b/tests/templates/kuttl/smoke/53-assert.yaml.j2 @@ -1680,22 +1680,3 @@ commands: exit 1 fi rm -f "$expected_file" "$actual_file" - - script: | - expected=$(cat <<'YAMLEOF' | yq -o=json - security.properties: | - networkaddress.cache.negative.ttl=0 - networkaddress.cache.ttl=30 - spark-env.sh: "" - YAMLEOF - ) - actual=$(kubectl -n $NAMESPACE get cm spark-pi-s3-1-submit-job -o yaml | yq -o=json '.data') - expected_file=$(mktemp) && actual_file=$(mktemp) - printf '%s\n' "$expected" > "$expected_file" - printf '%s\n' "$actual" > "$actual_file" - if ! diff_out=$(diff -u "$expected_file" "$actual_file"); then - echo "ERROR: ConfigMap spark-pi-s3-1-submit-job data drifted from snapshot." - printf '%s\n' "$diff_out" - rm -f "$expected_file" "$actual_file" - exit 1 - fi - rm -f "$expected_file" "$actual_file" diff --git a/tests/test-definition.yaml b/tests/test-definition.yaml index 8fb06924..c7c790b2 100644 --- a/tests/test-definition.yaml +++ b/tests/test-definition.yaml @@ -5,14 +5,14 @@ dimensions: - "false" - name: spark values: - - 3.5.8 + # - 3.5.8 - 4.1.1 # Alternatively, if you want to use a custom image, append a comma and the full image name to the product version # as in the example below. # - 3.5.6,oci.stackable.tech/sandbox/spark-k8s:3.5.6-stackable0.0.0-dev - name: spark-logging values: - - 3.5.8 + # - 3.5.8 - 4.1.1 - name: spark-hbase-connector values: @@ -21,13 +21,13 @@ dimensions: # - 4.1.1 - name: spark-delta-lake values: - - 3.5.8 + # - 3.5.8 - 4.1.1 # - 3.5.6,oci.stackable.tech/sandbox/spark-k8s:3.5.6-stackable0.0.0-dev # Note: We keep this list, as new Spark versions are only supported delayed by Iceberg - name: spark-iceberg values: - - 3.5.8 + # - 3.5.8 - 4.1.1 - name: iceberg-latest values: @@ -37,7 +37,7 @@ dimensions: - 4.0.0 - name: spark-connect values: - - 3.5.8 + # - 3.5.8 - 4.1.1 # - 3.5.6,oci.stackable.tech/sandbox/spark-k8s:3.5.6-stackable0.0.0-dev - name: krb5 @@ -60,7 +60,7 @@ dimensions: - 0.3.0 - name: s3-use-tls values: - - "false" + # - "false" - "true" tests: - name: smoke From 997308f65254023cbc7a28aa3a06465a2e98bdf0 Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Mon, 29 Jun 2026 13:38:39 +0200 Subject: [PATCH 2/6] refactor: deprecate application spec.mode --- CHANGELOG.md | 2 +- extra/crds.yaml | 56 +++++++++---------- rust/operator-binary/src/crd/mod.rs | 7 ++- rust/operator-binary/src/crd/roles.rs | 5 +- .../src/crd/template_merger.rs | 2 +- 5 files changed, 40 insertions(+), 32 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9916fb7a..15142224 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,7 +33,7 @@ All notable changes to this project will be documented in this file. ### Deprecated - `SparkApplication`/`SparkApplicationTemplate` `spec.job` is deprecated and ignored since `v1alpha2` (renamed to `deprecatedJob` in that version). The driver `Job` is now built from `spec.driver` ([#711]). -- `SparkApplication`/`SparkApplicationTemplate` `spec.mode` is deprecated and ignored: the operator always runs the driver in client mode internally ([#711]). +- `SparkApplication`/`SparkApplicationTemplate` `spec.mode` is deprecated and ignored (renamed to `deprecatedMode` in `v1alpha2`): the operator always runs the driver in client mode internally. `mode` is now optional in `v1alpha1` as well ([#711]). [#674]: https://github.com/stackabletech/spark-k8s-operator/pull/674 [#679]: https://github.com/stackabletech/spark-k8s-operator/pull/679 diff --git a/extra/crds.yaml b/extra/crds.yaml index 12dc6cd2..c251fc5c 100644 --- a/extra/crds.yaml +++ b/extra/crds.yaml @@ -241,6 +241,19 @@ spec: type: object x-kubernetes-preserve-unknown-fields: true type: object + deprecatedMode: + default: cluster + description: |- + Deprecated and ignored. + + Since v1alpha2 the operator no longer runs a separate `spark-submit` process. The driver + is launched directly as a Kubernetes Job running in client mode, so the deploy mode is + always client internally. This field is kept for backwards compatibility but has no + effect. + enum: + - cluster + - client + type: string deps: default: excludePackages: [] @@ -1977,18 +1990,6 @@ spec: description: The main class - i.e. entry point - for JVM artifacts. nullable: true type: string - mode: - description: |- - Deprecated and ignored. - - Since v1alpha2 the operator no longer runs a separate `spark-submit` process. The driver - is launched directly as a Kubernetes Job running in client mode, so the deploy mode is - always client internally. This field is kept for backwards compatibility but has no - effect. - enum: - - cluster - - client - type: string s3connection: description: |- Configure an S3 connection that the SparkApplication has access to. @@ -2226,7 +2227,6 @@ spec: type: array required: - mainApplicationFile - - mode - sparkImage type: object status: @@ -4233,6 +4233,7 @@ spec: nullable: true type: string mode: + default: cluster description: |- Deprecated and ignored. @@ -4481,7 +4482,6 @@ spec: type: array required: - mainApplicationFile - - mode - sparkImage type: object status: @@ -7140,6 +7140,19 @@ spec: type: object x-kubernetes-preserve-unknown-fields: true type: object + deprecatedMode: + default: cluster + description: |- + Deprecated and ignored. + + Since v1alpha2 the operator no longer runs a separate `spark-submit` process. The driver + is launched directly as a Kubernetes Job running in client mode, so the deploy mode is + always client internally. This field is kept for backwards compatibility but has no + effect. + enum: + - cluster + - client + type: string deps: default: excludePackages: [] @@ -8876,18 +8889,6 @@ spec: description: The main class - i.e. entry point - for JVM artifacts. nullable: true type: string - mode: - description: |- - Deprecated and ignored. - - Since v1alpha2 the operator no longer runs a separate `spark-submit` process. The driver - is launched directly as a Kubernetes Job running in client mode, so the deploy mode is - always client internally. This field is kept for backwards compatibility but has no - effect. - enum: - - cluster - - client - type: string s3connection: description: |- Configure an S3 connection that the SparkApplication has access to. @@ -9125,7 +9126,6 @@ spec: type: array required: - mainApplicationFile - - mode - sparkImage type: object required: @@ -11096,6 +11096,7 @@ spec: nullable: true type: string mode: + default: cluster description: |- Deprecated and ignored. @@ -11344,7 +11345,6 @@ spec: type: array required: - mainApplicationFile - - mode - sparkImage type: object required: diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index 23c6eb60..22475783 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -222,7 +222,12 @@ pub mod versioned { /// is launched directly as a Kubernetes Job running in client mode, so the deploy mode is /// always client internally. This field is kept for backwards compatibility but has no /// effect. - pub mode: SparkMode, + #[versioned(deprecated( + since = "v1alpha2", + note = "the operator always runs the driver in client mode; this field is ignored" + ))] + #[serde(default)] + pub deprecated_mode: SparkMode, /// The main class - i.e. entry point - for JVM artifacts. #[serde(default, skip_serializing_if = "Option::is_none")] diff --git a/rust/operator-binary/src/crd/roles.rs b/rust/operator-binary/src/crd/roles.rs index 8f15ada9..e1640efd 100644 --- a/rust/operator-binary/src/crd/roles.rs +++ b/rust/operator-binary/src/crd/roles.rs @@ -88,10 +88,13 @@ pub enum SparkContainer { Vector, Tls, } -#[derive(Clone, Debug, Deserialize, Display, JsonSchema, PartialEq, Serialize)] +#[derive(Clone, Debug, Default, Deserialize, Display, JsonSchema, PartialEq, Serialize)] #[serde(rename_all = "lowercase")] #[strum(serialize_all = "lowercase")] pub enum SparkMode { + // Deprecated and ignored: the operator always runs the driver in client mode. The default only + // exists so the deprecated `mode` field can be optional. + #[default] Cluster, Client, } diff --git a/rust/operator-binary/src/crd/template_merger.rs b/rust/operator-binary/src/crd/template_merger.rs index 5ccee4f4..af9f234f 100644 --- a/rust/operator-binary/src/crd/template_merger.rs +++ b/rust/operator-binary/src/crd/template_merger.rs @@ -37,7 +37,7 @@ pub fn deep_merge(base: &SparkApplication, overlay: &SparkApplication) -> SparkA // Merge spec fields let spec = super::v1alpha2::SparkApplicationSpec { // Scalar fields: overlay takes precedence - mode: overlay.spec.mode.clone(), + deprecated_mode: overlay.spec.deprecated_mode.clone(), main_application_file: overlay.spec.main_application_file.clone(), // Option fields: overlay if Some, otherwise base From fdc2c5c36d7ea56563a158ad3300beb4d38d551f Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Mon, 29 Jun 2026 13:39:25 +0200 Subject: [PATCH 3/6] refactor: update kuttl tests --- .../kuttl/resources/10-assert.yaml.j2 | 2 +- .../resources/10-deploy-spark-app.yaml.j2 | 2 +- .../kuttl/resources/12-assert.yaml.j2 | 9 ++++---- .../resources/12-deploy-spark-app.yaml.j2 | 22 ++++++++++++++----- .../kuttl/smoke/50-deploy-spark-app.yaml.j2 | 6 +++++ .../kuttl/spark-examples/10-assert.yaml | 2 +- .../10-deploy-spark-app.yaml.j2 | 11 ++++++++++ 7 files changed, 40 insertions(+), 14 deletions(-) diff --git a/tests/templates/kuttl/resources/10-assert.yaml.j2 b/tests/templates/kuttl/resources/10-assert.yaml.j2 index 413daf5c..2c3cf10b 100644 --- a/tests/templates/kuttl/resources/10-assert.yaml.j2 +++ b/tests/templates/kuttl/resources/10-assert.yaml.j2 @@ -9,7 +9,7 @@ apiVersion: v1 kind: Pod metadata: labels: - app.kubernetes.io/instance: resources-crd + # app.kubernetes.io/instance: resources-crd spark-role: driver spec: containers: diff --git a/tests/templates/kuttl/resources/10-deploy-spark-app.yaml.j2 b/tests/templates/kuttl/resources/10-deploy-spark-app.yaml.j2 index f56bef9b..e2294269 100644 --- a/tests/templates/kuttl/resources/10-deploy-spark-app.yaml.j2 +++ b/tests/templates/kuttl/resources/10-deploy-spark-app.yaml.j2 @@ -20,7 +20,7 @@ spec: mainApplicationFile: "local:///stackable/spark/examples/jars/spark-examples.jar" sparkConf: spark.kubernetes.submission.waitAppCompletion: "false" - spark.kubernetes.driver.pod.name: "resources-crd-driver" + # spark.kubernetes.driver.pod.name: "resources-crd-driver" spark.kubernetes.executor.podNamePrefix: "resources-crd" job: config: diff --git a/tests/templates/kuttl/resources/12-assert.yaml.j2 b/tests/templates/kuttl/resources/12-assert.yaml.j2 index f5380433..2c7b57eb 100644 --- a/tests/templates/kuttl/resources/12-assert.yaml.j2 +++ b/tests/templates/kuttl/resources/12-assert.yaml.j2 @@ -9,7 +9,6 @@ apiVersion: v1 kind: Pod metadata: labels: - app.kubernetes.io/instance: resources-sparkconf spark-role: driver spec: containers: @@ -19,11 +18,11 @@ spec: - name: spark resources: limits: - cpu: "1" - memory: "1433Mi" + cpu: "2" + memory: "2Gi" requests: - cpu: "1" - memory: "1433Mi" + cpu: "500m" + memory: "2Gi" --- apiVersion: v1 kind: Pod diff --git a/tests/templates/kuttl/resources/12-deploy-spark-app.yaml.j2 b/tests/templates/kuttl/resources/12-deploy-spark-app.yaml.j2 index d2abc94b..1fa24563 100644 --- a/tests/templates/kuttl/resources/12-deploy-spark-app.yaml.j2 +++ b/tests/templates/kuttl/resources/12-deploy-spark-app.yaml.j2 @@ -20,13 +20,15 @@ spec: mainApplicationFile: "local:///stackable/spark/examples/jars/spark-examples.jar" sparkConf: spark.kubernetes.submission.waitAppCompletion: "false" - spark.kubernetes.driver.pod.name: "resources-sparkconf-driver" + # NOTE: Setting the driver pod name is not possible anymore starting with v1alpha2 + # spark.kubernetes.driver.pod.name: "resources-sparkconf-driver" spark.kubernetes.executor.podNamePrefix: "resources-sparkconf" - spark.kubernetes.driver.request.cores: "1" - spark.kubernetes.driver.limit.cores: "1" - spark.driver.cores: "1" - spark.driver.memory: "1g" - spark.driver.memoryOverheadFactor: "0.4" + # NOTE: Driver pod resources are NOT set via sparkConf. The operator runs the driver as a + # Kubernetes Job with spark-submit in client mode, so Spark never creates the driver pod and + # the spark.kubernetes.driver.{request,limit}.cores / spark.driver.memory properties are + # inert. The driver pod's resources come solely from spec.driver.config.resources below. + # Executor resources, on the other hand, ARE driven by sparkConf because the driver still + # asks Spark to create the executor pods. spark.kubernetes.executor.request.cores: "1" spark.kubernetes.executor.limit.cores: "2" spark.executor.cores: "2" @@ -35,6 +37,14 @@ spec: spark.executor.instances: "1" driver: config: + # Values intentionally differ from the operator defaults (250m/1, 1Gi) to prove that + # user-provided role config takes priority. min -> request, max -> limit. + resources: + cpu: + min: "500m" + max: "2" + memory: + limit: "2Gi" logging: enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} executor: diff --git a/tests/templates/kuttl/smoke/50-deploy-spark-app.yaml.j2 b/tests/templates/kuttl/smoke/50-deploy-spark-app.yaml.j2 index a5b4c0ed..f572e758 100644 --- a/tests/templates/kuttl/smoke/50-deploy-spark-app.yaml.j2 +++ b/tests/templates/kuttl/smoke/50-deploy-spark-app.yaml.j2 @@ -20,6 +20,12 @@ spec: # spark-submit loads the application jar to s3a://my-bucket and thus makes it # available to the driver and executors. mainApplicationFile: "/stackable/spark/examples/jars/spark-examples.jar" + # The argument is the number of "slices" (partitions). SparkPi runs one task + # per slice sequentially on the single executor, so a high value stretches the + # runtime to ~1-2 minutes. This gives the assert in 50-assert.yaml enough time + # to exec into the running driver pod and for containerdebug to start. + args: + - "15000" sparkConf: spark.kubernetes.file.upload.path: "s3a://my-bucket" s3connection: diff --git a/tests/templates/kuttl/spark-examples/10-assert.yaml b/tests/templates/kuttl/spark-examples/10-assert.yaml index ceebc449..1e46741a 100644 --- a/tests/templates/kuttl/spark-examples/10-assert.yaml +++ b/tests/templates/kuttl/spark-examples/10-assert.yaml @@ -4,7 +4,7 @@ kind: TestAssert timeout: 900 --- # The Job starting the whole process -apiVersion: spark.stackable.tech/v1alpha1 +apiVersion: spark.stackable.tech/v1alpha2 kind: SparkApplication metadata: name: spark-examples diff --git a/tests/templates/kuttl/spark-examples/10-deploy-spark-app.yaml.j2 b/tests/templates/kuttl/spark-examples/10-deploy-spark-app.yaml.j2 index 6c70fc0d..cd1a9efe 100644 --- a/tests/templates/kuttl/spark-examples/10-deploy-spark-app.yaml.j2 +++ b/tests/templates/kuttl/spark-examples/10-deploy-spark-app.yaml.j2 @@ -34,6 +34,17 @@ spec: config: logging: enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + # TODO: This is copied from the spec.job.config above because that is ignored in v1alpha2 + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 1 + preference: + matchExpressions: + - key: disktype + operator: In + values: + - ssd executor: replicas: 1 config: From 283dda8d9507f6a2e04bf78746b2a4039018a45a Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Mon, 29 Jun 2026 15:24:01 +0200 Subject: [PATCH 4/6] fix: regression * ensure pre and post hooks run in the driver pod by calling run-spark.sh from the image. * give driver pods a stable hostname '-driver' which now differs from the pod name but it keeps backward compatibility relevant for logging. --- rust/operator-binary/src/crd/mod.rs | 5 ++-- .../src/spark_k8s_controller.rs | 25 ++++++++++++++----- 2 files changed, 21 insertions(+), 9 deletions(-) diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index 22475783..3f86d92a 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -642,9 +642,8 @@ impl v1alpha2::SparkApplication { }; let mut submit_cmd = vec![ - format!( - "containerdebug --output={VOLUME_MOUNT_PATH_LOG}/containerdebug-state.json --loop &" - ), + // containerdebug is started in the background by the image entrypoint (run-spark.sh) via + // the `_STACKABLE_PRE_HOOK` env var, so it must not be repeated here. build_truststore_commands, "/stackable/spark/bin/spark-submit".to_string(), "--verbose".to_string(), diff --git a/rust/operator-binary/src/spark_k8s_controller.rs b/rust/operator-binary/src/spark_k8s_controller.rs index 3928c2d7..28cab6b3 100644 --- a/rust/operator-binary/src/spark_k8s_controller.rs +++ b/rust/operator-binary/src/spark_k8s_controller.rs @@ -503,10 +503,11 @@ fn pod_template( logdir: &Option, spark_image: &ResolvedProductImage, service_account: &ServiceAccount, - // When set, the spark container runs this command (the spark-submit client invocation). This is - // used for the driver pod, which the operator now launches directly as a Kubernetes Job. When - // `None` (executor pod template), the container relies on the image entrypoint, since Spark sets - // the command on the executor pods it creates. + // When set, the spark container runs this command (the spark-submit client invocation) as the + // container `args`, leaving `command` unset so the image entrypoint (run-spark.sh) is used. This + // is the driver pod, which the operator launches directly as a Kubernetes Job. When `None` + // (executor pod template), the container also relies on the image entrypoint, since Spark sets + // the args on the executor pods it creates. command: Option<&[String]>, ) -> Result { let container_name = SparkContainer::Spark.to_string(); @@ -543,14 +544,20 @@ fn pod_template( )); } - cb.command(vec![ + // We pass the spark-submit invocation as container `args` and leave `command` unset so the + // image entrypoint (run-spark.sh) runs it. The entrypoint evaluates `_STACKABLE_PRE_HOOK` + // (which starts containerdebug) before, and `_STACKABLE_POST_HOOK` (which writes the Vector + // shutdown file) after the spark-submit process exits. The latter is what lets the Vector + // agent terminate so the driver Job pod can complete. run-spark.sh delegates to Spark's + // entrypoint.sh, which runs our `/bin/bash -c ...` in pass-through mode. + cb.args(vec![ "/bin/bash".to_string(), "-x".to_string(), "-euo".to_string(), "pipefail".to_string(), "-c".to_string(), + command.join("\n"), ]) - .args(vec![command.join("\n")]) .add_env_var("SPARK_SUBMIT_OPTS", spark_submit_opts.join(" ")) // TODO: move this to the image .add_env_var("SPARK_CONF_DIR", "/stackable/spark/conf"); @@ -909,6 +916,12 @@ fn driver_job( // A Job's pod must declare a restart policy. if let Some(spec) = template.spec.as_mut() { spec.restart_policy = Some("Never".to_string()); + // Give the driver pod a stable hostname ending in `-driver`. The Vector agent reports this + // hostname as the `pod` field of every log event (see `log_schema.host_key: pod`), and log + // aggregation/monitoring identifies driver logs by that `-driver` suffix. Without this the + // Job controller's generated pod name (`-`) would be used, which Spark previously + // avoided by naming the cluster-mode driver pod `-...-driver`. + spec.hostname = Some(spark_application.driver_service_name()); } let job = Job { From 5212eae27ee774dba7fa68063cfcf4f1c5115e67 Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Mon, 29 Jun 2026 16:35:51 +0200 Subject: [PATCH 5/6] refactor: remove pod_driver_controller.rs --- CHANGELOG.md | 1 + .../spark-k8s-operator/templates/roles.yaml | 11 - rust/operator-binary/src/crd/constants.rs | 4 - rust/operator-binary/src/main.rs | 48 +--- .../src/pod_driver_controller.rs | 132 ----------- .../src/spark_k8s_controller.rs | 215 +++++++++++++++++- 6 files changed, 216 insertions(+), 195 deletions(-) delete mode 100644 rust/operator-binary/src/pod_driver_controller.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 15142224..da94cd69 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ All notable changes to this project will be documented in this file. - BREAKING: The operator no longer runs a separate `spark-submit` process for `SparkApplication`s. The driver is launched directly as a Kubernetes `Job` (built from `spec.driver`) running `spark-submit` in client mode; executors are still created by the driver via Spark's Kubernetes backend. A headless driver `Service` is created so executors can reach the driver. This affects both `v1alpha1` (after conversion) and `v1alpha2` objects ([#711]). - The driver `Job` is no longer retried on failure (`backoffLimit` is `0`); the previous `spec.job.retryOnFailureCount` is deprecated and ignored ([#711]). +- The `SparkApplication` status (`status.phase`) is now derived from the driver `Job` status by the application controller instead of a dedicated pod-driver controller watching driver pods. The pod-driver controller and the operator's `pods` RBAC permissions have been removed ([#711]). - Document Helm deployed RBAC permissions and remove unnecessary permissions ([#674]). - BREAKING: Each custom resource accepts now only the known config files in `configOverrides`: - `SparkApplication`: `spark-env.sh` and `security.properties` diff --git a/deploy/helm/spark-k8s-operator/templates/roles.yaml b/deploy/helm/spark-k8s-operator/templates/roles.yaml index 4804aade..14fb2799 100644 --- a/deploy/helm/spark-k8s-operator/templates/roles.yaml +++ b/deploy/helm/spark-k8s-operator/templates/roles.yaml @@ -13,17 +13,6 @@ rules: - nodes/proxy verbs: - get - # The pod-driver controller watches Spark driver pods (labelled spark-role=driver) to track - # SparkApplication completion. Driver pods are owned by the driver Job and cleaned up via the - # Job's ttlSecondsAfterFinished, so the operator no longer deletes them. - - apiGroups: - - "" - resources: - - pods - verbs: - - get - - list - - watch # ConfigMaps hold pod templates and Spark configuration. All three controllers apply # them via Server-Side Apply (create + patch). The history and connect controllers # track them for orphan cleanup (list + delete). All controllers watch ConfigMaps via diff --git a/rust/operator-binary/src/crd/constants.rs b/rust/operator-binary/src/crd/constants.rs index 3aa32218..c8ecf271 100644 --- a/rust/operator-binary/src/crd/constants.rs +++ b/rust/operator-binary/src/crd/constants.rs @@ -66,10 +66,6 @@ pub const FIELD_MANAGER: &str = "spark-operator"; pub const SPARK_CONTROLLER_NAME: &str = "sparkapplication"; pub const SPARK_FULL_CONTROLLER_NAME: &str = concatcp!(SPARK_CONTROLLER_NAME, '.', OPERATOR_NAME); -pub const POD_DRIVER_CONTROLLER_NAME: &str = "pod-driver"; -pub const POD_DRIVER_FULL_CONTROLLER_NAME: &str = - concatcp!(POD_DRIVER_CONTROLLER_NAME, '.', OPERATOR_NAME); - pub const HISTORY_CONTROLLER_NAME: &str = "history"; pub const HISTORY_FULL_CONTROLLER_NAME: &str = concatcp!(HISTORY_CONTROLLER_NAME, '.', OPERATOR_NAME); diff --git a/rust/operator-binary/src/main.rs b/rust/operator-binary/src/main.rs index 060c7b81..e8780a51 100644 --- a/rust/operator-binary/src/main.rs +++ b/rust/operator-binary/src/main.rs @@ -16,7 +16,7 @@ use stackable_operator::{ k8s_openapi::api::{ apps::v1::StatefulSet, batch::v1::Job, - core::v1::{ConfigMap, Pod, Service}, + core::v1::{ConfigMap, Service}, }, kube::{ CustomResourceExt as _, @@ -40,8 +40,7 @@ use crate::{ crd::{ SparkApplication, constants::{ - HISTORY_FULL_CONTROLLER_NAME, OPERATOR_NAME, POD_DRIVER_FULL_CONTROLLER_NAME, - SPARK_CONTROLLER_NAME, SPARK_FULL_CONTROLLER_NAME, + HISTORY_FULL_CONTROLLER_NAME, OPERATOR_NAME, SPARK_FULL_CONTROLLER_NAME, }, history::SparkHistoryServer, template_spec::{SparkApplicationTemplate, SparkApplicationTemplateVersion}, @@ -53,7 +52,6 @@ mod config; mod connect; mod crd; mod history; -mod pod_driver_controller; mod product_logging; mod spark_k8s_controller; mod webhooks; @@ -199,47 +197,6 @@ async fn main() -> anyhow::Result<()> { ) .map(anyhow::Ok); - let pod_driver_event_recorder = Arc::new(Recorder::new( - client.as_kube_client(), - Reporter { - controller: POD_DRIVER_FULL_CONTROLLER_NAME.to_string(), - instance: None, - }, - )); - let pod_driver_controller = Controller::new( - watch_namespace.get_api::>(&client), - watcher::Config::default() - .labels(&format!("app.kubernetes.io/managed-by={OPERATOR_NAME}_{SPARK_CONTROLLER_NAME},spark-role=driver")), - ) - .owns( - watch_namespace.get_api::>(&client), - watcher::Config::default(), - ) - .graceful_shutdown_on(sigterm_watcher.handle()) - .run( - pod_driver_controller::reconcile, - pod_driver_controller::error_policy, - Arc::new(client.clone()), - ) - .instrument(info_span!("pod_driver_controller")) - // We can let the reporting happen in the background - .for_each_concurrent( - 16, // concurrency limit - |result| { - // The event_recorder needs to be shared across all invocations, so that - // events are correctly aggregated - let pod_driver_event_recorder = pod_driver_event_recorder.clone(); - async move { - report_controller_reconciled( - &pod_driver_event_recorder, - POD_DRIVER_FULL_CONTROLLER_NAME, - &result, - ) - .await; - } - }, - ).map(anyhow::Ok); - // Create new object because Ctx cannot be cloned let ctx = Ctx { client: client.clone(), @@ -399,7 +356,6 @@ async fn main() -> anyhow::Result<()> { // kube-runtime's Controller will tokio::spawn each reconciliation, so this only concerns the internal watch machinery futures::try_join!( - pod_driver_controller, delayed_history_controller, delayed_connect_controller, delayed_app_controller, diff --git a/rust/operator-binary/src/pod_driver_controller.rs b/rust/operator-binary/src/pod_driver_controller.rs deleted file mode 100644 index 02940ea3..00000000 --- a/rust/operator-binary/src/pod_driver_controller.rs +++ /dev/null @@ -1,132 +0,0 @@ -use std::sync::Arc; - -use snafu::{OptionExt, ResultExt, Snafu}; -use stackable_operator::{ - client::Client, - k8s_openapi::api::core::v1::Pod, - kube::{ - core::{DeserializeGuard, error_boundary}, - runtime::controller::Action, - }, - logging::controller::ReconcilerError, - shared::time::Duration, -}; -use strum::{EnumDiscriminants, IntoStaticStr}; - -use crate::crd::{constants::POD_DRIVER_CONTROLLER_NAME, v1alpha2}; - -const LABEL_NAME_INSTANCE: &str = "app.kubernetes.io/instance"; - -#[derive(Snafu, Debug, EnumDiscriminants)] -#[strum_discriminants(derive(IntoStaticStr))] -#[allow(clippy::enum_variant_names, clippy::large_enum_variant)] -pub enum Error { - #[snafu(display("Label [{LABEL_NAME_INSTANCE}] not found for pod name [{pod_name}]"))] - LabelInstanceNotFound { pod_name: String }, - - #[snafu(display("Failed to update status for application [{name}]"))] - ApplySparkApplicationStatus { - source: stackable_operator::client::Error, - name: String, - }, - - #[snafu(display("Pod name not found"))] - PodNameNotFound, - - #[snafu(display("Namespace not found"))] - NamespaceNotFound, - - #[snafu(display("Status phase not found for pod [{pod_name}]"))] - PodStatusPhaseNotFound { pod_name: String }, - - #[snafu(display("Spark application [{name}] not found"))] - SparkApplicationNotFound { - source: stackable_operator::client::Error, - name: String, - }, - - #[snafu(display("Pod object is invalid"))] - InvalidPod { - source: error_boundary::InvalidObject, - }, -} - -type Result = std::result::Result; - -impl ReconcilerError for Error { - fn category(&self) -> &'static str { - ErrorDiscriminants::from(self).into() - } -} - -/// This function serves two purposes: -/// 1. It updates the status of the SparkApplication CR based on the status of the driver pod. -/// 2. It deletes the driver pod when the SparkApplication reaches a terminal state (Succeeded or Failed). -pub async fn reconcile(pod: Arc>, client: Arc) -> Result { - tracing::info!("Starting reconcile driver pod"); - - let pod = pod - .0 - .as_ref() - .map_err(error_boundary::InvalidObject::clone) - .context(InvalidPodSnafu)?; - - let pod_name = pod.metadata.name.as_ref().context(PodNameNotFoundSnafu)?; - let app_name = pod - .metadata - .labels - .as_ref() - .and_then(|l| l.get(&String::from(LABEL_NAME_INSTANCE))) - .context(LabelInstanceNotFoundSnafu { - pod_name: pod_name.clone(), - })?; - let phase = pod.status.as_ref().and_then(|s| s.phase.as_ref()).context( - PodStatusPhaseNotFoundSnafu { - pod_name: pod_name.clone(), - }, - )?; - - let app = client - .get::( - app_name.as_ref(), - pod.metadata - .namespace - .as_ref() - .context(NamespaceNotFoundSnafu)?, - ) - .await - .context(SparkApplicationNotFoundSnafu { - name: app_name.clone(), - })?; - - tracing::info!("Update spark application [{app_name}] status to [{phase}]"); - - client - .apply_patch_status( - POD_DRIVER_CONTROLLER_NAME, - &app, - &crate::crd::SparkApplicationStatus { - phase: phase.clone(), - resolved_template_ref: app - .status - .as_ref() - .map(|s| s.resolved_template_ref.clone()) - .unwrap_or_default(), - }, - ) - .await - .with_context(|_| ApplySparkApplicationStatusSnafu { - name: app_name.clone(), - })?; - - // The driver pod is now owned by the driver Job. We must not delete it ourselves: the Job's - // `ttlSecondsAfterFinished` cleans up the Job and its pod once the application has finished. - Ok(Action::await_change()) -} - -pub fn error_policy(_obj: Arc>, error: &Error, _ctx: Arc) -> Action { - match error { - Error::InvalidPod { .. } => Action::await_change(), - _ => Action::requeue(*Duration::from_secs(5)), - } -} diff --git a/rust/operator-binary/src/spark_k8s_controller.rs b/rust/operator-binary/src/spark_k8s_controller.rs index 28cab6b3..4f1a8caa 100644 --- a/rust/operator-binary/src/spark_k8s_controller.rs +++ b/rust/operator-binary/src/spark_k8s_controller.rs @@ -16,6 +16,7 @@ use stackable_operator::{ volume::VolumeBuilder, }, }, + client::Client, commons::product_image_selection::ResolvedProductImage, crd::s3, k8s_openapi::{ @@ -166,6 +167,15 @@ pub enum Error { InvalidSparkApplication { source: error_boundary::InvalidObject, }, + + #[snafu(display("SparkApplication [{name}] has no namespace"))] + SparkApplicationHasNoNamespace { name: String }, + + #[snafu(display("failed to get driver Job for SparkApplication [{name}]"))] + GetDriverJob { + source: stackable_operator::client::Error, + name: String, + }, } type Result = std::result::Result; @@ -190,12 +200,15 @@ pub async fn reconcile( let client = &ctx.client; + // Once the driver Job has been created, we must not create it again (see #457). On every + // subsequent reconcile - triggered by the owned driver Job's status changing - we instead + // derive the SparkApplication status from that Job and stop. if spark_application.k8s_job_has_been_created() { tracing::info!( spark_application = spark_application.name_any(), - "Skipped reconciling SparkApplication with non empty status" + "Updating SparkApplication status from driver Job" ); - return Ok(Action::await_change()); + return update_status_from_driver_job(client, spark_application).await; } // It is important to do this at the top of the reconciliation function to ensure @@ -338,6 +351,88 @@ pub async fn reconcile( Ok(Action::await_change()) } +/// Derives the SparkApplication status from its driver Job and patches it. +/// +/// The driver Job carries the same name as the SparkApplication. Once it has finished, Kubernetes +/// garbage collects it via `ttlSecondsAfterFinished`; from then on there is nothing left to derive a +/// status from, so we keep the last known (terminal) status. We must never (re)create the Job here +/// (see #457). +async fn update_status_from_driver_job( + client: &Client, + spark_application: &v1alpha2::SparkApplication, +) -> Result { + let name = spark_application.name_any(); + let namespace = + spark_application + .metadata + .namespace + .as_ref() + .context(SparkApplicationHasNoNamespaceSnafu { name: name.clone() })?; + + let Some(job) = client + .get_opt::(&name, namespace) + .await + .context(GetDriverJobSnafu { name: name.clone() })? + else { + // The driver Job was already garbage collected. Keep the last known status. + return Ok(Action::await_change()); + }; + + let phase = driver_job_phase(&job); + + client + .apply_patch_status( + SPARK_CONTROLLER_NAME, + spark_application, + &crate::crd::SparkApplicationStatus { + phase, + resolved_template_ref: spark_application + .status + .as_ref() + .map(|s| s.resolved_template_ref.clone()) + .unwrap_or_default(), + }, + ) + .await + .with_context(|_| ApplySparkApplicationStatusSnafu { name })?; + + Ok(Action::await_change()) +} + +/// Maps the driver Job's status to a SparkApplication phase. +/// +/// The phase values mirror the pod phases reported previously (`Running`, `Succeeded`, `Failed`, +/// `Unknown`) so that the externally visible status does not change. A Job's `active` count includes +/// both pending and running pods, so a scheduled-but-not-yet-running driver also reports `Running`. +fn driver_job_phase(job: &Job) -> String { + let Some(status) = job.status.as_ref() else { + return "Unknown".to_string(); + }; + + // Terminal conditions take precedence over the pod counters. + if let Some(conditions) = status.conditions.as_ref() { + for condition in conditions { + if condition.status == "True" { + match condition.type_.as_str() { + "Complete" | "SuccessCriteriaMet" => return "Succeeded".to_string(), + "Failed" => return "Failed".to_string(), + _ => {} + } + } + } + } + + if status.active.unwrap_or(0) > 0 { + "Running".to_string() + } else if status.succeeded.unwrap_or(0) > 0 { + "Succeeded".to_string() + } else if status.failed.unwrap_or(0) > 0 { + "Failed".to_string() + } else { + "Unknown".to_string() + } +} + fn init_containers( spark_application: &v1alpha2::SparkApplication, logging: &Logging, @@ -1019,3 +1114,119 @@ pub fn error_policy( _ => Action::requeue(*Duration::from_secs(5)), } } + +#[cfg(test)] +mod tests { + use stackable_operator::k8s_openapi::api::batch::v1::{Job, JobCondition, JobStatus}; + + use super::driver_job_phase; + + fn job_with_status(status: Option) -> Job { + Job { + status, + ..Job::default() + } + } + + fn condition(type_: &str, status: &str) -> JobCondition { + JobCondition { + type_: type_.to_string(), + status: status.to_string(), + ..JobCondition::default() + } + } + + #[test] + fn no_status_is_unknown() { + assert_eq!(driver_job_phase(&job_with_status(None)), "Unknown"); + } + + #[test] + fn empty_status_is_unknown() { + assert_eq!( + driver_job_phase(&job_with_status(Some(JobStatus::default()))), + "Unknown" + ); + } + + #[test] + fn active_pod_is_running() { + assert_eq!( + driver_job_phase(&job_with_status(Some(JobStatus { + active: Some(1), + ..JobStatus::default() + }))), + "Running" + ); + } + + #[test] + fn succeeded_counter_is_succeeded() { + assert_eq!( + driver_job_phase(&job_with_status(Some(JobStatus { + succeeded: Some(1), + ..JobStatus::default() + }))), + "Succeeded" + ); + } + + #[test] + fn failed_counter_is_failed() { + assert_eq!( + driver_job_phase(&job_with_status(Some(JobStatus { + failed: Some(1), + ..JobStatus::default() + }))), + "Failed" + ); + } + + #[test] + fn complete_condition_is_succeeded() { + assert_eq!( + driver_job_phase(&job_with_status(Some(JobStatus { + conditions: Some(vec![condition("Complete", "True")]), + ..JobStatus::default() + }))), + "Succeeded" + ); + } + + #[test] + fn failed_condition_is_failed() { + assert_eq!( + driver_job_phase(&job_with_status(Some(JobStatus { + conditions: Some(vec![condition("Failed", "True")]), + ..JobStatus::default() + }))), + "Failed" + ); + } + + #[test] + fn terminal_condition_wins_over_active_counter() { + // A Job can still report an active pod while its terminal condition is being set; the + // terminal condition must take precedence so we don't report "Running" for a finished Job. + assert_eq!( + driver_job_phase(&job_with_status(Some(JobStatus { + active: Some(1), + conditions: Some(vec![condition("Complete", "True")]), + ..JobStatus::default() + }))), + "Succeeded" + ); + } + + #[test] + fn condition_with_false_status_is_ignored() { + assert_eq!( + driver_job_phase(&job_with_status(Some(JobStatus { + active: Some(1), + conditions: Some(vec![condition("Failed", "False")]), + ..JobStatus::default() + }))), + "Running" + ); + } +} From 20e670033def223d0ecfe470dbc93b722a7b7ca3 Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Mon, 29 Jun 2026 16:38:36 +0200 Subject: [PATCH 6/6] chore: rustfmt --- rust/operator-binary/src/main.rs | 4 +--- rust/operator-binary/src/spark_k8s_controller.rs | 11 +++++------ 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/rust/operator-binary/src/main.rs b/rust/operator-binary/src/main.rs index e8780a51..b691a452 100644 --- a/rust/operator-binary/src/main.rs +++ b/rust/operator-binary/src/main.rs @@ -39,9 +39,7 @@ use crate::{ connect::crd::SparkConnectServerVersion, crd::{ SparkApplication, - constants::{ - HISTORY_FULL_CONTROLLER_NAME, OPERATOR_NAME, SPARK_FULL_CONTROLLER_NAME, - }, + constants::{HISTORY_FULL_CONTROLLER_NAME, OPERATOR_NAME, SPARK_FULL_CONTROLLER_NAME}, history::SparkHistoryServer, template_spec::{SparkApplicationTemplate, SparkApplicationTemplateVersion}, }, diff --git a/rust/operator-binary/src/spark_k8s_controller.rs b/rust/operator-binary/src/spark_k8s_controller.rs index 4f1a8caa..93067c91 100644 --- a/rust/operator-binary/src/spark_k8s_controller.rs +++ b/rust/operator-binary/src/spark_k8s_controller.rs @@ -362,12 +362,11 @@ async fn update_status_from_driver_job( spark_application: &v1alpha2::SparkApplication, ) -> Result { let name = spark_application.name_any(); - let namespace = - spark_application - .metadata - .namespace - .as_ref() - .context(SparkApplicationHasNoNamespaceSnafu { name: name.clone() })?; + let namespace = spark_application + .metadata + .namespace + .as_ref() + .context(SparkApplicationHasNoNamespaceSnafu { name: name.clone() })?; let Some(job) = client .get_opt::(&name, namespace)