diff --git a/api/v4/common_types.go b/api/v4/common_types.go index 689540b37..e53317075 100644 --- a/api/v4/common_types.go +++ b/api/v4/common_types.go @@ -95,11 +95,11 @@ type Spec struct { // +kubebuilder:validation:Enum=Always;IfNotPresent;Never // +kubebuilder:default=IfNotPresent // +optional - ImagePullPolicy string `json:"imagePullPolicy"` + ImagePullPolicy string `json:"imagePullPolicy,omitempty"` // Name of Scheduler to use for pod placement (defaults to “default-scheduler”) // +optional - SchedulerName string `json:"schedulerName"` + SchedulerName string `json:"schedulerName,omitempty"` // Kubernetes Affinity rules that control how pods are assigned to particular nodes. Affinity corev1.Affinity `json:"affinity"` @@ -168,15 +168,15 @@ type CommonSplunkSpec struct { // Storage configuration for /opt/splunk/etc volume // +optional - EtcVolumeStorageConfig StorageClassSpec `json:"etcVolumeStorageConfig"` + EtcVolumeStorageConfig StorageClassSpec `json:"etcVolumeStorageConfig,omitempty"` // Storage configuration for /opt/splunk/var volume // +optional - VarVolumeStorageConfig StorageClassSpec `json:"varVolumeStorageConfig"` + VarVolumeStorageConfig StorageClassSpec `json:"varVolumeStorageConfig,omitempty"` // List of one or more Kubernetes volumes. These will be mounted in all pod containers as as /mnt/ // +optional - Volumes []corev1.Volume `json:"volumes"` + Volumes []corev1.Volume `json:"volumes,omitempty"` // Inline map of default.yml overrides used to initialize the environment Defaults string `json:"defaults"` @@ -217,7 +217,7 @@ type CommonSplunkSpec struct { // If not specified uses the default serviceAccount for the namespace as per // https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/#use-the-default-service-account-to-access-the-api-server // +optional - ServiceAccount string `json:"serviceAccount"` + ServiceAccount string `json:"serviceAccount,omitempty"` // ExtraEnv refers to extra environment variables to be passed to the Splunk instance containers // WARNING: Setting environment variables used by Splunk or Ansible will affect Splunk installation and operation @@ -261,16 +261,16 @@ type CommonSplunkSpec struct { type StorageClassSpec struct { // Name of StorageClass to use for persistent volume claims // +optional - StorageClassName string `json:"storageClassName"` + StorageClassName string `json:"storageClassName,omitempty"` // Storage capacity to request persistent volume claims (default="10Gi" for etc and "100Gi" for var) // +optional - StorageCapacity string `json:"storageCapacity"` + StorageCapacity string `json:"storageCapacity,omitempty"` // If true, ephemeral (emptyDir) storage will be used // +optional // +kubebuilder:default=false - EphemeralStorage bool `json:"ephemeralStorage"` + EphemeralStorage bool `json:"ephemeralStorage,omitempty"` } // SmartStoreSpec defines Splunk indexes and remote storage volume configuration @@ -346,7 +346,7 @@ type VolumeSpec struct { // Secret object name // +optional - SecretRef string `json:"secretRef"` + SecretRef string `json:"secretRef,omitempty"` // Remote Storage type. Supported values: s3, blob, gcs. s3 works with aws or minio providers, whereas blob works with azure provider, gcs works for gcp. // +kubebuilder:validation:Enum=s3;blob;gcs @@ -355,11 +355,11 @@ type VolumeSpec struct { // App Package Remote Store provider. Supported values: aws, minio, azure, gcp. // +optional // +kubebuilder:validation:Enum=aws;minio;azure;gcp - Provider string `json:"provider"` + Provider string `json:"provider,omitempty"` // Region of the remote storage volume where apps reside. Required for aws, optional for azure and gcp. // +optional - Region string `json:"region"` + Region string `json:"region,omitempty"` } // VolumeAndTypeSpec used to add any custom variables for volume implementation diff --git a/api/v4/indexercluster_types.go b/api/v4/indexercluster_types.go index 1adaf2576..fe223dd01 100644 --- a/api/v4/indexercluster_types.go +++ b/api/v4/indexercluster_types.go @@ -43,11 +43,11 @@ type IndexerClusterSpec struct { // +optional // Queue reference - QueueRef corev1.ObjectReference `json:"queueRef"` + QueueRef corev1.ObjectReference `json:"queueRef,omitempty"` // +optional // Object Storage reference - ObjectStorageRef corev1.ObjectReference `json:"objectStorageRef"` + ObjectStorageRef corev1.ObjectReference `json:"objectStorageRef,omitempty"` // Number of indexer cluster peers Replicas int32 `json:"replicas"` diff --git a/api/v4/objectstorage_types.go b/api/v4/objectstorage_types.go index cfdda5a86..f67f68c19 100644 --- a/api/v4/objectstorage_types.go +++ b/api/v4/objectstorage_types.go @@ -44,7 +44,7 @@ type S3Spec struct { // +optional // +kubebuilder:validation:Pattern=`^https?://[^\s/$.?#].[^\s]*$` // S3-compatible Service endpoint - Endpoint string `json:"endpoint"` + Endpoint string `json:"endpoint,omitempty"` // +kubebuilder:validation:Required // +kubebuilder:validation:Pattern=`^(?:s3://)?[a-z0-9.-]{3,63}(?:/[^\s]+)?$` diff --git a/api/v4/queue_types.go b/api/v4/queue_types.go index b86bd23a9..742e61acb 100644 --- a/api/v4/queue_types.go +++ b/api/v4/queue_types.go @@ -49,7 +49,6 @@ type SQSSpec struct { // Name of the queue Name string `json:"name"` - // +optional // +kubebuilder:validation:Pattern=`^(?:us|ap|eu|me|af|sa|ca|cn|il)(?:-[a-z]+){1,3}-\d$` // Auth Region of the resources AuthRegion string `json:"authRegion"` @@ -62,11 +61,24 @@ type SQSSpec struct { // +optional // +kubebuilder:validation:Pattern=`^https?://[^\s/$.?#].[^\s]*$` // Amazon SQS Service endpoint - Endpoint string `json:"endpoint"` + Endpoint string `json:"endpoint,omitempty"` // +optional // List of remote storage volumes - VolList []VolumeSpec `json:"volumes,omitempty"` + VolList []SQSVolumeSpec `json:"volumes,omitempty"` +} + +// SQSVolumeSpec defines a volume reference for SQS queue authentication +type SQSVolumeSpec struct { + // Remote volume name + // +kubebuilder:validation:Required + // +kubebuilder:validation:MinLength=1 + Name string `json:"name"` + + // Remote volume secret ref + // +kubebuilder:validation:Required + // +kubebuilder:validation:MinLength=1 + SecretRef string `json:"secretRef"` } // QueueStatus defines the observed state of Queue diff --git a/api/v4/searchheadcluster_types.go b/api/v4/searchheadcluster_types.go index 26edf1402..120da05e4 100644 --- a/api/v4/searchheadcluster_types.go +++ b/api/v4/searchheadcluster_types.go @@ -42,7 +42,7 @@ type SearchHeadClusterSpec struct { // Number of search head pods; a search head cluster will be created if > 1 // +optional // +kubebuilder:default=3 - Replicas int32 `json:"replicas"` + Replicas int32 `json:"replicas,omitempty"` // Splunk Enterprise App repository. Specifies remote App location and scope for Splunk App management AppFrameworkConfig AppFrameworkSpec `json:"appRepo,omitempty"` diff --git a/api/v4/standalone_types.go b/api/v4/standalone_types.go index 6a3761b56..eef48c2b1 100644 --- a/api/v4/standalone_types.go +++ b/api/v4/standalone_types.go @@ -41,7 +41,7 @@ type StandaloneSpec struct { // Number of standalone pods // +optional // +kubebuilder:default=1 - Replicas int32 `json:"replicas"` + Replicas int32 `json:"replicas,omitempty"` //Splunk Smartstore configuration. Refer to indexes.conf.spec and server.conf.spec on docs.splunk.com // +optional diff --git a/api/v4/zz_generated.deepcopy.go b/api/v4/zz_generated.deepcopy.go index c7759fa58..7ae136536 100644 --- a/api/v4/zz_generated.deepcopy.go +++ b/api/v4/zz_generated.deepcopy.go @@ -1086,7 +1086,7 @@ func (in *SQSSpec) DeepCopyInto(out *SQSSpec) { *out = *in if in.VolList != nil { in, out := &in.VolList, &out.VolList - *out = make([]VolumeSpec, len(*in)) + *out = make([]SQSVolumeSpec, len(*in)) copy(*out, *in) } } @@ -1101,6 +1101,21 @@ func (in *SQSSpec) DeepCopy() *SQSSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SQSVolumeSpec) DeepCopyInto(out *SQSVolumeSpec) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SQSVolumeSpec. +func (in *SQSVolumeSpec) DeepCopy() *SQSVolumeSpec { + if in == nil { + return nil + } + out := new(SQSVolumeSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *SearchHeadCluster) DeepCopyInto(out *SearchHeadCluster) { *out = *in diff --git a/config/crd/bases/enterprise.splunk.com_queues.yaml b/config/crd/bases/enterprise.splunk.com_queues.yaml index 5bf76f967..ec848987a 100644 --- a/config/crd/bases/enterprise.splunk.com_queues.yaml +++ b/config/crd/bases/enterprise.splunk.com_queues.yaml @@ -82,53 +82,21 @@ spec: volumes: description: List of remote storage volumes items: - description: VolumeSpec defines remote volume config + description: SQSVolumeSpec defines a volume reference for SQS + queue authentication properties: - endpoint: - description: Remote volume URI - minLength: 1 - type: string name: description: Remote volume name minLength: 1 type: string - path: - description: Remote volume path - minLength: 1 - type: string - provider: - description: 'App Package Remote Store provider. Supported - values: aws, minio, azure, gcp.' - enum: - - aws - - minio - - azure - - gcp - type: string - region: - description: Region of the remote storage volume where apps - reside. Required for aws, optional for azure and gcp. - type: string secretRef: - description: Secret object name - type: string - storageType: - description: 'Remote Storage type. Supported values: s3, - blob, gcs. s3 works with aws or minio providers, whereas - blob works with azure provider, gcs works for gcp.' - enum: - - s3 - - blob - - gcs + description: Remote volume secret ref + minLength: 1 type: string required: - - endpoint - name - - path + - secretRef type: object - x-kubernetes-validations: - - message: region is required when provider is aws - rule: self.provider != 'aws' || size(self.region) > 0 type: array required: - dlq diff --git a/config/samples/enterprise_v4_objectstorage.yaml b/config/samples/enterprise_v4_objectstorage.yaml index b693a14e0..75088e51c 100644 --- a/config/samples/enterprise_v4_objectstorage.yaml +++ b/config/samples/enterprise_v4_objectstorage.yaml @@ -4,5 +4,8 @@ metadata: name: objectstorage-sample finalizers: - "enterprise.splunk.com/delete-pvc" -spec: {} +spec: + provider: s3 + s3: + path: bucket/path # TODO(user): Add fields here diff --git a/config/samples/enterprise_v4_queue.yaml b/config/samples/enterprise_v4_queue.yaml index 374d4adb2..ae26a713f 100644 --- a/config/samples/enterprise_v4_queue.yaml +++ b/config/samples/enterprise_v4_queue.yaml @@ -4,5 +4,10 @@ metadata: name: queue-sample finalizers: - "enterprise.splunk.com/delete-pvc" -spec: {} +spec: + provider: sqs + sqs: + name: queue + authRegion: us-west-2 + dlq: dlq # TODO(user): Add fields here diff --git a/helm-chart/splunk-enterprise/templates/enterprise_v4_queues.yaml b/helm-chart/splunk-enterprise/templates/enterprise_v4_queues.yaml index 06a3c5dbd..27821d648 100644 --- a/helm-chart/splunk-enterprise/templates/enterprise_v4_queues.yaml +++ b/helm-chart/splunk-enterprise/templates/enterprise_v4_queues.yaml @@ -31,7 +31,7 @@ spec: {{- end }} {{- if .volumes }} volumes: - {{ toYaml . | indent 4 }} +{{ toYaml .volumes | nindent 4 }} {{- end }} {{- end }} {{- end }} diff --git a/kuttl/tests/helm/c3-with-apps-private-link/c3_config.yaml b/kuttl/tests/helm/c3-with-apps-private-link/c3_config.yaml index 53c3adc63..87227fa81 100644 --- a/kuttl/tests/helm/c3-with-apps-private-link/c3_config.yaml +++ b/kuttl/tests/helm/c3-with-apps-private-link/c3_config.yaml @@ -32,7 +32,7 @@ clusterManager: storageType: s3 provider: aws path: helm-test-c3-with-apps-testnew - endpoint: www.amazon.com + endpoint: https://s3-us-west-2.amazonaws.com region: us-west-2 secretRef: s3-secret smartstore: @@ -48,8 +48,10 @@ clusterManager: volumes: - name: helm-test-c3-with-apps-smartstore-testnew path: helm-test-c3-with-apps-smartstore-testnew/smartstore - endpoint: www.amazon.com + endpoint: https://s3-us-west-2.amazonaws.com secretRef: s3-secret + storageType: s3 + provider: aws region: us-west-2 cacheManager: hotlistBloomFilterRecencyHours: 360 # 15 days - splunk defaults - (can be lowered to 5 days) @@ -74,6 +76,6 @@ searchHeadCluster: storageType: s3 provider: aws path: helm-test-c3-with-apps-testnew - endpoint: www.amazon.com + endpoint: https://s3-us-west-2.amazonaws.com region: us-west-2 secretRef: s3-secret diff --git a/kuttl/tests/helm/c3-with-apps/c3_config.yaml b/kuttl/tests/helm/c3-with-apps/c3_config.yaml index 622f61ea3..2d570f521 100644 --- a/kuttl/tests/helm/c3-with-apps/c3_config.yaml +++ b/kuttl/tests/helm/c3-with-apps/c3_config.yaml @@ -51,6 +51,9 @@ clusterManager: path: helm-test-c3-with-apps-smartstore-test/smartstore endpoint: https://s3-us-west-2.amazonaws.com secretRef: s3-secret + storageType: s3 + provider: aws + region: us-west-2 cacheManager: hotlistBloomFilterRecencyHours: 360 # 15 days - splunk defaults - (can be lowered to 5 days) hotlistRecencySecs: 86400 # 24 hours - splunk defaults diff --git a/kuttl/tests/helm/s1-with-smartstore/s1_config.yaml b/kuttl/tests/helm/s1-with-smartstore/s1_config.yaml index 9045f896b..5fc781eae 100644 --- a/kuttl/tests/helm/s1-with-smartstore/s1_config.yaml +++ b/kuttl/tests/helm/s1-with-smartstore/s1_config.yaml @@ -23,6 +23,9 @@ standalone: path: helm-test-c3-with-apps-smartstore-test/smartstore endpoint: https://s3-us-west-2.amazonaws.com secretRef: s3-secret + storageType: s3 + provider: aws + region: us-west-2 cacheManager: hotlistBloomFilterRecencyHours: 360 # 15 days - splunk defaults - (can be lowered to 5 days) hotlistRecencySecs: 86400 # 24 hours - splunk defaults diff --git a/pkg/splunk/enterprise/indexercluster_test.go b/pkg/splunk/enterprise/indexercluster_test.go index ad97ab2e8..a628a5619 100644 --- a/pkg/splunk/enterprise/indexercluster_test.go +++ b/pkg/splunk/enterprise/indexercluster_test.go @@ -2118,7 +2118,7 @@ func TestGetQueueAndPipelineInputsForIndexerConfFiles(t *testing.T) { AuthRegion: "us-west-2", Endpoint: "https://sqs.us-west-2.amazonaws.com", DLQ: "sqs-dlq-test", - VolList: []enterpriseApi.VolumeSpec{ + VolList: []enterpriseApi.SQSVolumeSpec{ {SecretRef: "secret"}, }, }, @@ -2204,7 +2204,7 @@ func TestGetQueueAndPipelineInputsForIndexerConfFilesSQSCP(t *testing.T) { AuthRegion: "us-west-2", Endpoint: "https://sqs.us-west-2.amazonaws.com", DLQ: "sqs-dlq-test", - VolList: []enterpriseApi.VolumeSpec{ + VolList: []enterpriseApi.SQSVolumeSpec{ {SecretRef: "secret"}, }, }, diff --git a/pkg/splunk/enterprise/ingestorcluster_test.go b/pkg/splunk/enterprise/ingestorcluster_test.go index e96002372..4cef74f2b 100644 --- a/pkg/splunk/enterprise/ingestorcluster_test.go +++ b/pkg/splunk/enterprise/ingestorcluster_test.go @@ -422,7 +422,7 @@ func TestGetQueueAndPipelineInputsForIngestorConfFiles(t *testing.T) { AuthRegion: "us-west-2", Endpoint: "https://sqs.us-west-2.amazonaws.com", DLQ: "sqs-dlq-test", - VolList: []enterpriseApi.VolumeSpec{ + VolList: []enterpriseApi.SQSVolumeSpec{ {SecretRef: "secret"}, }, }, @@ -496,7 +496,7 @@ func TestGetQueueAndPipelineInputsForIngestorConfFilesSQSCP(t *testing.T) { AuthRegion: "us-west-2", Endpoint: "https://sqs.us-west-2.amazonaws.com", DLQ: "sqs-dlq-test", - VolList: []enterpriseApi.VolumeSpec{ + VolList: []enterpriseApi.SQSVolumeSpec{ {SecretRef: "secret"}, }, }, diff --git a/pkg/splunk/enterprise/util.go b/pkg/splunk/enterprise/util.go index cc48f69a7..fd7900787 100644 --- a/pkg/splunk/enterprise/util.go +++ b/pkg/splunk/enterprise/util.go @@ -455,7 +455,7 @@ func GetSmartstoreRemoteVolumeSecrets(ctx context.Context, volume enterpriseApi. } // GetQueueRemoteVolumeSecrets is used to retrieve access key and secrete key for Index & Ingestion separation -func GetQueueRemoteVolumeSecrets(ctx context.Context, volume enterpriseApi.VolumeSpec, client splcommon.ControllerClient, cr splcommon.MetaObject) (string, string, string, error) { +func GetQueueRemoteVolumeSecrets(ctx context.Context, volume enterpriseApi.SQSVolumeSpec, client splcommon.ControllerClient, cr splcommon.MetaObject) (string, string, string, error) { namespaceScopedSecret, err := splutil.GetSecretByName(ctx, client, cr.GetNamespace(), cr.GetName(), volume.SecretRef) if err != nil { return "", "", "", err diff --git a/test/index_and_ingestion_separation/index_and_ingestion_separation_test.go b/test/index_and_ingestion_separation/index_and_ingestion_separation_test.go index 85c7de276..d88a05211 100644 --- a/test/index_and_ingestion_separation/index_and_ingestion_separation_test.go +++ b/test/index_and_ingestion_separation/index_and_ingestion_separation_test.go @@ -81,7 +81,10 @@ var _ = Describe("indingsep test", func() { // testcaseEnvInst.CreateServiceAccount(serviceAccountName) // Secret reference - volumeSpec := []enterpriseApi.VolumeSpec{testenv.GenerateQueueVolumeSpec("queue-secret-ref-volume", testcaseEnvInst.GetIndexIngestSepSecretName())} + volumeSpec := []enterpriseApi.SQSVolumeSpec{testenv.GenerateQueueVolumeSpec( + "queue-secret-ref-volume", + testcaseEnvInst.GetIndexIngestSepSecretName(), + )} queue.SQS.VolList = volumeSpec // Deploy Queue @@ -159,7 +162,10 @@ var _ = Describe("indingsep test", func() { // testcaseEnvInst.CreateServiceAccount(serviceAccountName) // Secret reference - volumeSpec := []enterpriseApi.VolumeSpec{testenv.GenerateQueueVolumeSpec("queue-secret-ref-volume", testcaseEnvInst.GetIndexIngestSepSecretName())} + volumeSpec := []enterpriseApi.SQSVolumeSpec{testenv.GenerateQueueVolumeSpec( + "queue-secret-ref-volume", + testcaseEnvInst.GetIndexIngestSepSecretName(), + )} queue.SQS.VolList = volumeSpec // Deploy Queue @@ -269,7 +275,10 @@ var _ = Describe("indingsep test", func() { // testcaseEnvInst.CreateServiceAccount(serviceAccountName) // Secret reference - volumeSpec := []enterpriseApi.VolumeSpec{testenv.GenerateQueueVolumeSpec("queue-secret-ref-volume", testcaseEnvInst.GetIndexIngestSepSecretName())} + volumeSpec := []enterpriseApi.SQSVolumeSpec{testenv.GenerateQueueVolumeSpec( + "queue-secret-ref-volume", + testcaseEnvInst.GetIndexIngestSepSecretName(), + )} queue.SQS.VolList = volumeSpec // Deploy Queue diff --git a/test/testenv/remote_index_utils.go b/test/testenv/remote_index_utils.go index f696a4a17..551a377d0 100644 --- a/test/testenv/remote_index_utils.go +++ b/test/testenv/remote_index_utils.go @@ -86,10 +86,10 @@ func RollHotToWarm(ctx context.Context, deployment *Deployment, podName string, return true } -// GenerateQueueVolumeSpec return VolumeSpec struct with given values -func GenerateQueueVolumeSpec(name, secretRef string) enterpriseApi.VolumeSpec { - return enterpriseApi.VolumeSpec{ - Name: name, +// GenerateQueueVolumeSpec return SQSVolumeSpec struct with given values +func GenerateQueueVolumeSpec(volumeName string, secretRef string) enterpriseApi.SQSVolumeSpec { + return enterpriseApi.SQSVolumeSpec{ + Name: volumeName, SecretRef: secretRef, } }