test(backup-restore): Add the option to disable TLS in S3

tests/templates/kuttl/backup-restore/11_minio-values.yaml → tests/templates/kuttl/backup-restore/11_minio-values.yaml.j2

@@ -16,7 +16,7 @@ console:
     repository: bitnamilegacy/minio-object-browser
 
 tls:
-  enabled: true
+  enabled: {{ test_scenario['values']['s3-use-tls'] }}
 
 persistence:
   enabled: false # "false" means, that an emptyDir is used instead of a persistentVolumeClaim

tests/templates/kuttl/backup-restore/21-install-opensearch-1.yaml.j2

@@ -45,13 +45,19 @@ spec:
         plugins.security.authcz.admin_dn: CN=opensearch-1-admin-certificate
         plugins.security.restapi.roles_enabled: all_access
         plugins.security.ssl.http.pemtrustedcas_filepath: /stackable/opensearch/config/tls/concatenated/ca.crt
+{% if test_scenario['values']['s3-use-tls'] == 'true' %}
         s3.client.default.endpoint: https://minio:9000/
         s3.client.default.protocol: https
+{% else %}
+        s3.client.default.endpoint: http://minio:9000/
+        s3.client.default.protocol: http
+{% endif %}
         s3.client.default.region: unused # but required
         s3.client.default.path_style_access: "true"
     podOverrides:
       spec:
         initContainers:
+{% if test_scenario['values']['s3-use-tls'] == 'true' %}
           - name: init-system-keystore
 {% if test_scenario['values']['opensearch'].find(",") > 0 %}
             image: "{{ test_scenario['values']['opensearch'].split(',')[1] }}"
@@ -72,6 +78,7 @@ spec:
                 mountPath: /etc/pki/ca-trust/source/anchors/s3-ca.crt
                 subPath: tls.crt
                 readOnly: true
+{% endif %}
           - name: init-tls
 {% if test_scenario['values']['opensearch'].find(",") > 0 %}
             image: "{{ test_scenario['values']['opensearch'].split(',')[1] }}"
@@ -106,18 +113,16 @@ spec:
               - name: security-config
                 mountPath: /stackable/opensearch/config/opensearch-security
                 readOnly: true
+{% if test_scenario['values']['s3-use-tls'] == 'true' %}
               - name: system-trust-store
                 mountPath: /etc/pki/java/cacerts
                 subPath: java/cacerts
                 readOnly: true
+{% endif %}
               - name: tls-concatenated
                 mountPath: /stackable/opensearch/config/tls/concatenated
                 readOnly: true
         volumes:
-          - name: s3-ca-crt
-            secret:
-              secretName: minio-ca-crt
-              defaultMode: 0o660
           - name: admin-certificate
             secret:
               secretName: opensearch-1-admin-certificate
@@ -126,9 +131,15 @@ spec:
             secret:
               secretName: opensearch-1-security-config
               defaultMode: 0o660
+{% if test_scenario['values']['s3-use-tls'] == 'true' %}
+          - name: s3-ca-crt
+            secret:
+              secretName: minio-ca-crt
+              defaultMode: 0o660
           - name: system-trust-store
             emptyDir:
               sizeLimit: 10Mi
+{% endif %}
           - name: tls-concatenated
             emptyDir:
               sizeLimit: 1Mi

tests/templates/kuttl/backup-restore/30-create-snapshot.yaml

@@ -88,7 +88,10 @@ data:
       body={
         "type": "s3",
         "settings": {
-          "bucket": "opensearch-data"
+          "bucket": "opensearch-data",
+          # The S3CrtClient that was introduced in OpenSearch 3.3.0, does not
+          # work with a TLS-secured MinIO. Use the old Netty client instead.
+          "s3_async_client_type": "netty"
         }
       }
     )

tests/templates/kuttl/backup-restore/31-backup-security-indices.yaml.j2

@@ -8,7 +8,11 @@ spec:
     spec:
       initContainers:
         - name: backup-security-indices
+{% if test_scenario['values']['opensearch'].find(",") > 0 %}
+          image: "{{ test_scenario['values']['opensearch'].split(',')[1] }}"
+{% else %}
           image: oci.stackable.tech/sdp/opensearch:{{ test_scenario['values']['opensearch'].split(',')[0] }}-stackable{{ test_scenario['values']['release'] }}
+{% endif %}
           command:
             - /stackable/scripts/backup-security-indices.sh
           env:
@@ -62,8 +66,13 @@ spec:
           volumeMounts:
             - name: scripts
               mountPath: /root/scripts
+{% if test_scenario['values']['s3-use-tls'] == 'true' %}
             - name: s3-ca-crt
               mountPath: /root/.mc/certs/CAs
+{% else %}
+            - name: mc-config
+              mountPath: /root/.mc
+{% endif %}
             - name: backup
               mountPath: /tmp/backup
           securityContext:
@@ -92,10 +101,16 @@ spec:
           secret:
             secretName: opensearch-1-admin-certificate
             defaultMode: 0o660
+{% if test_scenario['values']['s3-use-tls'] == 'true' %}
         - name: s3-ca-crt
           secret:
             secretName: minio-ca-crt
             defaultMode: 0o660
+{% else %}
+        - name: mc-config
+          emptyDir:
+            sizeLimit: 1Mi
+{% endif %}
         - name: backup
           emptyDir:
             sizeLimit: 1Mi
@@ -121,6 +136,11 @@ data:
   upload-security-indices-backup.sh: |
     #!/usr/bin/env sh
 
-    mc alias set minio https://minio:9000/ $S3_ACCESS_KEY $S3_SECRET_KEY
+{% if test_scenario['values']['s3-use-tls'] == 'true' %}
+    S3_PROTOCOL=https
+{% else %}
+    S3_PROTOCOL=http
+{% endif %}
+    mc alias set minio $S3_PROTOCOL://minio:9000/ $S3_ACCESS_KEY $S3_SECRET_KEY
     mc mb minio/opensearch-security
     mc cp /tmp/backup/* minio/opensearch-security/

tests/templates/kuttl/backup-restore/51-install-opensearch-2.yaml.j2

@@ -45,13 +45,19 @@ spec:
         plugins.security.authcz.admin_dn: CN=opensearch-2-admin-certificate
         plugins.security.restapi.roles_enabled: all_access
         plugins.security.ssl.http.pemtrustedcas_filepath: /stackable/opensearch/config/tls/concatenated/ca.crt
+{% if test_scenario['values']['s3-use-tls'] == 'true' %}
         s3.client.default.endpoint: https://minio:9000/
         s3.client.default.protocol: https
+{% else %}
+        s3.client.default.endpoint: http://minio:9000/
+        s3.client.default.protocol: http
+{% endif %}
         s3.client.default.region: unused # but required
         s3.client.default.path_style_access: "true"
     podOverrides:
       spec:
         initContainers:
+{% if test_scenario['values']['s3-use-tls'] == 'true' %}
           - name: init-system-keystore
 {% if test_scenario['values']['opensearch'].find(",") > 0 %}
             image: "{{ test_scenario['values']['opensearch'].split(',')[1] }}"
@@ -72,6 +78,7 @@ spec:
                 mountPath: /etc/pki/ca-trust/source/anchors/s3-ca.crt
                 subPath: tls.crt
                 readOnly: true
+{% endif %}
           - name: init-tls
 {% if test_scenario['values']['opensearch'].find(",") > 0 %}
             image: "{{ test_scenario['values']['opensearch'].split(',')[1] }}"
@@ -106,18 +113,16 @@ spec:
               - name: security-config
                 mountPath: /stackable/opensearch/config/opensearch-security
                 readOnly: true
+{% if test_scenario['values']['s3-use-tls'] == 'true' %}
               - name: system-trust-store
                 mountPath: /etc/pki/java/cacerts
                 subPath: java/cacerts
                 readOnly: true
+{% endif %}
               - name: tls-concatenated
                 mountPath: /stackable/opensearch/config/tls/concatenated
                 readOnly: true
         volumes:
-          - name: s3-ca-crt
-            secret:
-              secretName: minio-ca-crt
-              defaultMode: 0o660
           - name: admin-certificate
             secret:
               secretName: opensearch-2-admin-certificate
@@ -126,9 +131,15 @@ spec:
             secret:
               secretName: opensearch-2-security-config
               defaultMode: 0o660
+{% if test_scenario['values']['s3-use-tls'] == 'true' %}
+          - name: s3-ca-crt
+            secret:
+              secretName: minio-ca-crt
+              defaultMode: 0o660
           - name: system-trust-store
             emptyDir:
               sizeLimit: 10Mi
+{% endif %}
           - name: tls-concatenated
             emptyDir:
               sizeLimit: 1Mi

tests/templates/kuttl/backup-restore/60-restore-security-indices.yaml.j2

@@ -25,8 +25,13 @@ spec:
           volumeMounts:
             - name: scripts
               mountPath: /root/scripts
+{% if test_scenario['values']['s3-use-tls'] == 'true' %}
             - name: s3-ca-crt
               mountPath: /root/.mc/certs/CAs
+{% else %}
+            - name: mc-config
+              mountPath: /root/.mc
+{% endif %}
             - name: backup
               mountPath: /tmp/backup
           securityContext:
@@ -45,7 +50,11 @@ spec:
               cpu: 400m
       containers:
         - name: restore-security-indices
+{% if test_scenario['values']['opensearch'].find(",") > 0 %}
+          image: "{{ test_scenario['values']['opensearch'].split(',')[1] }}"
+{% else %}
           image: oci.stackable.tech/sdp/opensearch:{{ test_scenario['values']['opensearch'].split(',')[0] }}-stackable{{ test_scenario['values']['release'] }}
+{% endif %}
           command:
             - /stackable/scripts/restore-security-indices.sh
           env:
@@ -92,10 +101,16 @@ spec:
           secret:
             secretName: opensearch-2-admin-certificate
             defaultMode: 0o660
+{% if test_scenario['values']['s3-use-tls'] == 'true' %}
         - name: s3-ca-crt
           secret:
             secretName: minio-ca-crt
             defaultMode: 0o660
+{% else %}
+        - name: mc-config
+          emptyDir:
+            sizeLimit: 1Mi
+{% endif %}
         - name: backup
           emptyDir:
             sizeLimit: 1Mi
@@ -112,7 +127,12 @@ data:
   download-security-indices-backup.sh: |
     #!/usr/bin/env sh
 
-    mc alias set minio https://minio:9000/ $S3_ACCESS_KEY $S3_SECRET_KEY
+{% if test_scenario['values']['s3-use-tls'] == 'true' %}
+    S3_PROTOCOL=https
+{% else %}
+    S3_PROTOCOL=http
+{% endif %}
+    mc alias set minio $S3_PROTOCOL://minio:9000/ $S3_ACCESS_KEY $S3_SECRET_KEY
     mc cp --recursive minio/opensearch-security/ /tmp/backup
   restore-security-indices.sh: |
     #!/usr/bin/env sh

tests/templates/kuttl/backup-restore/61-restore-snapshot.yaml

@@ -88,7 +88,10 @@ data:
       body={
         "type": "s3",
         "settings": {
-          "bucket": "opensearch-data"
+          "bucket": "opensearch-data",
+          # The S3CrtClient that was introduced in OpenSearch 3.3.0, does not
+          # work with a TLS-secured MinIO. Use the old Netty client instead.
+          "s3_async_client_type": "netty"
         }
       }
     )

tests/test-definition.yaml

@@ -13,6 +13,9 @@ dimensions:
     values:
       - "true"
       - "false"
+  - name: s3-use-tls
+    values:
+      - "true"
   # The release must sometimes be known in podOverrides or Helm values
   - name: release
     values:
@@ -50,6 +53,7 @@ tests:
     dimensions:
       - opensearch
       - release
+      - s3-use-tls
 suites:
   - name: nightly
     patch: