Add SSE-C encryption config

Author: rdettai-sk

quickwit/quickwit-config/src/lib.rs

@@ -80,7 +80,8 @@ pub use crate::node_config::{
 use crate::source_config::serialize::{SourceConfigV0_7, SourceConfigV0_8, VersionedSourceConfig};
 pub use crate::storage_config::{
     AzureStorageConfig, FileStorageConfig, GoogleCloudStorageConfig, RamStorageConfig,
-    S3StorageConfig, StorageBackend, StorageBackendFlavor, StorageConfig, StorageConfigs,
+    S3EncryptionConfig, S3StorageConfig, StorageBackend, StorageBackendFlavor, StorageConfig,
+    StorageConfigs,
 };
 
 /// Returns true if the ingest API v2 is enabled.

quickwit/quickwit-config/src/storage_config.rs

@@ -308,6 +308,29 @@ impl fmt::Debug for AzureStorageConfig {
     }
 }
 
+#[derive(Clone, Eq, PartialEq, Serialize, Deserialize)]
+#[serde(tag = "type", rename_all = "snake_case")]
+pub enum S3EncryptionConfig {
+    /// This is the standard AES256 SSE-C header config. Key is expected to be a
+    /// 256bit base64-encoded string, and key_md5 is expected to be the
+    /// base64-encoded MD5 digest of the (binary) key. Akamai gen1 buckets don't
+    /// respect this (only the a 32 hex char key is expected).
+    SseC { key: String, key_md5: String },
+}
+
+impl fmt::Debug for S3EncryptionConfig {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            S3EncryptionConfig::SseC { key_md5, .. } => f
+                .debug_struct("S3EncryptionConfig")
+                .field("type", &"sse_c")
+                .field("key", &"***redacted***")
+                .field("key_md5", key_md5)
+                .finish(),
+        }
+    }
+}
+
 #[derive(Clone, Default, Eq, PartialEq, Serialize, Deserialize)]
 #[serde(deny_unknown_fields)]
 pub struct S3StorageConfig {
@@ -329,6 +352,8 @@ pub struct S3StorageConfig {
     pub disable_multi_object_delete: bool,
     #[serde(default)]
     pub disable_multipart_upload: bool,
+    #[serde(default)]
+    pub encryption: Option<S3EncryptionConfig>,
 }
 
 impl S3StorageConfig {
@@ -685,4 +710,29 @@ mod tests {
             assert_eq!(s3_storage_config.flavor, Some(StorageBackendFlavor::MinIO));
         }
     }
+
+    #[test]
+    fn test_storage_s3_config_encryption_serde() {
+        {
+            let s3_storage_config_yaml = r#"
+                endpoint: http://localhost:4566
+                encryption:
+                    type: sse_c
+                    key: test-customer-key
+                    key_md5: test-customer-key-md5
+            "#;
+            let s3_storage_config: S3StorageConfig =
+                serde_yaml::from_str(s3_storage_config_yaml).unwrap();
+
+            let expected_s3_config = S3StorageConfig {
+                endpoint: Some("http://localhost:4566".to_string()),
+                encryption: Some(S3EncryptionConfig::SseC {
+                    key: "test-customer-key".to_string(),
+                    key_md5: "test-customer-key-md5".to_string(),
+                }),
+                ..Default::default()
+            };
+            assert_eq!(s3_storage_config, expected_s3_config);
+        }
+    }
 }