[TASK] Shift secrets management for CNPG Credentials to External Secrets Operator

infrastructure/main.tf

@@ -90,7 +90,7 @@ module "garage" {
 
 # Cloudnative PG Deployment for PostgreSQL Database Solution
 module "cnpg" {
-  source = "git::https://github.com/necro-cloud/modules//modules/cnpg?ref=main"
+  source = "git::https://github.com/necro-cloud/modules//modules/cnpg?ref=task/117/cnpg-external-secrets"
 
   // Cluster Secret Store Details
   cluster_secret_store_name = module.openbao.cluster_secret_store_name
@@ -171,8 +171,11 @@ module "ferretdb" {
 
 # Keycloak Cluster Deployment for Identity Solution
 module "keycloak" {
-  source = "git::https://github.com/necro-cloud/modules//modules/keycloak?ref=main"
+  source = "git::https://github.com/necro-cloud/modules//modules/keycloak?ref=task/117/cnpg-external-secrets"
 
+  // Cluster Secret Store Details
+  cluster_secret_store_name = module.openbao.cluster_secret_store_name
+
   // PostgreSQL Database Details for database details
   cluster_issuer_name                        = module.cluster-issuer.cluster-issuer-name
   postgres_namespace                         = module.cnpg.namespace

modules/cnpg/README.md

@@ -7,6 +7,7 @@ Required Modules to deploy Cloudnative PG PostgreSQL Database:
 2. [Cluster Issuer](../cluster-issuer)
 3. [Garage](../garage)
 4. [Observability](../observability)
+5. [OpenBao](../openbao)
 
 ## Providers
 
@@ -26,50 +27,55 @@ Required Modules to deploy Cloudnative PG PostgreSQL Database:
 | [kubernetes_manifest.barman_object_store](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/manifest) | resource |
 | [kubernetes_manifest.client_certificate_authority](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/manifest) | resource |
 | [kubernetes_manifest.client_certificates](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/manifest) | resource |
+| [kubernetes_manifest.client_database_credentials_sync](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/manifest) | resource |
 | [kubernetes_manifest.client_issuer](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/manifest) | resource |
 | [kubernetes_manifest.client_keycloak_certificate](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/manifest) | resource |
 | [kubernetes_manifest.client_streaming_replica_certificate](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/manifest) | resource |
 | [kubernetes_manifest.cluster](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/manifest) | resource |
 | [kubernetes_manifest.cluster_image_catalog](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/manifest) | resource |
 | [kubernetes_manifest.databases](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/manifest) | resource |
+| [kubernetes_manifest.garage_certificate_authority_sync](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/manifest) | resource |
+| [kubernetes_manifest.garage_configuration_sync](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/manifest) | resource |
 | [kubernetes_manifest.ingress_certificate](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/manifest) | resource |
 | [kubernetes_manifest.keycloak_database](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/manifest) | resource |
+| [kubernetes_manifest.keycloak_database_credentials_sync](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/manifest) | resource |
+| [kubernetes_manifest.password_generator](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/manifest) | resource |
+| [kubernetes_manifest.pgadmin_credentials_sync](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/manifest) | resource |
 | [kubernetes_manifest.pgadmin_internal_certificate](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/manifest) | resource |
 | [kubernetes_manifest.public_issuer](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/manifest) | resource |
+| [kubernetes_manifest.push_client_certificates](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/manifest) | resource |
+| [kubernetes_manifest.push_client_database_credentials](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/manifest) | resource |
+| [kubernetes_manifest.push_client_keycloak_certificate](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/manifest) | resource |
+| [kubernetes_manifest.push_keycloak_database_credentials](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/manifest) | resource |
+| [kubernetes_manifest.push_pgadmin_credentials](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/manifest) | resource |
+| [kubernetes_manifest.push_server_certificate_authority](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/manifest) | resource |
 | [kubernetes_manifest.server_certificate](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/manifest) | resource |
 | [kubernetes_manifest.server_certificate_authority](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/manifest) | resource |
 | [kubernetes_manifest.server_issuer](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/manifest) | resource |
 | [kubernetes_namespace.namespace](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/namespace) | resource |
 | [kubernetes_network_policy.cnpg_network_policy](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/network_policy) | resource |
 | [kubernetes_pod_disruption_budget_v1.cnpg_pdb](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/pod_disruption_budget_v1) | resource |
-| [kubernetes_secret.client_database_credentials](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/secret) | resource |
 | [kubernetes_secret.cloudflare_token](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/secret) | resource |
-| [kubernetes_secret.garage_certificate_authority](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/secret) | resource |
-| [kubernetes_secret.garage_configuration](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/secret) | resource |
-| [kubernetes_secret.keycloak_database_credentials](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/secret) | resource |
-| [kubernetes_secret.pgadmin_credentials](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/secret) | resource |
 | [kubernetes_service.pgadmin4](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/service) | resource |
-| [random_password.client_password](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/password) | resource |
-| [random_password.keycloak_password](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/password) | resource |
-| [random_password.pgadmin_password](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/password) | resource |
 
 ## Inputs
 
 | Name | Description | Type | Default | Required |
 |------|-------------|------|---------|:--------:|
-| <a name="input_acme_server"></a> [acme\_server](#input\_acme\_server) | URL for the ACME Server to be used, defaults to production URL for LetsEncrypt | `string` | `"https://acme-v02.api.letsencrypt.org/directory"` | no |
+| <a name="input_acme_server"></a> [acme\_server](#input\_acme\_server) | URL for the ACME Server to be used, defaults to production URL for LetsEncrypt | `string` | `"https://acme-v02
 | <a name="input_app_name"></a> [app\_name](#input\_app\_name) | App name for deploying PostgreSQL Database | `string` | `"postgres"` | no |
 | <a name="input_backup_bucket_name"></a> [backup\_bucket\_name](#input\_backup\_bucket\_name) | Name of the bucket for storing PITR Backups in Garage | `string` | n/a | yes |
-| <a name="input_client_certificate_authority_name"></a> [client\_certificate\_authority\_name](#input\_client\_certificate\_authority\_name) | Name of the Certificate Authority to be used with PostgreSQL Client | `string` | `"postgresql-client-certificate-authority"` | no |
-| <a name="input_client_issuer_name"></a> [client\_issuer\_name](#input\_client\_issuer\_name) | Name of the Issuer to be used with PostgreSQL Client | `string` | `"postgresql-client-issuer"` | no |
+| <a name="input_client_certificate_authority_name"></a> [client\_certificate\_authority\_name](#input\_client\_certificate\_authority\_name) | Name of the Certificate Authority to be
+| <a name="input_client_issuer_name"></a> [client\_issuer\_name](#input\_client\_issuer\_name) | Name of the Issuer to be used with PostgreSQL Client | `string` | `"postgresql-client-i
 | <a name="input_client_streaming_replica_certificate_name"></a> [client\_streaming\_replica\_certificate\_name](#input\_client\_streaming\_replica\_certificate\_name) | Name of the Certificate to be used with PostgreSQL Streaming Replica Client | `string` | `"postgresql-streaming-replica-client-certificate"` | no |
 | <a name="input_clients"></a> [clients](#input\_clients) | Object List of clients who need databases and users to be configured for | <pre>list(object({<br/>    namespace          = string<br/>    user               = string<br/>    database           = string<br/>    derRequired        = bool<br/>    privateKeyEncoding = string<br/>  }))</pre> | `[]` | no |
-| <a name="input_cloudflare_email"></a> [cloudflare\_email](#input\_cloudflare\_email) | Email for generating Ingress Certificates to be associated with PGAdmin | `string` | n/a | yes |
+| <a name="input_cloudflare_email"></a> [cloudflare\_email](#input\_cloudflare\_email) | Email for generating Ingress Certificates to be associated with PGAdmin | `string` | n/a | yes
 | <a name="input_cloudflare_issuer_name"></a> [cloudflare\_issuer\_name](#input\_cloudflare\_issuer\_name) | Name of the Cloudflare Issuer to be associated with PGAdmin | `string` | `"cnpg-cloudflare-issuer"` | no |
-| <a name="input_cloudflare_token"></a> [cloudflare\_token](#input\_cloudflare\_token) | Token for generating Ingress Certificates to be associated with PGAdmin | `string` | n/a | yes |
-| <a name="input_cluster_issuer_name"></a> [cluster\_issuer\_name](#input\_cluster\_issuer\_name) | Name for the Cluster Issuer to be used to generate internal self signed certificates | `string` | n/a | yes |
+| <a name="input_cloudflare_token"></a> [cloudflare\_token](#input\_cloudflare\_token) | Token for generating Ingress Certificates to be associated with PGAdmin | `string` | n/a | yes
+| <a name="input_cluster_issuer_name"></a> [cluster\_issuer\_name](#input\_cluster\_issuer\_name) | Name for the Cluster Issuer to be used to generate internal self signed certificates
 | <a name="input_cluster_name"></a> [cluster\_name](#input\_cluster\_name) | Name of the PostgreSQL Database Cluster to be created | `string` | `"postgresql-cluster"` | no |
 | <a name="input_cluster_postgresql_version"></a> [cluster\_postgresql\_version](#input\_cluster\_postgresql\_version) | Version of PostgreSQL Database to use and deploy | `number` | `17` | no |
+| <a name="input_cluster_secret_store_name"></a> [cluster\_secret\_store\_name](#input\_cluster\_secret\_store\_name) | Name of the cluster secret store to be used for pulling and push
 | <a name="input_cluster_size"></a> [cluster\_size](#input\_cluster\_size) | Number of pods to deploy for the PostgreSQL Cluster | `number` | `2` | no |
 | <a name="input_country_name"></a> [country\_name](#input\_country\_name) | Country name for deploying PostgreSQL Database | `string` | `"India"` | no |
 | <a name="input_domain"></a> [domain](#input\_domain) | Domain for which Ingress Certificate is to be generated for | `string` | n/a | yes |

modules/cnpg/certificates.tf

@@ -98,6 +98,50 @@ resource "kubernetes_manifest" "server_certificate_authority" {
   }
 }
 
+// Pushing the certificate to OpenBao for distribution
+resource "kubernetes_manifest" "push_server_certificate_authority" {
+  manifest = {
+    apiVersion = "external-secrets.io/v1alpha1"
+    kind       = "PushSecret"
+    metadata = {
+      name      = "push-server-certificate-authority"
+      namespace = kubernetes_namespace.namespace.metadata[0].name
+    }
+    spec = {
+      refreshInterval = "1h"
+      deletionPolicy  = "None"
+      secretStoreRefs = [{
+        name = var.cluster_secret_store_name
+        kind = "ClusterSecretStore"
+      }]
+      selector = {
+        secret = {
+          name = kubernetes_manifest.server_certificate_authority.object.spec.secretName
+        }
+      }
+      data = [
+        {
+          match = {
+            remoteRef = {
+              remoteKey = "${kubernetes_namespace.namespace.metadata[0].name}/certificates/${kubernetes_manifest.server_certificate_authority.object.spec.secretName}"
+            }
+          }
+        }
+      ]
+    }
+  }
+
+  // Ensure the certificate is actually issued before trying to push it
+  depends_on = [kubernetes_manifest.server_certificate_authority]
+
+  wait {
+    condition {
+      type   = "Ready"
+      status = "True"
+    }
+  }
+}
+
 // Issuer to be used with PostgreSQL Server
 resource "kubernetes_manifest" "server_issuer" {
   manifest = {
@@ -339,11 +383,6 @@ resource "kubernetes_manifest" "client_keycloak_certificate" {
           "reflector.v1.k8s.emberstack.com/reflection-allowed-namespaces" = "keycloak"
         }
       }
-      "additionalOutputFormats" = [
-        {
-          "type" : "DER"
-        }
-      ]
       "privateKey" = {
         "encoding" = "PKCS8"
       }
@@ -367,6 +406,53 @@ resource "kubernetes_manifest" "client_keycloak_certificate" {
   }
 }
 
+// Pushing the certificate to OpenBao for distribution
+resource "kubernetes_manifest" "push_client_keycloak_certificate" {
+  manifest = {
+    apiVersion = "external-secrets.io/v1alpha1"
+    kind       = "PushSecret"
+    metadata = {
+      name      = "push-client-keycloak-certificate"
+      namespace = kubernetes_namespace.namespace.metadata[0].name
+    }
+    spec = {
+      refreshInterval = "1h"
+      deletionPolicy  = "None"
+      secretStoreRefs = [{
+        name = var.cluster_secret_store_name
+        kind = "ClusterSecretStore"
+      }]
+      selector = {
+        secret = {
+          name = kubernetes_manifest.client_keycloak_certificate.object.spec.secretName
+        }
+      }
+      data = [
+        {
+          match = {
+            remoteRef = {
+              remoteKey = "${kubernetes_namespace.namespace.metadata[0].name}/certificates/${kubernetes_manifest.client_keycloak_certificate.object.spec.secretName}"
+            }
+          }
+          metadata = {
+            encoding = "base64"
+          }
+        }
+      ]
+    }
+  }
+
+  // Ensure the certificate is actually issued before trying to push it
+  depends_on = [kubernetes_manifest.client_keycloak_certificate]
+
+  wait {
+    condition {
+      type   = "Ready"
+      status = "True"
+    }
+  }
+}
+
 // Certificates for all clients
 resource "kubernetes_manifest" "client_certificates" {
   count = length(var.clients)
@@ -424,6 +510,51 @@ resource "kubernetes_manifest" "client_certificates" {
   }
 }
 
+// Pushing the certificate to OpenBao for distribution
+resource "kubernetes_manifest" "push_client_certificates" {
+  count = length(var.clients)
+  manifest = {
+    apiVersion = "external-secrets.io/v1alpha1"
+    kind       = "PushSecret"
+    metadata = {
+      name      = "push-${var.clients[count.index].user}-client-certificate"
+      namespace = kubernetes_namespace.namespace.metadata[0].name
+    }
+    spec = {
+      refreshInterval = "1h"
+      deletionPolicy  = "None"
+      secretStoreRefs = [{
+        name = var.cluster_secret_store_name
+        kind = "ClusterSecretStore"
+      }]
+      selector = {
+        secret = {
+          name = kubernetes_manifest.client_certificates[count.index].object.spec.secretName
+        }
+      }
+      data = [
+        {
+          match = {
+            remoteRef = {
+              remoteKey = "${kubernetes_namespace.namespace.metadata[0].name}/certificates/${kubernetes_manifest.client_certificates[count.index].object.spec.secretName}"
+            }
+          }
+        }
+      ]
+    }
+  }
+
+  // Ensure the certificate is actually issued before trying to push it
+  depends_on = [kubernetes_manifest.client_certificates]
+
+  wait {
+    condition {
+      type   = "Ready"
+      status = "True"
+    }
+  }
+}
+
 // Internal Certificate for PGAdmin
 resource "kubernetes_manifest" "pgadmin_internal_certificate" {
   manifest = {

modules/cnpg/cluster.tf

@@ -56,7 +56,7 @@ resource "kubernetes_manifest" "cluster" {
             "login"           = true
             "name"            = "keycloak"
             "passwordSecret" = {
-              "name" = kubernetes_secret.keycloak_database_credentials.metadata[0].name
+              "name" = kubernetes_manifest.keycloak_database_credentials_sync.object.spec.target.name
             }
             "replication" = false
             "superuser"   = false
@@ -123,4 +123,11 @@ resource "kubernetes_manifest" "cluster" {
     update = "10m"
     delete = "10m"
   }
+
+  depends_on = [
+    kubernetes_manifest.barman_object_store,
+    kubernetes_manifest.keycloak_database_credentials_sync,
+    kubernetes_manifest.client_database_credentials_sync,
+    kubernetes_manifest.garage_configuration_sync   
+  ]
 }

modules/cnpg/locals.tf

@@ -1,17 +1,17 @@
 locals {
   access_namespaces = [for config in var.clients : config.namespace]
-  managed_roles = [for secret in kubernetes_secret.client_database_credentials : {
+  managed_roles = [for index, client in var.clients : {
     "bypassrls"       = false
-    "comment"         = "${secret.data.username} user for postgresql"
+    "comment"         = "${client.user} user for postgresql"
     "connectionLimit" = -1
     "createdb"        = true
     "createrole"      = true
     "ensure"          = "present"
     "inherit"         = true
     "login"           = true
-    "name"            = secret.data.username
+    "name"            = client.user
     "passwordSecret" = {
-      "name" = secret.metadata[0].name
+      "name" = kubernetes_manifest.client_database_credentials_sync[index].object.spec.target.name
     }
     "replication" = false
     "superuser"   = false

modules/cnpg/pgadmin.tf

@@ -30,7 +30,7 @@ resource "kubernetes_deployment" "pgadmin" {
 
           env_from {
             secret_ref {
-              name = kubernetes_secret.pgadmin_credentials.metadata[0].name
+              name = kubernetes_manifest.pgadmin_credentials_sync.object.spec.target.name
             }
           }
 
@@ -175,20 +175,20 @@ resource "kubernetes_deployment" "pgadmin" {
           projected {
             sources {
               secret {
-                name = kubernetes_secret.keycloak_database_credentials.metadata[0].name
+                name = kubernetes_manifest.keycloak_database_credentials_sync.object.spec.target.name
                 items {
                   key  = "password"
                   path = "keycloak/password"
                 }
               }
 
               dynamic "secret" {
-                for_each = kubernetes_secret.client_database_credentials
+                for_each = kubernetes_manifest.client_database_credentials_sync
                 content {
-                  name = secret.value.metadata[0].name
+                  name = secret.value.object.spec.target.name
                   items {
                     key  = "password"
-                    path = "${split("-", secret.value.metadata[0].name)[1]}/password"
+                    path = "${split("-", secret.value.object.spec.target.name)[1]}/password"
                   }
                 }
               }