From 8e60d70c0a60b46ac16bca35ec926b1013e6feb6 Mon Sep 17 00:00:00 2001 From: Christopher Tineo Date: Thu, 13 Aug 2026 00:26:47 -0400 Subject: [PATCH 1/2] feat(infra): install CloudNativePG operator and shared-cluster MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit First half of moving Balancer off AWS RDS (CodeForPhilly/balancer-main#526). Production still reads from RDS; this only stands the cluster up beside it. Mirrors cfp-sandbox-cluster's layout: chart v0.28.0 as a holosource, projected to _infra/cloudnative-pg/operator and rendered by a helm3 lens with CRDs included; the Cluster CR and namespace sit outside the lens root and pass through raw. Two deliberate omissions, both documented inline: - no managed.roles — the balancer role's passwordSecret must be sealed first, or cnpg reports a reconcile error until it exists - no backup stanza — needs an object store and credentials that do not exist yet. Nothing may depend on this cluster as a sole copy until that lands. Storage class is pinned to linode-block-storage-retain rather than taking the cluster default, which sandbox does. Refs CodeForPhilly/balancer-main#526 --- .claude/CLAUDE.md | 2 +- .../_infra/cloudnative-pg/operator.toml | 4 ++++ .holo/lenses/cloudnative-pg.toml | 14 +++++++++++ .holo/sources/cloudnative-pg-chart.toml | 3 +++ _infra/cloudnative-pg/namespaces.yaml | 4 ++++ _infra/cloudnative-pg/shared-cluster.yaml | 24 +++++++++++++++++++ 6 files changed, 50 insertions(+), 1 deletion(-) create mode 100644 .holo/branches/k8s-manifests/_infra/cloudnative-pg/operator.toml create mode 100644 .holo/lenses/cloudnative-pg.toml create mode 100644 .holo/sources/cloudnative-pg-chart.toml create mode 100644 _infra/cloudnative-pg/namespaces.yaml create mode 100644 _infra/cloudnative-pg/shared-cluster.yaml diff --git a/.claude/CLAUDE.md b/.claude/CLAUDE.md index de96526..b3da150 100644 --- a/.claude/CLAUDE.md +++ b/.claude/CLAUDE.md @@ -140,7 +140,7 @@ Things not in any single grep-able file: - **Wildcard DNS**: `*.live.k8s.phl.io` → the Envoy LB `45.79.246.168`. DNS is managed in OpenTofu at [CodeForPhilly/ops](https://github.com/CodeForPhilly/ops) → `tofu/dns`; a host with no specific record simply follows the wildcard, so new apps need no DNS change at all. - **Apex domains in tree**: `balancerproject.org`, `choosenativeplants.com` (+ `www.`), `codeforphilly.org` (+ `www.`), `penn-chime.phl.io`, `vaultwarden.phl.io`, `bitwarden.phl.io`. Apex ACME challenges only work once DNS points at Envoy — plan cutover and cert issuance together for these. `choosenativeplants.com` is at **Namecheap**, not Cloud DNS, so it can't be moved from the ops repo. - **A new hostname is briefly down between DNS and cert.** The cert can't issue until the hostname resolves to Envoy (Let's Encrypt has to reach the solver), and Envoy's HTTPS listener doesn't program until the cert Secret exists — meanwhile HTTP 301s into a listener that isn't there. Roughly 60–90s. Keep TTLs at 60s. -- **No cnpg / shared-cluster** on this cluster yet. If a database is needed, it ships per-app (e.g. vaultwarden runs its own PostgreSQL StatefulSet via the gissilabs chart; chime + third-places similar). +- **cnpg is landing.** `_infra/cloudnative-pg/` installs the operator (chart v0.28.0) and a `shared-cluster` Cluster, mirroring cfp-sandbox-cluster. It has **no backup configuration** — do not let it hold the only copy of anything until an object store lands (CodeForPhilly/balancer-main#526). Existing apps still ship their own database per-app (vaultwarden runs a PostgreSQL StatefulSet via the gissilabs chart; chime + third-places similar); nothing has been migrated onto the shared cluster yet. ## Guardrails diff --git a/.holo/branches/k8s-manifests/_infra/cloudnative-pg/operator.toml b/.holo/branches/k8s-manifests/_infra/cloudnative-pg/operator.toml new file mode 100644 index 0000000..71776e8 --- /dev/null +++ b/.holo/branches/k8s-manifests/_infra/cloudnative-pg/operator.toml @@ -0,0 +1,4 @@ +[holomapping] +holosource = "cloudnative-pg-chart" +root = "charts/cloudnative-pg" +files = "**" diff --git a/.holo/lenses/cloudnative-pg.toml b/.holo/lenses/cloudnative-pg.toml new file mode 100644 index 0000000..2227b87 --- /dev/null +++ b/.holo/lenses/cloudnative-pg.toml @@ -0,0 +1,14 @@ +[hololens] +container = "ghcr.io/hologit/lenses/helm3:latest" + +[hololens.input] +root = "_infra/cloudnative-pg/operator" +files = "**" + +[hololens.output] +merge = "replace" + +[hololens.helm] +namespace = "cloudnative-pg" +release_name = "cloudnative-pg" +include_crds = true diff --git a/.holo/sources/cloudnative-pg-chart.toml b/.holo/sources/cloudnative-pg-chart.toml new file mode 100644 index 0000000..1ac9f69 --- /dev/null +++ b/.holo/sources/cloudnative-pg-chart.toml @@ -0,0 +1,3 @@ +[holosource] +url = "https://github.com/cloudnative-pg/charts.git" +ref = "refs/tags/cloudnative-pg-v0.28.0" diff --git a/_infra/cloudnative-pg/namespaces.yaml b/_infra/cloudnative-pg/namespaces.yaml new file mode 100644 index 0000000..e689a91 --- /dev/null +++ b/_infra/cloudnative-pg/namespaces.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: cloudnative-pg diff --git a/_infra/cloudnative-pg/shared-cluster.yaml b/_infra/cloudnative-pg/shared-cluster.yaml new file mode 100644 index 0000000..7540c9a --- /dev/null +++ b/_infra/cloudnative-pg/shared-cluster.yaml @@ -0,0 +1,24 @@ +apiVersion: postgresql.cnpg.io/v1 +kind: Cluster +metadata: + name: shared-cluster + namespace: cloudnative-pg +spec: + instances: 2 + imageName: ghcr.io/cloudnative-pg/postgis:18-3-system-trixie + + storage: + # Explicit, unlike sandbox, which takes the cluster default. `-retain` keeps + # the Linode volume if the PVC is ever deleted — on this cluster that is the + # difference between an incident and an outage. + storageClass: linode-block-storage-retain + size: 20Gi + + # No `managed.roles` yet: the balancer role's passwordSecret has to be sealed + # into cloudnative-pg.secrets/ first, and cnpg reports a role reconcile error + # for as long as the Secret it names is absent. The role and the balancer + # Database CR land together in the follow-up PR. + # + # No `backup:` stanza yet either — it needs an object store bucket plus + # credentials that do not exist. This cluster must not hold the only copy of + # production data until that lands. See CodeForPhilly/balancer-main#526. From bd1a5a03083dbe9173f5cff8646fba1a7d3f2e98 Mon Sep 17 00:00:00 2001 From: Christopher Tineo Date: Thu, 13 Aug 2026 00:27:49 -0400 Subject: [PATCH 2/2] feat(balancer): declare the cnpg balancer role and Database MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Second half of standing cnpg up for Balancer (CodeForPhilly/balancer-main#526). Creates an empty `balancer` database on shared-cluster with pgvector enabled, so the RDS dump has somewhere to be restored to. Still no cutover: the app's sealed SQL_HOST is untouched and production keeps reading from RDS. Splits balancer/ into app/ + cnpg/ to match cfp-sandbox-cluster. The Database CR has to stay in the cloudnative-pg namespace — cnpg requires it to sit beside the Cluster it targets — so it cannot live under the kustomization that rewrites everything to `balancer`. The existing kustomization moves to app/ unchanged, along with its holomapping. The managed role names balancer-db-credentials, which must be sealed into cloudnative-pg.secrets/ before this merges or cnpg will report a role reconcile error. Note cnpg does not pick up a newly-created passwordSecret on its periodic resync — force it with: kubectl annotate cluster -n cloudnative-pg shared-cluster \ cnpg.io/reconciliationLoop="$(date +%s)" --overwrite Refs CodeForPhilly/balancer-main#526, CodeForPhilly/cfp-sandbox-cluster#162 --- .../balancer/{ => app}/manifests.toml | 0 _infra/cloudnative-pg/shared-cluster.yaml | 14 ++--- balancer/app/kustomization.yaml | 50 +++++++++++++++++ balancer/cnpg/database.yaml | 16 ++++++ balancer/cnpg/kustomization.yaml | 8 +++ balancer/kustomization.yaml | 53 +++---------------- 6 files changed, 89 insertions(+), 52 deletions(-) rename .holo/branches/k8s-manifests/balancer/{ => app}/manifests.toml (100%) create mode 100644 balancer/app/kustomization.yaml create mode 100644 balancer/cnpg/database.yaml create mode 100644 balancer/cnpg/kustomization.yaml diff --git a/.holo/branches/k8s-manifests/balancer/manifests.toml b/.holo/branches/k8s-manifests/balancer/app/manifests.toml similarity index 100% rename from .holo/branches/k8s-manifests/balancer/manifests.toml rename to .holo/branches/k8s-manifests/balancer/app/manifests.toml diff --git a/_infra/cloudnative-pg/shared-cluster.yaml b/_infra/cloudnative-pg/shared-cluster.yaml index 7540c9a..97f3241 100644 --- a/_infra/cloudnative-pg/shared-cluster.yaml +++ b/_infra/cloudnative-pg/shared-cluster.yaml @@ -14,11 +14,13 @@ spec: storageClass: linode-block-storage-retain size: 20Gi - # No `managed.roles` yet: the balancer role's passwordSecret has to be sealed - # into cloudnative-pg.secrets/ first, and cnpg reports a role reconcile error - # for as long as the Secret it names is absent. The role and the balancer - # Database CR land together in the follow-up PR. - # - # No `backup:` stanza yet either — it needs an object store bucket plus + managed: + roles: + - name: balancer + login: true + passwordSecret: + name: balancer-db-credentials + + # No `backup:` stanza yet — it needs an object store bucket plus # credentials that do not exist. This cluster must not hold the only copy of # production data until that lands. See CodeForPhilly/balancer-main#526. diff --git a/balancer/app/kustomization.yaml b/balancer/app/kustomization.yaml new file mode 100644 index 0000000..d715cfa --- /dev/null +++ b/balancer/app/kustomization.yaml @@ -0,0 +1,50 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namespace: balancer + +# balancer-main v1.1.7 owns its own HTTPRoute (paths, backends, ports) and no +# longer ships an Ingress. This repo owns the Gateway — hostname, TLS, issuer — +# in _gateways/balancer.yaml. The route attaches to it by name and declares no +# hostnames, so it inherits the Gateway's listener hostname. +# +# We deliberately do not take upstream's gateway-listeners.yaml, and v1.1.7 +# removed it: it declared a ListenerSet, which Envoy Gateway v1.7.3 does not +# reconcile ("XListenerSet CRD not found, skipping XListenerSet watch"). It +# would apply cleanly and then be silently ignored. +resources: + - manifests/namespace.yaml + - manifests/deployment.yaml + - manifests/service.yaml + - manifests/httproute.yaml + +# Stays on the running version. v1.1.7 is a deploy-manifest release with no +# application changes, so bumping the image here would ship app code under cover +# of a routing change. Bump it deliberately, on its own. +images: + - name: ghcr.io/codeforphilly/balancer-main/app + newTag: "1.1.5" + +# v1.1.7's Deployment does `envFrom: configMapRef: balancer-config` alongside the +# existing `secretRef: balancer-config` (our SealedSecret, holding SECRET_KEY, +# SQL_*, and the API keys). A ConfigMap and a Secret may share a name. +# +# balancer.env is empty upstream, and the app reads ALLOWED_HOSTS rather than +# HOSTNAME — so this ConfigMap carries nothing. It exists solely so configMapRef +# resolves; without it the pods fail with CreateContainerConfigError. +configMapGenerator: + - name: balancer-config + envs: + - manifests/balancer.env + +generatorOptions: + disableNameSuffixHash: true + +patches: + - target: + kind: Namespace + name: balancer + patch: |- + - op: replace + path: /metadata/name + value: balancer diff --git a/balancer/cnpg/database.yaml b/balancer/cnpg/database.yaml new file mode 100644 index 0000000..dd0f191 --- /dev/null +++ b/balancer/cnpg/database.yaml @@ -0,0 +1,16 @@ +apiVersion: postgresql.cnpg.io/v1 +kind: Database +metadata: + name: balancer + namespace: cloudnative-pg +spec: + name: balancer + owner: balancer + cluster: + name: shared-cluster + # Django's api.0005_embeddings migration creates a vector(384) column. + # cnpg-controller runs CREATE EXTENSION as the postgres superuser; the + # `balancer` role has no superuser perms and cannot enable extensions itself. + extensions: + - name: vector + ensure: present diff --git a/balancer/cnpg/kustomization.yaml b/balancer/cnpg/kustomization.yaml new file mode 100644 index 0000000..041546b --- /dev/null +++ b/balancer/cnpg/kustomization.yaml @@ -0,0 +1,8 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +# No namespace — database.yaml declares its own (cloudnative-pg). cnpg requires +# the Database CR to live in the same namespace as the Cluster it targets, so we +# keep it out of the balancer-app rewrite. +resources: + - database.yaml diff --git a/balancer/kustomization.yaml b/balancer/kustomization.yaml index d715cfa..2c5460c 100644 --- a/balancer/kustomization.yaml +++ b/balancer/kustomization.yaml @@ -1,50 +1,11 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization -namespace: balancer - -# balancer-main v1.1.7 owns its own HTTPRoute (paths, backends, ports) and no -# longer ships an Ingress. This repo owns the Gateway — hostname, TLS, issuer — -# in _gateways/balancer.yaml. The route attaches to it by name and declares no -# hostnames, so it inherits the Gateway's listener hostname. -# -# We deliberately do not take upstream's gateway-listeners.yaml, and v1.1.7 -# removed it: it declared a ListenerSet, which Envoy Gateway v1.7.3 does not -# reconcile ("XListenerSet CRD not found, skipping XListenerSet watch"). It -# would apply cleanly and then be silently ignored. +# No namespace here — each sub-kustomization sets its own: +# - app/ rewrites resources to `balancer` +# - cnpg/ leaves the Database CR in `cloudnative-pg`, where cnpg requires it +# kustomize emits a single stream; k8s-normalize routes by each resource's +# metadata.namespace at deploy time. Same split as cfp-sandbox-cluster. resources: - - manifests/namespace.yaml - - manifests/deployment.yaml - - manifests/service.yaml - - manifests/httproute.yaml - -# Stays on the running version. v1.1.7 is a deploy-manifest release with no -# application changes, so bumping the image here would ship app code under cover -# of a routing change. Bump it deliberately, on its own. -images: - - name: ghcr.io/codeforphilly/balancer-main/app - newTag: "1.1.5" - -# v1.1.7's Deployment does `envFrom: configMapRef: balancer-config` alongside the -# existing `secretRef: balancer-config` (our SealedSecret, holding SECRET_KEY, -# SQL_*, and the API keys). A ConfigMap and a Secret may share a name. -# -# balancer.env is empty upstream, and the app reads ALLOWED_HOSTS rather than -# HOSTNAME — so this ConfigMap carries nothing. It exists solely so configMapRef -# resolves; without it the pods fail with CreateContainerConfigError. -configMapGenerator: - - name: balancer-config - envs: - - manifests/balancer.env - -generatorOptions: - disableNameSuffixHash: true - -patches: - - target: - kind: Namespace - name: balancer - patch: |- - - op: replace - path: /metadata/name - value: balancer + - app + - cnpg