From a3314595fc747b195729be7637f482901ae8af38 Mon Sep 17 00:00:00 2001 From: Dave Page Date: Thu, 2 Jul 2026 15:29:11 +0100 Subject: [PATCH] feat(coldfront): control-plane support for ColdFront (single-node) Add the control-plane side of ColdFront transparent data tiering: deploy and bootstrap the Lakekeeper Iceberg catalog per database, load the extension config, and schedule the tiering jobs. Consumes the `lakekeeper` service the saas control plane sends. This is the single-node scope; it is one of several per-repo PRs for the feature. Included: - Register the `lakekeeper` service type (image, launch on port 8181, config resource, validator, Goa enum), following the MCP recipe. - External Lakekeeper catalog Postgres via a configurable connection URL (Cloud supplies a managed instance; the control plane does not provision it), with a `migrate`-before-`serve` dependency and fail-loud if the URL is absent. - Post-deploy bootstrap: idempotent Lakekeeper REST warehouse creation (bootstrap -> warehouse -> namespace) with the correct S3 storage-profile (`flavor`/`path-style-access`/`key-prefix`), and `coldfront.set_storage_secret` / `_azure` on the database with the object-store credential bound as query arguments (never interpolated or logged). - Schedule the archiver/partitioner/compactor via the existing gocron/etcd scheduler, running each single-pass in the primary node's Postgres container and capturing the exit code (recorded as `task.TypeTiering`); the archiver's "no tables configured" exit is treated as benign. - Reject enabling ColdFront on a multi-node database (fail-loud), pending the deferred mesh `snowflake.node` reconciliation. Deferred to follow-ups (see PR description): the per-node mesh GUCs for multi-node ColdFront (needs a CP + ColdFront-author decision on `snowflake.node` ownership); expansion of the saas lakekeeper contract (`catalog_db_url`, `pg_encryption_key`, `provider`/`bucket`/`region`/ `endpoint`); the ColdFront-enabled Postgres image; and confirmation of the pinned Lakekeeper image tag. --- .gitignore | 3 + api/apiv1/design/database.go | 3 +- .../gen/http/control_plane/client/types.go | 8 +- .../gen/http/control_plane/server/types.go | 8 +- api/apiv1/gen/http/openapi.json | 19 +- api/apiv1/gen/http/openapi.yaml | 17 +- api/apiv1/gen/http/openapi3.json | 137 +++---- api/apiv1/gen/http/openapi3.yaml | 123 +++--- server/internal/api/apiv1/validate.go | 102 ++++- server/internal/api/apiv1/validate_test.go | 169 +++++++++ .../swarm/coldfront_tiering_wiring_test.go | 105 ++++++ .../swarm/lakekeeper_bootstrap.go | 350 ++++++++++++++++++ .../swarm/lakekeeper_bootstrap_resource.go | 127 +++++++ .../swarm/lakekeeper_bootstrap_test.go | 238 ++++++++++++ .../swarm/lakekeeper_config_resource.go | 127 +++++++ .../swarm/lakekeeper_migrate_resource.go | 177 +++++++++ .../lakekeeper_storage_secret_resource.go | 217 +++++++++++ ...lakekeeper_storage_secret_resource_test.go | 129 +++++++ .../orchestrator/swarm/orchestrator.go | 216 +++++++++++ .../orchestrator/swarm/orchestrator_test.go | 205 ++++++++++ .../internal/orchestrator/swarm/resources.go | 4 + .../orchestrator/swarm/service_images.go | 14 + .../orchestrator/swarm/service_images_test.go | 7 + .../swarm/service_instance_spec.go | 16 +- .../orchestrator/swarm/service_spec.go | 51 ++- .../orchestrator/swarm/service_spec_test.go | 82 +++- .../coldfront_tiering_executor_test.go | 119 ++++++ .../scheduler/scheduled_job_executor.go | 55 +++ server/internal/scheduler/types.go | 4 + server/internal/task/task.go | 1 + .../workflows/activities/activities.go | 1 + .../workflows/activities/coldfront_tiering.go | 300 +++++++++++++++ .../activities/coldfront_tiering_test.go | 190 ++++++++++ .../internal/workflows/coldfront_tiering.go | 127 +++++++ server/internal/workflows/plan_update.go | 4 + server/internal/workflows/service.go | 20 + server/internal/workflows/workflows.go | 1 + 37 files changed, 3315 insertions(+), 161 deletions(-) create mode 100644 server/internal/orchestrator/swarm/coldfront_tiering_wiring_test.go create mode 100644 server/internal/orchestrator/swarm/lakekeeper_bootstrap.go create mode 100644 server/internal/orchestrator/swarm/lakekeeper_bootstrap_resource.go create mode 100644 server/internal/orchestrator/swarm/lakekeeper_bootstrap_test.go create mode 100644 server/internal/orchestrator/swarm/lakekeeper_config_resource.go create mode 100644 server/internal/orchestrator/swarm/lakekeeper_migrate_resource.go create mode 100644 server/internal/orchestrator/swarm/lakekeeper_storage_secret_resource.go create mode 100644 server/internal/orchestrator/swarm/lakekeeper_storage_secret_resource_test.go create mode 100644 server/internal/scheduler/coldfront_tiering_executor_test.go create mode 100644 server/internal/workflows/activities/coldfront_tiering.go create mode 100644 server/internal/workflows/activities/coldfront_tiering_test.go create mode 100644 server/internal/workflows/coldfront_tiering.go diff --git a/.gitignore b/.gitignore index 8d2fdc30..e62d7c27 100644 --- a/.gitignore +++ b/.gitignore @@ -25,3 +25,6 @@ control-plane pgedge-control-plane !docker/control-plane e2e/debug + +# Git worktrees (local; never committed) +.worktrees/ diff --git a/api/apiv1/design/database.go b/api/apiv1/design/database.go index 4aa3a982..b4794761 100644 --- a/api/apiv1/design/database.go +++ b/api/apiv1/design/database.go @@ -169,10 +169,11 @@ var ServiceSpec = g.Type("ServiceSpec", func() { }) g.Attribute("service_type", g.String, func() { g.Description("The type of service to run.") - g.Enum("mcp", "postgrest", "rag") + g.Enum("mcp", "postgrest", "rag", "lakekeeper") g.Example("mcp") g.Example("postgrest") g.Example("rag") + g.Example("lakekeeper") g.Meta("struct:tag:json", "service_type") }) g.Attribute("version", g.String, func() { diff --git a/api/apiv1/gen/http/control_plane/client/types.go b/api/apiv1/gen/http/control_plane/client/types.go index ad3e3013..fe37c76c 100644 --- a/api/apiv1/gen/http/control_plane/client/types.go +++ b/api/apiv1/gen/http/control_plane/client/types.go @@ -6385,8 +6385,8 @@ func ValidateServiceSpecRequestBody(body *ServiceSpecRequestBody) (err error) { if utf8.RuneCountInString(body.ServiceID) > 36 { err = goa.MergeErrors(err, goa.InvalidLengthError("body.service_id", body.ServiceID, utf8.RuneCountInString(body.ServiceID), 36, false)) } - if !(body.ServiceType == "mcp" || body.ServiceType == "postgrest" || body.ServiceType == "rag") { - err = goa.MergeErrors(err, goa.InvalidEnumValueError("body.service_type", body.ServiceType, []any{"mcp", "postgrest", "rag"})) + if !(body.ServiceType == "mcp" || body.ServiceType == "postgrest" || body.ServiceType == "rag" || body.ServiceType == "lakekeeper") { + err = goa.MergeErrors(err, goa.InvalidEnumValueError("body.service_type", body.ServiceType, []any{"mcp", "postgrest", "rag", "lakekeeper"})) } err = goa.MergeErrors(err, goa.ValidatePattern("body.version", body.Version, "^(\\d+\\.\\d+(\\.\\d+)?|latest)$")) if len(body.HostIds) < 1 { @@ -7191,8 +7191,8 @@ func ValidateServiceSpecRequestBodyRequestBody(body *ServiceSpecRequestBodyReque if utf8.RuneCountInString(body.ServiceID) > 36 { err = goa.MergeErrors(err, goa.InvalidLengthError("body.service_id", body.ServiceID, utf8.RuneCountInString(body.ServiceID), 36, false)) } - if !(body.ServiceType == "mcp" || body.ServiceType == "postgrest" || body.ServiceType == "rag") { - err = goa.MergeErrors(err, goa.InvalidEnumValueError("body.service_type", body.ServiceType, []any{"mcp", "postgrest", "rag"})) + if !(body.ServiceType == "mcp" || body.ServiceType == "postgrest" || body.ServiceType == "rag" || body.ServiceType == "lakekeeper") { + err = goa.MergeErrors(err, goa.InvalidEnumValueError("body.service_type", body.ServiceType, []any{"mcp", "postgrest", "rag", "lakekeeper"})) } err = goa.MergeErrors(err, goa.ValidatePattern("body.version", body.Version, "^(\\d+\\.\\d+(\\.\\d+)?|latest)$")) if len(body.HostIds) < 1 { diff --git a/api/apiv1/gen/http/control_plane/server/types.go b/api/apiv1/gen/http/control_plane/server/types.go index c61cc5d1..43f6dfaa 100644 --- a/api/apiv1/gen/http/control_plane/server/types.go +++ b/api/apiv1/gen/http/control_plane/server/types.go @@ -5906,8 +5906,8 @@ func ValidateServiceSpecRequestBody(body *ServiceSpecRequestBody) (err error) { } } if body.ServiceType != nil { - if !(*body.ServiceType == "mcp" || *body.ServiceType == "postgrest" || *body.ServiceType == "rag") { - err = goa.MergeErrors(err, goa.InvalidEnumValueError("body.service_type", *body.ServiceType, []any{"mcp", "postgrest", "rag"})) + if !(*body.ServiceType == "mcp" || *body.ServiceType == "postgrest" || *body.ServiceType == "rag" || *body.ServiceType == "lakekeeper") { + err = goa.MergeErrors(err, goa.InvalidEnumValueError("body.service_type", *body.ServiceType, []any{"mcp", "postgrest", "rag", "lakekeeper"})) } } if body.Version != nil { @@ -6687,8 +6687,8 @@ func ValidateServiceSpecRequestBodyRequestBody(body *ServiceSpecRequestBodyReque } } if body.ServiceType != nil { - if !(*body.ServiceType == "mcp" || *body.ServiceType == "postgrest" || *body.ServiceType == "rag") { - err = goa.MergeErrors(err, goa.InvalidEnumValueError("body.service_type", *body.ServiceType, []any{"mcp", "postgrest", "rag"})) + if !(*body.ServiceType == "mcp" || *body.ServiceType == "postgrest" || *body.ServiceType == "rag" || *body.ServiceType == "lakekeeper") { + err = goa.MergeErrors(err, goa.InvalidEnumValueError("body.service_type", *body.ServiceType, []any{"mcp", "postgrest", "rag", "lakekeeper"})) } } if body.Version != nil { diff --git a/api/apiv1/gen/http/openapi.json b/api/apiv1/gen/http/openapi.json index 6ad3ca98..329fb806 100644 --- a/api/apiv1/gen/http/openapi.json +++ b/api/apiv1/gen/http/openapi.json @@ -4667,7 +4667,7 @@ }, "port": 0, "service_id": "76f9b8c0-4958-11f0-a489-3bb29577c696", - "service_type": "rag", + "service_type": "lakekeeper", "version": "latest" }, { @@ -4746,7 +4746,7 @@ }, "port": 0, "service_id": "76f9b8c0-4958-11f0-a489-3bb29577c696", - "service_type": "rag", + "service_type": "lakekeeper", "version": "latest" }, { @@ -4825,7 +4825,7 @@ }, "port": 0, "service_id": "76f9b8c0-4958-11f0-a489-3bb29577c696", - "service_type": "rag", + "service_type": "lakekeeper", "version": "latest" }, { @@ -4904,7 +4904,7 @@ }, "port": 0, "service_id": "76f9b8c0-4958-11f0-a489-3bb29577c696", - "service_type": "rag", + "service_type": "lakekeeper", "version": "latest" } ] @@ -5631,7 +5631,7 @@ }, "port": 0, "service_id": "76f9b8c0-4958-11f0-a489-3bb29577c696", - "service_type": "rag", + "service_type": "lakekeeper", "version": "latest" }, { @@ -5710,7 +5710,7 @@ }, "port": 0, "service_id": "76f9b8c0-4958-11f0-a489-3bb29577c696", - "service_type": "rag", + "service_type": "lakekeeper", "version": "latest" } ], @@ -9016,11 +9016,12 @@ "service_type": { "type": "string", "description": "The type of service to run.", - "example": "rag", + "example": "lakekeeper", "enum": [ "mcp", "postgrest", - "rag" + "rag", + "lakekeeper" ] }, "version": { @@ -9105,7 +9106,7 @@ }, "port": 0, "service_id": "76f9b8c0-4958-11f0-a489-3bb29577c696", - "service_type": "rag", + "service_type": "lakekeeper", "version": "latest" }, "required": [ diff --git a/api/apiv1/gen/http/openapi.yaml b/api/apiv1/gen/http/openapi.yaml index 1b86fa7b..595a7d9c 100644 --- a/api/apiv1/gen/http/openapi.yaml +++ b/api/apiv1/gen/http/openapi.yaml @@ -3334,7 +3334,7 @@ definitions: host_path: /Users/user/backups/host port: 0 service_id: 76f9b8c0-4958-11f0-a489-3bb29577c696 - service_type: rag + service_type: lakekeeper version: latest - config: llm_model: gpt-4 @@ -3384,7 +3384,7 @@ definitions: host_path: /Users/user/backups/host port: 0 service_id: 76f9b8c0-4958-11f0-a489-3bb29577c696 - service_type: rag + service_type: lakekeeper version: latest - config: llm_model: gpt-4 @@ -3434,7 +3434,7 @@ definitions: host_path: /Users/user/backups/host port: 0 service_id: 76f9b8c0-4958-11f0-a489-3bb29577c696 - service_type: rag + service_type: lakekeeper version: latest - config: llm_model: gpt-4 @@ -3484,7 +3484,7 @@ definitions: host_path: /Users/user/backups/host port: 0 service_id: 76f9b8c0-4958-11f0-a489-3bb29577c696 - service_type: rag + service_type: lakekeeper version: latest spock_version: type: string @@ -4011,7 +4011,7 @@ definitions: host_path: /Users/user/backups/host port: 0 service_id: 76f9b8c0-4958-11f0-a489-3bb29577c696 - service_type: rag + service_type: lakekeeper version: latest - config: llm_model: gpt-4 @@ -4061,7 +4061,7 @@ definitions: host_path: /Users/user/backups/host port: 0 service_id: 76f9b8c0-4958-11f0-a489-3bb29577c696 - service_type: rag + service_type: lakekeeper version: latest spock_version: "5" required: @@ -6474,11 +6474,12 @@ definitions: service_type: type: string description: The type of service to run. - example: rag + example: lakekeeper enum: - mcp - postgrest - rag + - lakekeeper version: type: string description: The version of the service (e.g., '1.0.0', '14.5') or the literal 'latest'. @@ -6532,7 +6533,7 @@ definitions: host_path: /Users/user/backups/host port: 0 service_id: 76f9b8c0-4958-11f0-a489-3bb29577c696 - service_type: rag + service_type: lakekeeper version: latest required: - service_id diff --git a/api/apiv1/gen/http/openapi3.json b/api/apiv1/gen/http/openapi3.json index cd629941..f2095930 100644 --- a/api/apiv1/gen/http/openapi3.json +++ b/api/apiv1/gen/http/openapi3.json @@ -12056,7 +12056,7 @@ }, "port": 0, "service_id": "analytics-service", - "service_type": "rag", + "service_type": "lakekeeper", "version": "latest" }, { @@ -12135,7 +12135,7 @@ }, "port": 0, "service_id": "analytics-service", - "service_type": "rag", + "service_type": "lakekeeper", "version": "latest" } ] @@ -12961,7 +12961,7 @@ }, "port": 0, "service_id": "analytics-service", - "service_type": "rag", + "service_type": "lakekeeper", "version": "latest" }, { @@ -13040,7 +13040,7 @@ }, "port": 0, "service_id": "analytics-service", - "service_type": "rag", + "service_type": "lakekeeper", "version": "latest" }, { @@ -13119,7 +13119,7 @@ }, "port": 0, "service_id": "analytics-service", - "service_type": "rag", + "service_type": "lakekeeper", "version": "latest" }, { @@ -13198,7 +13198,7 @@ }, "port": 0, "service_id": "analytics-service", - "service_type": "rag", + "service_type": "lakekeeper", "version": "latest" } ], @@ -14031,7 +14031,7 @@ }, "port": 0, "service_id": "76f9b8c0-4958-11f0-a489-3bb29577c696", - "service_type": "rag", + "service_type": "lakekeeper", "version": "latest" }, { @@ -14110,7 +14110,7 @@ }, "port": 0, "service_id": "76f9b8c0-4958-11f0-a489-3bb29577c696", - "service_type": "rag", + "service_type": "lakekeeper", "version": "latest" }, { @@ -14189,7 +14189,7 @@ }, "port": 0, "service_id": "76f9b8c0-4958-11f0-a489-3bb29577c696", - "service_type": "rag", + "service_type": "lakekeeper", "version": "latest" }, { @@ -14268,7 +14268,7 @@ }, "port": 0, "service_id": "76f9b8c0-4958-11f0-a489-3bb29577c696", - "service_type": "rag", + "service_type": "lakekeeper", "version": "latest" } ] @@ -14800,7 +14800,7 @@ }, "port": 0, "service_id": "76f9b8c0-4958-11f0-a489-3bb29577c696", - "service_type": "rag", + "service_type": "lakekeeper", "version": "latest" }, { @@ -14879,7 +14879,7 @@ }, "port": 0, "service_id": "76f9b8c0-4958-11f0-a489-3bb29577c696", - "service_type": "rag", + "service_type": "lakekeeper", "version": "latest" }, { @@ -14958,7 +14958,7 @@ }, "port": 0, "service_id": "76f9b8c0-4958-11f0-a489-3bb29577c696", - "service_type": "rag", + "service_type": "lakekeeper", "version": "latest" }, { @@ -15037,7 +15037,7 @@ }, "port": 0, "service_id": "76f9b8c0-4958-11f0-a489-3bb29577c696", - "service_type": "rag", + "service_type": "lakekeeper", "version": "latest" } ], @@ -15475,7 +15475,7 @@ }, "port": 0, "service_id": "76f9b8c0-4958-11f0-a489-3bb29577c696", - "service_type": "rag", + "service_type": "lakekeeper", "version": "latest" }, { @@ -15554,7 +15554,7 @@ }, "port": 0, "service_id": "76f9b8c0-4958-11f0-a489-3bb29577c696", - "service_type": "rag", + "service_type": "lakekeeper", "version": "latest" }, { @@ -15633,7 +15633,7 @@ }, "port": 0, "service_id": "76f9b8c0-4958-11f0-a489-3bb29577c696", - "service_type": "rag", + "service_type": "lakekeeper", "version": "latest" } ] @@ -16360,7 +16360,7 @@ }, "port": 0, "service_id": "76f9b8c0-4958-11f0-a489-3bb29577c696", - "service_type": "rag", + "service_type": "lakekeeper", "version": "latest" }, { @@ -16439,7 +16439,7 @@ }, "port": 0, "service_id": "76f9b8c0-4958-11f0-a489-3bb29577c696", - "service_type": "rag", + "service_type": "lakekeeper", "version": "latest" } ], @@ -16877,7 +16877,7 @@ }, "port": 0, "service_id": "76f9b8c0-4958-11f0-a489-3bb29577c696", - "service_type": "rag", + "service_type": "lakekeeper", "version": "latest" }, { @@ -16956,7 +16956,7 @@ }, "port": 0, "service_id": "76f9b8c0-4958-11f0-a489-3bb29577c696", - "service_type": "rag", + "service_type": "lakekeeper", "version": "latest" }, { @@ -17035,7 +17035,7 @@ }, "port": 0, "service_id": "76f9b8c0-4958-11f0-a489-3bb29577c696", - "service_type": "rag", + "service_type": "lakekeeper", "version": "latest" }, { @@ -17114,7 +17114,7 @@ }, "port": 0, "service_id": "76f9b8c0-4958-11f0-a489-3bb29577c696", - "service_type": "rag", + "service_type": "lakekeeper", "version": "latest" } ] @@ -18037,7 +18037,7 @@ }, "port": 0, "service_id": "76f9b8c0-4958-11f0-a489-3bb29577c696", - "service_type": "rag", + "service_type": "lakekeeper", "version": "latest" }, { @@ -18116,7 +18116,7 @@ }, "port": 0, "service_id": "76f9b8c0-4958-11f0-a489-3bb29577c696", - "service_type": "rag", + "service_type": "lakekeeper", "version": "latest" }, { @@ -18195,7 +18195,7 @@ }, "port": 0, "service_id": "76f9b8c0-4958-11f0-a489-3bb29577c696", - "service_type": "rag", + "service_type": "lakekeeper", "version": "latest" }, { @@ -18274,7 +18274,7 @@ }, "port": 0, "service_id": "76f9b8c0-4958-11f0-a489-3bb29577c696", - "service_type": "rag", + "service_type": "lakekeeper", "version": "latest" } ], @@ -18711,7 +18711,7 @@ }, "port": 0, "service_id": "76f9b8c0-4958-11f0-a489-3bb29577c696", - "service_type": "rag", + "service_type": "lakekeeper", "version": "latest" }, { @@ -18789,7 +18789,7 @@ }, "port": 0, "service_id": "76f9b8c0-4958-11f0-a489-3bb29577c696", - "service_type": "rag", + "service_type": "lakekeeper", "version": "latest" }, { @@ -18867,7 +18867,7 @@ }, "port": 0, "service_id": "76f9b8c0-4958-11f0-a489-3bb29577c696", - "service_type": "rag", + "service_type": "lakekeeper", "version": "latest" }, { @@ -18945,7 +18945,7 @@ }, "port": 0, "service_id": "76f9b8c0-4958-11f0-a489-3bb29577c696", - "service_type": "rag", + "service_type": "lakekeeper", "version": "latest" } ] @@ -19475,7 +19475,7 @@ }, "port": 0, "service_id": "76f9b8c0-4958-11f0-a489-3bb29577c696", - "service_type": "rag", + "service_type": "lakekeeper", "version": "latest" }, { @@ -19553,7 +19553,7 @@ }, "port": 0, "service_id": "76f9b8c0-4958-11f0-a489-3bb29577c696", - "service_type": "rag", + "service_type": "lakekeeper", "version": "latest" } ], @@ -19991,7 +19991,7 @@ }, "port": 0, "service_id": "76f9b8c0-4958-11f0-a489-3bb29577c696", - "service_type": "rag", + "service_type": "lakekeeper", "version": "latest" }, { @@ -20070,7 +20070,7 @@ }, "port": 0, "service_id": "76f9b8c0-4958-11f0-a489-3bb29577c696", - "service_type": "rag", + "service_type": "lakekeeper", "version": "latest" }, { @@ -20149,7 +20149,7 @@ }, "port": 0, "service_id": "76f9b8c0-4958-11f0-a489-3bb29577c696", - "service_type": "rag", + "service_type": "lakekeeper", "version": "latest" }, { @@ -20228,7 +20228,7 @@ }, "port": 0, "service_id": "76f9b8c0-4958-11f0-a489-3bb29577c696", - "service_type": "rag", + "service_type": "lakekeeper", "version": "latest" } ] @@ -20955,7 +20955,7 @@ }, "port": 0, "service_id": "76f9b8c0-4958-11f0-a489-3bb29577c696", - "service_type": "rag", + "service_type": "lakekeeper", "version": "latest" }, { @@ -21034,7 +21034,7 @@ }, "port": 0, "service_id": "76f9b8c0-4958-11f0-a489-3bb29577c696", - "service_type": "rag", + "service_type": "lakekeeper", "version": "latest" }, { @@ -21113,7 +21113,7 @@ }, "port": 0, "service_id": "76f9b8c0-4958-11f0-a489-3bb29577c696", - "service_type": "rag", + "service_type": "lakekeeper", "version": "latest" } ], @@ -21943,7 +21943,7 @@ }, "port": 0, "service_id": "76f9b8c0-4958-11f0-a489-3bb29577c696", - "service_type": "rag", + "service_type": "lakekeeper", "version": "latest" }, { @@ -22022,7 +22022,7 @@ }, "port": 0, "service_id": "76f9b8c0-4958-11f0-a489-3bb29577c696", - "service_type": "rag", + "service_type": "lakekeeper", "version": "latest" } ] @@ -22945,7 +22945,7 @@ }, "port": 0, "service_id": "76f9b8c0-4958-11f0-a489-3bb29577c696", - "service_type": "rag", + "service_type": "lakekeeper", "version": "latest" }, { @@ -23024,7 +23024,7 @@ }, "port": 0, "service_id": "76f9b8c0-4958-11f0-a489-3bb29577c696", - "service_type": "rag", + "service_type": "lakekeeper", "version": "latest" } ], @@ -26890,11 +26890,12 @@ "service_type": { "type": "string", "description": "The type of service to run.", - "example": "rag", + "example": "lakekeeper", "enum": [ "mcp", "postgrest", - "rag" + "rag", + "lakekeeper" ] }, "version": { @@ -26979,7 +26980,7 @@ }, "port": 0, "service_id": "analytics-service", - "service_type": "rag", + "service_type": "lakekeeper", "version": "latest" }, "required": [ @@ -27059,11 +27060,12 @@ "service_type": { "type": "string", "description": "The type of service to run.", - "example": "rag", + "example": "lakekeeper", "enum": [ "mcp", "postgrest", - "rag" + "rag", + "lakekeeper" ] }, "version": { @@ -27150,7 +27152,7 @@ }, "port": 0, "service_id": "76f9b8c0-4958-11f0-a489-3bb29577c696", - "service_type": "rag", + "service_type": "lakekeeper", "version": "latest" }, "required": [ @@ -27230,11 +27232,12 @@ "service_type": { "type": "string", "description": "The type of service to run.", - "example": "rag", + "example": "lakekeeper", "enum": [ "mcp", "postgrest", - "rag" + "rag", + "lakekeeper" ] }, "version": { @@ -27320,7 +27323,7 @@ }, "port": 0, "service_id": "76f9b8c0-4958-11f0-a489-3bb29577c696", - "service_type": "rag", + "service_type": "lakekeeper", "version": "latest" }, "required": [ @@ -27399,11 +27402,12 @@ "service_type": { "type": "string", "description": "The type of service to run.", - "example": "rag", + "example": "lakekeeper", "enum": [ "mcp", "postgrest", - "rag" + "rag", + "lakekeeper" ] }, "version": { @@ -27490,7 +27494,7 @@ }, "port": 0, "service_id": "76f9b8c0-4958-11f0-a489-3bb29577c696", - "service_type": "rag", + "service_type": "lakekeeper", "version": "latest" }, "required": [ @@ -27569,11 +27573,12 @@ "service_type": { "type": "string", "description": "The type of service to run.", - "example": "rag", + "example": "lakekeeper", "enum": [ "mcp", "postgrest", - "rag" + "rag", + "lakekeeper" ] }, "version": { @@ -27658,7 +27663,7 @@ }, "port": 0, "service_id": "76f9b8c0-4958-11f0-a489-3bb29577c696", - "service_type": "rag", + "service_type": "lakekeeper", "version": "latest" }, "required": [ @@ -27738,11 +27743,12 @@ "service_type": { "type": "string", "description": "The type of service to run.", - "example": "rag", + "example": "lakekeeper", "enum": [ "mcp", "postgrest", - "rag" + "rag", + "lakekeeper" ] }, "version": { @@ -27829,7 +27835,7 @@ }, "port": 0, "service_id": "76f9b8c0-4958-11f0-a489-3bb29577c696", - "service_type": "rag", + "service_type": "lakekeeper", "version": "latest" }, "required": [ @@ -27908,11 +27914,12 @@ "service_type": { "type": "string", "description": "The type of service to run.", - "example": "rag", + "example": "lakekeeper", "enum": [ "mcp", "postgrest", - "rag" + "rag", + "lakekeeper" ] }, "version": { @@ -27998,7 +28005,7 @@ }, "port": 0, "service_id": "76f9b8c0-4958-11f0-a489-3bb29577c696", - "service_type": "rag", + "service_type": "lakekeeper", "version": "latest" }, "required": [ diff --git a/api/apiv1/gen/http/openapi3.yaml b/api/apiv1/gen/http/openapi3.yaml index 98efeaf1..839d96fb 100644 --- a/api/apiv1/gen/http/openapi3.yaml +++ b/api/apiv1/gen/http/openapi3.yaml @@ -8464,7 +8464,7 @@ components: host_path: /Users/user/backups/host port: 0 service_id: analytics-service - service_type: rag + service_type: lakekeeper version: latest - config: llm_model: gpt-4 @@ -8514,7 +8514,7 @@ components: host_path: /Users/user/backups/host port: 0 service_id: analytics-service - service_type: rag + service_type: lakekeeper version: latest spock_version: type: string @@ -9101,7 +9101,7 @@ components: host_path: /Users/user/backups/host port: 0 service_id: analytics-service - service_type: rag + service_type: lakekeeper version: latest - config: llm_model: gpt-4 @@ -9151,7 +9151,7 @@ components: host_path: /Users/user/backups/host port: 0 service_id: analytics-service - service_type: rag + service_type: lakekeeper version: latest - config: llm_model: gpt-4 @@ -9201,7 +9201,7 @@ components: host_path: /Users/user/backups/host port: 0 service_id: analytics-service - service_type: rag + service_type: lakekeeper version: latest - config: llm_model: gpt-4 @@ -9251,7 +9251,7 @@ components: host_path: /Users/user/backups/host port: 0 service_id: analytics-service - service_type: rag + service_type: lakekeeper version: latest spock_version: "5" required: @@ -9861,7 +9861,7 @@ components: host_path: /Users/user/backups/host port: 0 service_id: 76f9b8c0-4958-11f0-a489-3bb29577c696 - service_type: rag + service_type: lakekeeper version: latest - config: llm_model: gpt-4 @@ -9911,7 +9911,7 @@ components: host_path: /Users/user/backups/host port: 0 service_id: 76f9b8c0-4958-11f0-a489-3bb29577c696 - service_type: rag + service_type: lakekeeper version: latest - config: llm_model: gpt-4 @@ -9961,7 +9961,7 @@ components: host_path: /Users/user/backups/host port: 0 service_id: 76f9b8c0-4958-11f0-a489-3bb29577c696 - service_type: rag + service_type: lakekeeper version: latest - config: llm_model: gpt-4 @@ -10011,7 +10011,7 @@ components: host_path: /Users/user/backups/host port: 0 service_id: 76f9b8c0-4958-11f0-a489-3bb29577c696 - service_type: rag + service_type: lakekeeper version: latest spock_version: type: string @@ -10394,7 +10394,7 @@ components: host_path: /Users/user/backups/host port: 0 service_id: 76f9b8c0-4958-11f0-a489-3bb29577c696 - service_type: rag + service_type: lakekeeper version: latest - config: llm_model: gpt-4 @@ -10444,7 +10444,7 @@ components: host_path: /Users/user/backups/host port: 0 service_id: 76f9b8c0-4958-11f0-a489-3bb29577c696 - service_type: rag + service_type: lakekeeper version: latest - config: llm_model: gpt-4 @@ -10494,7 +10494,7 @@ components: host_path: /Users/user/backups/host port: 0 service_id: 76f9b8c0-4958-11f0-a489-3bb29577c696 - service_type: rag + service_type: lakekeeper version: latest - config: llm_model: gpt-4 @@ -10544,7 +10544,7 @@ components: host_path: /Users/user/backups/host port: 0 service_id: 76f9b8c0-4958-11f0-a489-3bb29577c696 - service_type: rag + service_type: lakekeeper version: latest spock_version: "5" required: @@ -10861,7 +10861,7 @@ components: host_path: /Users/user/backups/host port: 0 service_id: 76f9b8c0-4958-11f0-a489-3bb29577c696 - service_type: rag + service_type: lakekeeper version: latest - config: llm_model: gpt-4 @@ -10911,7 +10911,7 @@ components: host_path: /Users/user/backups/host port: 0 service_id: 76f9b8c0-4958-11f0-a489-3bb29577c696 - service_type: rag + service_type: lakekeeper version: latest - config: llm_model: gpt-4 @@ -10961,7 +10961,7 @@ components: host_path: /Users/user/backups/host port: 0 service_id: 76f9b8c0-4958-11f0-a489-3bb29577c696 - service_type: rag + service_type: lakekeeper version: latest spock_version: type: string @@ -11488,7 +11488,7 @@ components: host_path: /Users/user/backups/host port: 0 service_id: 76f9b8c0-4958-11f0-a489-3bb29577c696 - service_type: rag + service_type: lakekeeper version: latest - config: llm_model: gpt-4 @@ -11538,7 +11538,7 @@ components: host_path: /Users/user/backups/host port: 0 service_id: 76f9b8c0-4958-11f0-a489-3bb29577c696 - service_type: rag + service_type: lakekeeper version: latest spock_version: "5" required: @@ -11855,7 +11855,7 @@ components: host_path: /Users/user/backups/host port: 0 service_id: 76f9b8c0-4958-11f0-a489-3bb29577c696 - service_type: rag + service_type: lakekeeper version: latest - config: llm_model: gpt-4 @@ -11905,7 +11905,7 @@ components: host_path: /Users/user/backups/host port: 0 service_id: 76f9b8c0-4958-11f0-a489-3bb29577c696 - service_type: rag + service_type: lakekeeper version: latest - config: llm_model: gpt-4 @@ -11955,7 +11955,7 @@ components: host_path: /Users/user/backups/host port: 0 service_id: 76f9b8c0-4958-11f0-a489-3bb29577c696 - service_type: rag + service_type: lakekeeper version: latest - config: llm_model: gpt-4 @@ -12005,7 +12005,7 @@ components: host_path: /Users/user/backups/host port: 0 service_id: 76f9b8c0-4958-11f0-a489-3bb29577c696 - service_type: rag + service_type: lakekeeper version: latest spock_version: type: string @@ -12677,7 +12677,7 @@ components: host_path: /Users/user/backups/host port: 0 service_id: 76f9b8c0-4958-11f0-a489-3bb29577c696 - service_type: rag + service_type: lakekeeper version: latest - config: llm_model: gpt-4 @@ -12727,7 +12727,7 @@ components: host_path: /Users/user/backups/host port: 0 service_id: 76f9b8c0-4958-11f0-a489-3bb29577c696 - service_type: rag + service_type: lakekeeper version: latest - config: llm_model: gpt-4 @@ -12777,7 +12777,7 @@ components: host_path: /Users/user/backups/host port: 0 service_id: 76f9b8c0-4958-11f0-a489-3bb29577c696 - service_type: rag + service_type: lakekeeper version: latest - config: llm_model: gpt-4 @@ -12827,7 +12827,7 @@ components: host_path: /Users/user/backups/host port: 0 service_id: 76f9b8c0-4958-11f0-a489-3bb29577c696 - service_type: rag + service_type: lakekeeper version: latest spock_version: "5" required: @@ -13143,7 +13143,7 @@ components: host_path: /Users/user/backups/host port: 0 service_id: 76f9b8c0-4958-11f0-a489-3bb29577c696 - service_type: rag + service_type: lakekeeper version: latest - config: llm_model: gpt-4 @@ -13192,7 +13192,7 @@ components: host_path: /Users/user/backups/host port: 0 service_id: 76f9b8c0-4958-11f0-a489-3bb29577c696 - service_type: rag + service_type: lakekeeper version: latest - config: llm_model: gpt-4 @@ -13241,7 +13241,7 @@ components: host_path: /Users/user/backups/host port: 0 service_id: 76f9b8c0-4958-11f0-a489-3bb29577c696 - service_type: rag + service_type: lakekeeper version: latest - config: llm_model: gpt-4 @@ -13290,7 +13290,7 @@ components: host_path: /Users/user/backups/host port: 0 service_id: 76f9b8c0-4958-11f0-a489-3bb29577c696 - service_type: rag + service_type: lakekeeper version: latest spock_version: type: string @@ -13671,7 +13671,7 @@ components: host_path: /Users/user/backups/host port: 0 service_id: 76f9b8c0-4958-11f0-a489-3bb29577c696 - service_type: rag + service_type: lakekeeper version: latest - config: llm_model: gpt-4 @@ -13720,7 +13720,7 @@ components: host_path: /Users/user/backups/host port: 0 service_id: 76f9b8c0-4958-11f0-a489-3bb29577c696 - service_type: rag + service_type: lakekeeper version: latest spock_version: "5" required: @@ -14037,7 +14037,7 @@ components: host_path: /Users/user/backups/host port: 0 service_id: 76f9b8c0-4958-11f0-a489-3bb29577c696 - service_type: rag + service_type: lakekeeper version: latest - config: llm_model: gpt-4 @@ -14087,7 +14087,7 @@ components: host_path: /Users/user/backups/host port: 0 service_id: 76f9b8c0-4958-11f0-a489-3bb29577c696 - service_type: rag + service_type: lakekeeper version: latest - config: llm_model: gpt-4 @@ -14137,7 +14137,7 @@ components: host_path: /Users/user/backups/host port: 0 service_id: 76f9b8c0-4958-11f0-a489-3bb29577c696 - service_type: rag + service_type: lakekeeper version: latest - config: llm_model: gpt-4 @@ -14187,7 +14187,7 @@ components: host_path: /Users/user/backups/host port: 0 service_id: 76f9b8c0-4958-11f0-a489-3bb29577c696 - service_type: rag + service_type: lakekeeper version: latest spock_version: type: string @@ -14714,7 +14714,7 @@ components: host_path: /Users/user/backups/host port: 0 service_id: 76f9b8c0-4958-11f0-a489-3bb29577c696 - service_type: rag + service_type: lakekeeper version: latest - config: llm_model: gpt-4 @@ -14764,7 +14764,7 @@ components: host_path: /Users/user/backups/host port: 0 service_id: 76f9b8c0-4958-11f0-a489-3bb29577c696 - service_type: rag + service_type: lakekeeper version: latest - config: llm_model: gpt-4 @@ -14814,7 +14814,7 @@ components: host_path: /Users/user/backups/host port: 0 service_id: 76f9b8c0-4958-11f0-a489-3bb29577c696 - service_type: rag + service_type: lakekeeper version: latest spock_version: "5" required: @@ -15421,7 +15421,7 @@ components: host_path: /Users/user/backups/host port: 0 service_id: 76f9b8c0-4958-11f0-a489-3bb29577c696 - service_type: rag + service_type: lakekeeper version: latest - config: llm_model: gpt-4 @@ -15471,7 +15471,7 @@ components: host_path: /Users/user/backups/host port: 0 service_id: 76f9b8c0-4958-11f0-a489-3bb29577c696 - service_type: rag + service_type: lakekeeper version: latest spock_version: type: string @@ -16143,7 +16143,7 @@ components: host_path: /Users/user/backups/host port: 0 service_id: 76f9b8c0-4958-11f0-a489-3bb29577c696 - service_type: rag + service_type: lakekeeper version: latest - config: llm_model: gpt-4 @@ -16193,7 +16193,7 @@ components: host_path: /Users/user/backups/host port: 0 service_id: 76f9b8c0-4958-11f0-a489-3bb29577c696 - service_type: rag + service_type: lakekeeper version: latest spock_version: "5" required: @@ -19004,11 +19004,12 @@ components: service_type: type: string description: The type of service to run. - example: rag + example: lakekeeper enum: - mcp - postgrest - rag + - lakekeeper version: type: string description: The version of the service (e.g., '1.0.0', '14.5') or the literal 'latest'. @@ -19062,7 +19063,7 @@ components: host_path: /Users/user/backups/host port: 0 service_id: analytics-service - service_type: rag + service_type: lakekeeper version: latest required: - service_id @@ -19127,11 +19128,12 @@ components: service_type: type: string description: The type of service to run. - example: rag + example: lakekeeper enum: - mcp - postgrest - rag + - lakekeeper version: type: string description: The version of the service (e.g., '1.0.0', '14.5') or the literal 'latest'. @@ -19187,7 +19189,7 @@ components: host_path: /Users/user/backups/host port: 0 service_id: 76f9b8c0-4958-11f0-a489-3bb29577c696 - service_type: rag + service_type: lakekeeper version: latest required: - service_id @@ -19252,11 +19254,12 @@ components: service_type: type: string description: The type of service to run. - example: rag + example: lakekeeper enum: - mcp - postgrest - rag + - lakekeeper version: type: string description: The version of the service (e.g., '1.0.0', '14.5') or the literal 'latest'. @@ -19311,7 +19314,7 @@ components: host_path: /Users/user/backups/host port: 0 service_id: 76f9b8c0-4958-11f0-a489-3bb29577c696 - service_type: rag + service_type: lakekeeper version: latest required: - service_id @@ -19375,11 +19378,12 @@ components: service_type: type: string description: The type of service to run. - example: rag + example: lakekeeper enum: - mcp - postgrest - rag + - lakekeeper version: type: string description: The version of the service (e.g., '1.0.0', '14.5') or the literal 'latest'. @@ -19435,7 +19439,7 @@ components: host_path: /Users/user/backups/host port: 0 service_id: 76f9b8c0-4958-11f0-a489-3bb29577c696 - service_type: rag + service_type: lakekeeper version: latest required: - service_id @@ -19499,11 +19503,12 @@ components: service_type: type: string description: The type of service to run. - example: rag + example: lakekeeper enum: - mcp - postgrest - rag + - lakekeeper version: type: string description: The version of the service (e.g., '1.0.0', '14.5') or the literal 'latest'. @@ -19557,7 +19562,7 @@ components: host_path: /Users/user/backups/host port: 0 service_id: 76f9b8c0-4958-11f0-a489-3bb29577c696 - service_type: rag + service_type: lakekeeper version: latest required: - service_id @@ -19622,11 +19627,12 @@ components: service_type: type: string description: The type of service to run. - example: rag + example: lakekeeper enum: - mcp - postgrest - rag + - lakekeeper version: type: string description: The version of the service (e.g., '1.0.0', '14.5') or the literal 'latest'. @@ -19682,7 +19688,7 @@ components: host_path: /Users/user/backups/host port: 0 service_id: 76f9b8c0-4958-11f0-a489-3bb29577c696 - service_type: rag + service_type: lakekeeper version: latest required: - service_id @@ -19746,11 +19752,12 @@ components: service_type: type: string description: The type of service to run. - example: rag + example: lakekeeper enum: - mcp - postgrest - rag + - lakekeeper version: type: string description: The version of the service (e.g., '1.0.0', '14.5') or the literal 'latest'. @@ -19805,7 +19812,7 @@ components: host_path: /Users/user/backups/host port: 0 service_id: 76f9b8c0-4958-11f0-a489-3bb29577c696 - service_type: rag + service_type: lakekeeper version: latest required: - service_id diff --git a/server/internal/api/apiv1/validate.go b/server/internal/api/apiv1/validate.go index 947d94bc..e1b0537b 100644 --- a/server/internal/api/apiv1/validate.go +++ b/server/internal/api/apiv1/validate.go @@ -168,6 +168,9 @@ func validateDatabaseSpec(orchestrator config.Orchestrator, databaseID string, s } seenServiceIDs.Add(string(svc.ServiceID)) + // ColdFront (a lakekeeper service) is single-node only for now. + errs = append(errs, validateColdFrontSingleNode(svc, len(spec.Nodes), svcPath)...) + errs = append(errs, validateServiceSpec(svc, svcPath, false, databaseID, spec.DatabaseUsers, seenNodeNames)...) } } @@ -231,6 +234,9 @@ func validateDatabaseUpdate(old *database.Spec, new *api.DatabaseSpec) error { svcPath := validation.NewPath("services", validation.ArrayIndexElement(i)) isExistingService := existingServiceIDs.Has(string(svc.ServiceID)) + // ColdFront (a lakekeeper service) is single-node only for now. + errs = append(errs, validateColdFrontSingleNode(svc, len(new.Nodes), svcPath)...) + errs = append(errs, validateServiceSpec(svc, svcPath, isExistingService, old.DatabaseID, new.DatabaseUsers, newNodeNames)...) } @@ -325,7 +331,7 @@ func validateServiceSpec(svc *api.ServiceSpec, path validation.Path, isUpdate bo } // Validate service_type allowlist - supportedServiceTypes := validation.NewPath("mcp", "postgrest", "rag") + supportedServiceTypes := validation.NewPath("mcp", "postgrest", "rag", "lakekeeper") if !slices.Contains(supportedServiceTypes, svc.ServiceType) { err := fmt.Errorf("unsupported service type %q (supported: %s)", svc.ServiceType, strings.Join(supportedServiceTypes, ", ")) @@ -362,6 +368,8 @@ func validateServiceSpec(svc *api.ServiceSpec, path validation.Path, isUpdate bo errs = append(errs, validatePostgRESTServiceConfig(svc.Config, path.Append("config"))...) case "rag": errs = append(errs, validateRAGServiceConfig(svc.Config, path.Append("config"), isUpdate)...) + case "lakekeeper": + errs = append(errs, validateLakekeeperServiceConfig(svc.Config, path.Append("config"))...) } // Validate database_connection if provided @@ -486,6 +494,98 @@ func validateRAGServiceConfig(config map[string]any, path validation.Path, isUpd return result } +// coldFrontMultiNodeError is the message returned when a lakekeeper (ColdFront) +// service is requested on a database that spans more than one node. Kept as a +// package-level value so the orchestrator-side guard can return an identical +// message. +const coldFrontMultiNodeError = "coldfront: multi-node ColdFront is not yet supported " + + "(mesh snowflake.node alignment pending); enable ColdFront only on a single-node database" + +// validateColdFrontSingleNode rejects a lakekeeper service on a multi-node +// database. ColdFront's tiering bakery requires +// snowflake.node = hashtext(spock_node_name)&1023, but control-plane currently +// assigns node ordinals rather than reconciling that mesh GUC, so a multi-node +// deployment would silently fail its tiering cron jobs. Fail loudly at +// validation time until the mesh snowflake.node reconciliation lands. +func validateColdFrontSingleNode(svc *api.ServiceSpec, nodeCount int, path validation.Path) []error { + if svc.ServiceType != "lakekeeper" { + return nil + } + if nodeCount <= 1 { + return nil + } + return []error{validation.NewError(errors.New(coldFrontMultiNodeError), path.Append("service_type"))} +} + +// validateLakekeeperServiceConfig checks that the two required catalog +// configuration keys are present and non-empty. An absent or empty +// catalog_db_url would cause Lakekeeper to start with a blank connection +// string and crash-loop silently; we surface the error at spec-validation +// time instead. +func validateLakekeeperServiceConfig(config map[string]any, path validation.Path) []error { + var errs []error + + catalogDBURL, _ := config["catalog_db_url"].(string) + if catalogDBURL == "" { + errs = append(errs, validation.NewError( + errors.New("catalog_db_url is required: provide the connection URL for the external Lakekeeper catalog Postgres"), + path.Append("catalog_db_url"), + )) + } + + pgEncryptionKey, _ := config["pg_encryption_key"].(string) + if pgEncryptionKey == "" { + errs = append(errs, validation.NewError( + errors.New("pg_encryption_key is required: provide the encryption key for Lakekeeper's catalog Postgres"), + path.Append("pg_encryption_key"), + )) + } + + // The warehouse bootstrap and coldfront.set_storage_secret both need the + // object-store coordinates and a provider-specific credential. Fail loud + // here so a database is never left with an unbootstrapped (broken) + // warehouse. Some of these keys are a saas follow-up. + provider, _ := config["provider"].(string) + switch provider { + case "aws", "azure", "gcs": + case "": + errs = append(errs, validation.NewError( + errors.New("provider is required: one of aws, azure, gcs"), + path.Append("provider"), + )) + default: + errs = append(errs, validation.NewError( + fmt.Errorf("unsupported provider %q: expected one of aws, azure, gcs", provider), + path.Append("provider"), + )) + } + + if warehouse, _ := config["warehouse"].(string); warehouse == "" { + errs = append(errs, validation.NewError( + errors.New("warehouse is required: the warehouse name to create in Lakekeeper"), + path.Append("warehouse"), + )) + } + + if credential, _ := config["credential"].(string); credential == "" { + errs = append(errs, validation.NewError( + errors.New("credential is required: a provider-specific credential JSON string"), + path.Append("credential"), + )) + } + + if provider == "aws" || provider == "gcs" { + if bucket, _ := config["bucket"].(string); bucket == "" { + errs = append(errs, validation.NewError( + errors.New("bucket is required for aws and gcs providers"), + path.Append("bucket"), + )) + } + } + + return errs +} + func validateCPUs(value *string, path validation.Path) []error { var errs []error diff --git a/server/internal/api/apiv1/validate_test.go b/server/internal/api/apiv1/validate_test.go index b43aa701..9cb20fa4 100644 --- a/server/internal/api/apiv1/validate_test.go +++ b/server/internal/api/apiv1/validate_test.go @@ -1307,6 +1307,80 @@ func TestValidateDatabaseSpec(t *testing.T) { `nodes[0].postgresql_conf[HBA_FILE]: "HBA_FILE" is not allowed`, }, }, + { + name: "valid single-node ColdFront (lakekeeper)", + spec: &api.DatabaseSpec{ + DatabaseName: "testdb", + PostgresVersion: utils.PointerTo("17.6"), + Nodes: []*api.DatabaseNodeSpec{ + { + Name: "n1", + HostIds: []api.Identifier{api.Identifier("host-1")}, + }, + }, + DatabaseUsers: []*api.DatabaseUserSpec{ + {Username: "app", DbOwner: utils.PointerTo(true)}, + }, + Services: []*api.ServiceSpec{ + { + ServiceID: "coldfront", + ServiceType: "lakekeeper", + Version: "0.9.0", + HostIds: []api.Identifier{"host-1"}, + ConnectAs: "app", + Config: map[string]any{ + "catalog_db_url": "postgres://lk:secret@catalog:5432/lakekeeper?sslmode=disable", + "pg_encryption_key": "dGVzdGtleXRlc3RrZXl0ZXN0a2V5dGVzdA==", + "provider": "aws", + "warehouse": "analytics", + "credential": `{"aws_access_key_id":"AKIA...","aws_secret_access_key":"..."}`, + "bucket": "coldfront-warehouse", + }, + }, + }, + }, + }, + { + name: "invalid multi-node ColdFront (lakekeeper) is rejected", + spec: &api.DatabaseSpec{ + DatabaseName: "testdb", + PostgresVersion: utils.PointerTo("17.6"), + Nodes: []*api.DatabaseNodeSpec{ + { + Name: "n1", + HostIds: []api.Identifier{api.Identifier("host-1")}, + }, + { + Name: "n2", + HostIds: []api.Identifier{api.Identifier("host-2")}, + }, + }, + DatabaseUsers: []*api.DatabaseUserSpec{ + {Username: "app", DbOwner: utils.PointerTo(true)}, + }, + Services: []*api.ServiceSpec{ + { + ServiceID: "coldfront", + ServiceType: "lakekeeper", + Version: "0.9.0", + HostIds: []api.Identifier{"host-1"}, + ConnectAs: "app", + Config: map[string]any{ + "catalog_db_url": "postgres://lk:secret@catalog:5432/lakekeeper?sslmode=disable", + "pg_encryption_key": "dGVzdGtleXRlc3RrZXl0ZXN0a2V5dGVzdA==", + "provider": "aws", + "warehouse": "analytics", + "credential": `{"aws_access_key_id":"AKIA...","aws_secret_access_key":"..."}`, + "bucket": "coldfront-warehouse", + }, + }, + }, + }, + expected: []string{ + "multi-node ColdFront is not yet supported", + "mesh snowflake.node alignment pending", + }, + }, } { t.Run(tc.name, func(t *testing.T) { err := validateDatabaseSpec(config.OrchestratorSwarm, "test-db", tc.spec) @@ -1477,6 +1551,101 @@ func TestValidateServiceSpec(t *testing.T) { "pipelines is required", }, }, + { + name: "lakekeeper with nil config fails — catalog_db_url required", + svc: &api.ServiceSpec{ + ServiceID: "my-lakekeeper", + ServiceType: "lakekeeper", + Version: "0.9.0", + HostIds: []api.Identifier{"host-1"}, + }, + expected: []string{ + "catalog_db_url is required", + "pg_encryption_key is required", + }, + }, + { + name: "lakekeeper with empty config fails — catalog_db_url required", + svc: &api.ServiceSpec{ + ServiceID: "my-lakekeeper", + ServiceType: "lakekeeper", + Version: "0.9.0", + HostIds: []api.Identifier{"host-1"}, + Config: map[string]any{}, + }, + expected: []string{ + "catalog_db_url is required", + "pg_encryption_key is required", + }, + }, + { + name: "lakekeeper missing pg_encryption_key fails", + svc: &api.ServiceSpec{ + ServiceID: "my-lakekeeper", + ServiceType: "lakekeeper", + Version: "0.9.0", + HostIds: []api.Identifier{"host-1"}, + Config: map[string]any{ + "catalog_db_url": "postgres://lakekeeper:secret@pg-host:5432/lakekeeper?sslmode=disable", + }, + }, + expected: []string{ + "pg_encryption_key is required", + }, + }, + { + name: "lakekeeper missing warehouse bootstrap keys fails", + svc: &api.ServiceSpec{ + ServiceID: "my-lakekeeper", + ServiceType: "lakekeeper", + Version: "0.9.0", + HostIds: []api.Identifier{"host-1"}, + Config: map[string]any{ + "catalog_db_url": "postgres://lakekeeper:secret@pg-host:5432/lakekeeper?sslmode=disable", + "pg_encryption_key": "dGVzdGtleXRlc3RrZXl0ZXN0a2V5dGVzdA==", + }, + }, + expected: []string{ + "provider is required", + "warehouse is required", + "credential is required", + }, + }, + { + name: "lakekeeper aws missing bucket fails", + svc: &api.ServiceSpec{ + ServiceID: "my-lakekeeper", + ServiceType: "lakekeeper", + Version: "0.9.0", + HostIds: []api.Identifier{"host-1"}, + Config: map[string]any{ + "catalog_db_url": "postgres://lakekeeper:secret@pg-host:5432/lakekeeper?sslmode=disable", + "pg_encryption_key": "dGVzdGtleXRlc3RrZXl0ZXN0a2V5dGVzdA==", + "provider": "aws", + "warehouse": "wh1", + "credential": `{"access_key_id":"a","secret_access_key":"s"}`, + }, + }, + expected: []string{"bucket is required for aws"}, + }, + { + name: "valid lakekeeper with all required config keys", + svc: &api.ServiceSpec{ + ServiceID: "my-lakekeeper", + ServiceType: "lakekeeper", + Version: "0.9.0", + HostIds: []api.Identifier{"host-1"}, + Config: map[string]any{ + "catalog_db_url": "postgres://lakekeeper:secret@pg-host:5432/lakekeeper?sslmode=disable", + "pg_encryption_key": "dGVzdGtleXRlc3RrZXl0ZXN0a2V5dGVzdA==", + "provider": "aws", + "bucket": "my-bucket", + "region": "us-east-1", + "warehouse": "wh1", + "credential": `{"access_key_id":"a","secret_access_key":"s"}`, + }, + }, + }, { name: "postgrest with empty config fails — db_anon_role required", svc: &api.ServiceSpec{ diff --git a/server/internal/orchestrator/swarm/coldfront_tiering_wiring_test.go b/server/internal/orchestrator/swarm/coldfront_tiering_wiring_test.go new file mode 100644 index 00000000..46209cc7 --- /dev/null +++ b/server/internal/orchestrator/swarm/coldfront_tiering_wiring_test.go @@ -0,0 +1,105 @@ +package swarm + +import ( + "testing" + + "github.com/pgEdge/control-plane/server/internal/database" + "github.com/pgEdge/control-plane/server/internal/resource" + "github.com/pgEdge/control-plane/server/internal/scheduler" +) + +// makeLakekeeperSpecWithStorage extends makeLakekeeperSpec with the storage +// config required to produce tiering schedule resources. +func makeLakekeeperSpecWithStorage() *database.ServiceInstanceSpec { + cfg := map[string]any{ + "catalog_db_url": "postgres://lakekeeper:secret@pg:5432/lk?sslmode=disable", + "pg_encryption_key": "dGVzdGtleXRlc3RrZXl0ZXN0a2V5dGVzdA==", + "provider": "aws", + "warehouse": "s3://my-bucket/warehouse", + "bucket": "my-bucket", + "region": "us-east-1", + "credential": `{"access_key_id":"AKID","secret_access_key":"SECRET"}`, + } + return &database.ServiceInstanceSpec{ + ServiceInstanceID: "inst-lakekeeper-1", + ServiceSpec: &database.ServiceSpec{ + ServiceID: "lakekeeper-svc", + ServiceType: "lakekeeper", + Version: "0.9.0", + Config: cfg, + }, + DatabaseID: "db-1", + DatabaseName: "testdb", + HostID: "host-1", + NodeName: "n1", + } +} + +// TestGenerateLakekeeperInstanceResources_TieringSchedules verifies that when +// a lakekeeper service has storage config (provider + credential), the +// generated resource graph includes ScheduledJobResource entries for the +// archiver, partitioner, and compactor tiering jobs. +func TestGenerateLakekeeperInstanceResources_TieringSchedules(t *testing.T) { + o := newLakekeeperTestOrchestrator() + spec := makeLakekeeperSpecWithStorage() + + result, err := o.generateLakekeeperInstanceResources(spec) + if err != nil { + t.Fatalf("generateLakekeeperInstanceResources() unexpected error: %v", err) + } + + // Collect all ScheduledJobResource identifiers from the result. + var scheduledIDs []string + for _, rd := range result.Resources { + if rd.Identifier.Type == scheduler.ResourceTypeScheduledJob { + scheduledIDs = append(scheduledIDs, rd.Identifier.ID) + } + } + + wantWorkflows := []string{ + scheduler.WorkflowColdFrontArchive, + scheduler.WorkflowColdFrontPartition, + scheduler.WorkflowColdFrontCompact, + } + + // Decode each ScheduledJobResource and verify all three workflow types are present. + foundWorkflows := map[string]bool{} + for _, rd := range result.Resources { + if rd.Identifier.Type != scheduler.ResourceTypeScheduledJob { + continue + } + job, decErr := resource.ToResource[*scheduler.ScheduledJobResource](rd) + if decErr != nil { + t.Fatalf("failed to decode ScheduledJobResource %q: %v", rd.Identifier.ID, decErr) + } + foundWorkflows[job.Workflow] = true + } + + for _, wf := range wantWorkflows { + if !foundWorkflows[wf] { + t.Errorf("missing scheduled job for workflow %q; found: %v", wf, scheduledIDs) + } + } +} + +// TestGenerateLakekeeperInstanceResources_TieringSchedules_NoStorage verifies +// that when no storage config is present (provider absent), no tiering +// schedule resources are generated rather than returning an error. +func TestGenerateLakekeeperInstanceResources_TieringSchedules_NoStorage(t *testing.T) { + o := newLakekeeperTestOrchestrator() + spec := makeLakekeeperSpec( + "postgres://lakekeeper:secret@pg:5432/lk?sslmode=disable", + "dGVzdGtleXRlc3RrZXl0ZXN0a2V5dGVzdA==", + ) + + result, err := o.generateLakekeeperInstanceResources(spec) + if err != nil { + t.Fatalf("generateLakekeeperInstanceResources() unexpected error: %v", err) + } + + for _, rd := range result.Resources { + if rd.Identifier.Type == scheduler.ResourceTypeScheduledJob { + t.Errorf("expected no ScheduledJobResource when provider absent, got %q", rd.Identifier.ID) + } + } +} diff --git a/server/internal/orchestrator/swarm/lakekeeper_bootstrap.go b/server/internal/orchestrator/swarm/lakekeeper_bootstrap.go new file mode 100644 index 00000000..ba227547 --- /dev/null +++ b/server/internal/orchestrator/swarm/lakekeeper_bootstrap.go @@ -0,0 +1,350 @@ +package swarm + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "strings" + "time" +) + +// lakekeeperStorageConfig holds the object-store coordinates and parsed +// credential needed to build a Lakekeeper warehouse storage profile. It is +// derived from the lakekeeper ServiceSpec.Config supplied by saas. +// +// The Credential map is provider-specific and MUST NOT be logged: it carries +// secret access keys / connection strings. +type lakekeeperStorageConfig struct { + Provider string // "aws" | "azure" | "gcs" + Bucket string + Region string + Endpoint string + Warehouse string + PathPrefix string + // Credential is the parsed provider-specific credential JSON. + // aws: {"access_key_id","secret_access_key"} + // azure: {"connection_string"} + // gcs: {"hmac_access_id","hmac_secret"} + Credential map[string]string +} + +// lakekeeperBootstrapHTTPTimeout bounds each individual REST call to the +// Lakekeeper management/catalog API. +const lakekeeperBootstrapHTTPTimeout = 30 * time.Second + +// parseLakekeeperStorageConfig extracts the object-store configuration and +// parses the provider-specific credential JSON from a lakekeeper +// ServiceSpec.Config map. It fails loud when a required key is missing so that +// a database is never left with an unbootstrapped (broken) warehouse. +// +// Some of these keys are a saas follow-up; an absent required key therefore +// yields a clear, actionable error rather than a silent misconfiguration. +func parseLakekeeperStorageConfig(config map[string]any) (*lakekeeperStorageConfig, error) { + get := func(key string) string { + v, _ := config[key].(string) + return strings.TrimSpace(v) + } + + provider := get("provider") + switch provider { + case "aws", "azure", "gcs": + case "": + return nil, fmt.Errorf("lakekeeper bootstrap: provider is required in config (one of aws, azure, gcs)") + default: + return nil, fmt.Errorf("lakekeeper bootstrap: unsupported provider %q (expected one of aws, azure, gcs)", provider) + } + + warehouse := get("warehouse") + if warehouse == "" { + return nil, fmt.Errorf("lakekeeper bootstrap: warehouse is required in config") + } + + credRaw := get("credential") + if credRaw == "" { + return nil, fmt.Errorf("lakekeeper bootstrap: credential is required in config") + } + var cred map[string]string + if err := json.Unmarshal([]byte(credRaw), &cred); err != nil { + // Deliberately do not include credRaw in the error: it is a secret. + return nil, fmt.Errorf("lakekeeper bootstrap: credential is not valid JSON") + } + + cfg := &lakekeeperStorageConfig{ + Provider: provider, + Bucket: get("bucket"), + Region: get("region"), + Endpoint: get("endpoint"), + Warehouse: warehouse, + PathPrefix: get("path_prefix"), + Credential: cred, + } + + // Provider-specific required fields. + switch provider { + case "aws", "gcs": + if cfg.Bucket == "" { + return nil, fmt.Errorf("lakekeeper bootstrap: bucket is required in config for provider %q", provider) + } + if provider == "aws" { + if cred["access_key_id"] == "" || cred["secret_access_key"] == "" { + return nil, fmt.Errorf("lakekeeper bootstrap: aws credential must contain access_key_id and secret_access_key") + } + } else { // gcs + if cred["hmac_access_id"] == "" || cred["hmac_secret"] == "" { + return nil, fmt.Errorf("lakekeeper bootstrap: gcs credential must contain hmac_access_id and hmac_secret") + } + } + case "azure": + if cfg.Bucket == "" { + return nil, fmt.Errorf("lakekeeper bootstrap: bucket (container/account) is required in config for provider %q", provider) + } + if cred["connection_string"] == "" { + return nil, fmt.Errorf("lakekeeper bootstrap: azure credential must contain connection_string") + } + } + + return cfg, nil +} + +// buildWarehouseRequestBody builds the POST /management/v1/warehouse request +// body for the configured provider. +// +// For aws and gcs an s3 storage profile is used (gcs is reached through its +// S3-compatible HMAC interface). For azure an adls profile is used. +func buildWarehouseRequestBody(cfg *lakekeeperStorageConfig) (map[string]any, error) { + switch cfg.Provider { + case "aws", "gcs": + profile := map[string]any{ + "type": "s3", + "bucket": cfg.Bucket, + "region": cfg.Region, + "sts-enabled": false, + "remote-signing-enabled": false, + } + // Endpoint presence is the discriminator between native cloud AWS and + // an S3-compatible store (GCS via storage.googleapis.com, SeaweedFS, + // etc.). Cloud AWS must use virtual-hosted addressing (flavor "aws", + // path-style-access false); path-style fails on any Region launched + // after 2019. An S3-compatible endpoint uses flavor "s3-compat" with + // path-style addressing. (ColdFront docs/object_store.md, installation.md.) + if cfg.Endpoint != "" { + profile["endpoint"] = cfg.Endpoint + profile["flavor"] = "s3-compat" + profile["path-style-access"] = true + } else { + profile["flavor"] = "aws" + profile["path-style-access"] = false + } + if cfg.PathPrefix != "" { + profile["key-prefix"] = cfg.PathPrefix + } + + var credential map[string]any + if cfg.Provider == "aws" { + credential = map[string]any{ + "type": "s3", + "credential-type": "access-key", + "aws-access-key-id": cfg.Credential["access_key_id"], + "aws-secret-access-key": cfg.Credential["secret_access_key"], + } + } else { // gcs via S3-compatible HMAC + credential = map[string]any{ + "type": "s3", + "credential-type": "access-key", + "aws-access-key-id": cfg.Credential["hmac_access_id"], + "aws-secret-access-key": cfg.Credential["hmac_secret"], + } + } + + return map[string]any{ + "warehouse-name": cfg.Warehouse, + "storage-profile": profile, + "storage-credential": credential, + }, nil + case "azure": + profile := map[string]any{ + "type": "adls", + "account-name": cfg.Region, // account name carried in region for azure + "filesystem": cfg.Bucket, // container + } + if cfg.Endpoint != "" { + profile["endpoint-suffix"] = cfg.Endpoint + } + if cfg.PathPrefix != "" { + profile["path-prefix"] = cfg.PathPrefix + } + credential := map[string]any{ + "type": "az", + "credential-type": "shared-access-key", + "connection-string": cfg.Credential["connection_string"], + } + return map[string]any{ + "warehouse-name": cfg.Warehouse, + "storage-profile": profile, + "storage-credential": credential, + }, nil + default: + return nil, fmt.Errorf("lakekeeper bootstrap: unsupported provider %q", cfg.Provider) + } +} + +// runLakekeeperBootstrap performs the Lakekeeper REST bootstrap sequence +// against baseURL (the Lakekeeper service, e.g. http://:8181) using +// the supplied HTTP client, in order and idempotently: +// +// 1. POST /management/v1/bootstrap {"accept-terms-of-use": true} +// 2. POST /management/v1/warehouse (storage profile + credential) +// 3. extract warehouse-id from the warehouse response (or look it up on +// conflict) +// 4. POST /catalog/v1/{warehouse-id}/namespaces {"namespace":["default"]} +// +// Already-bootstrapped / already-exists responses (4xx conflicts) are treated +// as success so the sequence is safe to re-run. +func runLakekeeperBootstrap(ctx context.Context, client *http.Client, baseURL string, cfg *lakekeeperStorageConfig) error { + baseURL = strings.TrimRight(baseURL, "/") + + // Step 1: bootstrap (accept terms). Idempotent: an already-bootstrapped + // server returns a client error (400/409) which we tolerate. + if err := lakekeeperPost(ctx, client, baseURL+"/management/v1/bootstrap", + map[string]any{"accept-terms-of-use": true}, nil, true); err != nil { + return fmt.Errorf("lakekeeper bootstrap: accept-terms failed: %w", err) + } + + // Step 2: create the warehouse. + body, err := buildWarehouseRequestBody(cfg) + if err != nil { + return err + } + var whResp struct { + WarehouseID string `json:"warehouse-id"` + ID string `json:"id"` + } + created, err := lakekeeperPostDecode(ctx, client, baseURL+"/management/v1/warehouse", body, &whResp) + if err != nil { + return fmt.Errorf("lakekeeper bootstrap: create warehouse failed: %w", err) + } + + // Step 3: resolve the warehouse id. On a fresh create it is in the + // response; on conflict (already exists) we look it up by name. + warehouseID := firstNonEmpty(whResp.WarehouseID, whResp.ID) + if !created || warehouseID == "" { + warehouseID, err = lookupLakekeeperWarehouseID(ctx, client, baseURL, cfg.Warehouse) + if err != nil { + return fmt.Errorf("lakekeeper bootstrap: resolve warehouse id: %w", err) + } + } + if warehouseID == "" { + return fmt.Errorf("lakekeeper bootstrap: could not determine warehouse id for warehouse %q", cfg.Warehouse) + } + + // Step 4: create the default namespace (required for compaction to be able + // to create tables). Idempotent: already-exists is tolerated. + nsURL := fmt.Sprintf("%s/catalog/v1/%s/namespaces", baseURL, warehouseID) + if err := lakekeeperPost(ctx, client, nsURL, + map[string]any{"namespace": []string{"default"}}, nil, true); err != nil { + return fmt.Errorf("lakekeeper bootstrap: create default namespace failed: %w", err) + } + + return nil +} + +// lookupLakekeeperWarehouseID lists warehouses and returns the id of the one +// matching name. Used when the warehouse already exists (create returned a +// conflict). +func lookupLakekeeperWarehouseID(ctx context.Context, client *http.Client, baseURL, name string) (string, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, baseURL+"/management/v1/warehouse", nil) + if err != nil { + return "", err + } + req.Header.Set("Accept", "application/json") + resp, err := client.Do(req) + if err != nil { + return "", err + } + defer resp.Body.Close() + data, _ := io.ReadAll(resp.Body) + if resp.StatusCode >= 300 { + return "", fmt.Errorf("list warehouses returned status %d", resp.StatusCode) + } + var listResp struct { + Warehouses []struct { + WarehouseID string `json:"warehouse-id"` + ID string `json:"id"` + Name string `json:"name"` + } `json:"warehouses"` + } + if err := json.Unmarshal(data, &listResp); err != nil { + return "", fmt.Errorf("decode warehouse list: %w", err) + } + for _, w := range listResp.Warehouses { + if w.Name == name { + // Prefer the canonical warehouse-id field; fall back to the + // deprecated id, matching the create-response handling. + return firstNonEmpty(w.WarehouseID, w.ID), nil + } + } + return "", nil +} + +// lakekeeperPost issues a POST with a JSON body. When out is non-nil the +// response body is decoded into it. When tolerateConflict is true a 4xx +// response is treated as success (idempotent already-exists), otherwise any +// status >= 300 is an error. +func lakekeeperPost(ctx context.Context, client *http.Client, url string, body any, out any, tolerateConflict bool) error { + _, err := lakekeeperDo(ctx, client, url, body, out, tolerateConflict) + return err +} + +// lakekeeperPostDecode issues a POST that always tolerates conflicts and +// decodes a success response into out. It returns whether the resource was +// newly created (2xx) as opposed to already existing (4xx conflict). +func lakekeeperPostDecode(ctx context.Context, client *http.Client, url string, body any, out any) (created bool, err error) { + return lakekeeperDo(ctx, client, url, body, out, true) +} + +func lakekeeperDo(ctx context.Context, client *http.Client, url string, body any, out any, tolerateConflict bool) (created bool, err error) { + payload, err := json.Marshal(body) + if err != nil { + return false, fmt.Errorf("marshal request body: %w", err) + } + req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(payload)) + if err != nil { + return false, err + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Accept", "application/json") + + resp, err := client.Do(req) + if err != nil { + return false, err + } + defer resp.Body.Close() + data, _ := io.ReadAll(resp.Body) + + switch { + case resp.StatusCode >= 200 && resp.StatusCode < 300: + if out != nil && len(data) > 0 { + if err := json.Unmarshal(data, out); err != nil { + return true, fmt.Errorf("decode response: %w", err) + } + } + return true, nil + case tolerateConflict && resp.StatusCode >= 400 && resp.StatusCode < 500: + // Already bootstrapped / already exists — idempotent success. + return false, nil + default: + return false, fmt.Errorf("POST %s returned status %d: %s", url, resp.StatusCode, string(data)) + } +} + +func firstNonEmpty(vals ...string) string { + for _, v := range vals { + if v != "" { + return v + } + } + return "" +} diff --git a/server/internal/orchestrator/swarm/lakekeeper_bootstrap_resource.go b/server/internal/orchestrator/swarm/lakekeeper_bootstrap_resource.go new file mode 100644 index 00000000..d2bc32c0 --- /dev/null +++ b/server/internal/orchestrator/swarm/lakekeeper_bootstrap_resource.go @@ -0,0 +1,127 @@ +package swarm + +import ( + "context" + "fmt" + "net/http" + + "github.com/pgEdge/control-plane/server/internal/resource" +) + +var _ resource.Resource = (*LakekeeperBootstrapResource)(nil) + +const ResourceTypeLakekeeperBootstrap resource.Type = "swarm.lakekeeper_bootstrap" + +func LakekeeperBootstrapResourceIdentifier(serviceInstanceID string) resource.Identifier { + return resource.Identifier{ + ID: serviceInstanceID, + Type: ResourceTypeLakekeeperBootstrap, + } +} + +// LakekeeperBootstrapResource performs the post-deploy REST bootstrap of a +// Lakekeeper serve instance: it accepts the terms of use, creates the +// configured warehouse (with its object-store storage profile and credential), +// and creates the "default" namespace so that compaction can create tables. +// +// It depends on the serve ServiceInstanceResource, so it only runs after the +// Docker Swarm service is confirmed healthy (WaitForService). It runs on the +// same host as the serve container (HostExecutor) and reaches Lakekeeper over +// the bridge network at http://:8181, exactly as the migrate +// resource reaches the same Docker daemon/network context. +// +// A bootstrap FAILURE blocks: a database with an unbootstrapped warehouse is +// broken, so the error is returned from Create (surfaced by the resource +// engine) rather than merely logged in a best-effort PostDeploy hook. +// +// The sequence is idempotent — already-bootstrapped / already-exists responses +// are treated as success — so Create and Update re-run safely. BootstrapDone is +// a sentinel so Refresh can distinguish "never run" from "already applied". +type LakekeeperBootstrapResource struct { + ServiceInstanceID string `json:"service_instance_id"` + HostID string `json:"host_id"` + ServiceName string `json:"service_name"` + Port int `json:"port"` + Config map[string]any `json:"config"` + BootstrapDone bool `json:"bootstrap_done"` +} + +func (r *LakekeeperBootstrapResource) ResourceVersion() string { return "1" } + +func (r *LakekeeperBootstrapResource) DiffIgnore() []string { + // BootstrapDone is runtime state written by Create; exclude it from diffs + // so a completed bootstrap does not trigger spurious updates. + return []string{"/bootstrap_done"} +} + +func (r *LakekeeperBootstrapResource) Identifier() resource.Identifier { + return LakekeeperBootstrapResourceIdentifier(r.ServiceInstanceID) +} + +func (r *LakekeeperBootstrapResource) Executor() resource.Executor { + // Run on the same host as the serve container so we can reach Lakekeeper + // over the bridge network and share its Docker network context. + return resource.HostExecutor(r.HostID) +} + +func (r *LakekeeperBootstrapResource) Dependencies() []resource.Identifier { + // Depend on the serve ServiceInstanceResource so bootstrap only runs after + // the Docker service is confirmed healthy. + return []resource.Identifier{ + ServiceInstanceResourceIdentifier(r.ServiceInstanceID), + } +} + +func (r *LakekeeperBootstrapResource) TypeDependencies() []resource.Type { + return nil +} + +// Refresh returns ErrNotFound until the bootstrap has completed at least once, +// causing the resource engine to call Create. +func (r *LakekeeperBootstrapResource) Refresh(ctx context.Context, rc *resource.Context) error { + if !r.BootstrapDone { + return fmt.Errorf("%w: lakekeeper warehouse bootstrap has not yet run", resource.ErrNotFound) + } + return nil +} + +func (r *LakekeeperBootstrapResource) Create(ctx context.Context, rc *resource.Context) error { + if err := r.bootstrap(ctx, rc); err != nil { + return err + } + r.BootstrapDone = true + return nil +} + +func (r *LakekeeperBootstrapResource) Update(ctx context.Context, rc *resource.Context) error { + // Re-running the bootstrap is safe: the REST sequence is idempotent. + if err := r.bootstrap(ctx, rc); err != nil { + return err + } + r.BootstrapDone = true + return nil +} + +func (r *LakekeeperBootstrapResource) Delete(ctx context.Context, rc *resource.Context) error { + // Deleting the service removes the warehouse configuration; nothing to do. + return nil +} + +func (r *LakekeeperBootstrapResource) bootstrap(ctx context.Context, rc *resource.Context) error { + cfg, err := parseLakekeeperStorageConfig(r.Config) + if err != nil { + return err + } + + port := r.Port + if port == 0 { + port = 8181 + } + baseURL := fmt.Sprintf("http://%s:%d", r.ServiceName, port) + + client := &http.Client{Timeout: lakekeeperBootstrapHTTPTimeout} + if err := runLakekeeperBootstrap(ctx, client, baseURL, cfg); err != nil { + return err + } + return nil +} diff --git a/server/internal/orchestrator/swarm/lakekeeper_bootstrap_test.go b/server/internal/orchestrator/swarm/lakekeeper_bootstrap_test.go new file mode 100644 index 00000000..6a1b591e --- /dev/null +++ b/server/internal/orchestrator/swarm/lakekeeper_bootstrap_test.go @@ -0,0 +1,238 @@ +package swarm + +import ( + "context" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// awsBootstrapConfig is a native cloud-AWS config: no endpoint, so +// virtual-hosted addressing (flavor "aws", path-style-access false). +func awsBootstrapConfig() map[string]any { + return map[string]any{ + "provider": "aws", + "bucket": "my-bucket", + "region": "us-east-1", + "warehouse": "wh1", + "path_prefix": "iceberg", + "credential": `{"access_key_id":"AKIA_TEST","secret_access_key":"SECRET_TEST"}`, + } +} + +// gcsBootstrapConfig is an S3-compatible config (GCS): endpoint present, so +// flavor "s3-compat" and path-style-access true. +func gcsBootstrapConfig() map[string]any { + return map[string]any{ + "provider": "gcs", + "bucket": "my-bucket", + "region": "us", + "endpoint": "https://storage.googleapis.com", + "warehouse": "wh1", + "path_prefix": "iceberg", + "credential": `{"hmac_access_id":"GOOG_ID","hmac_secret":"GOOG_SECRET"}`, + } +} + +func TestParseLakekeeperStorageConfig_FailLoud(t *testing.T) { + tests := []struct { + name string + config map[string]any + errSub string + }{ + {"missing provider", map[string]any{"warehouse": "w", "credential": "{}"}, "provider is required"}, + {"bad provider", map[string]any{"provider": "digitalocean", "warehouse": "w", "credential": "{}"}, "unsupported provider"}, + {"missing warehouse", map[string]any{"provider": "aws", "credential": "{}"}, "warehouse is required"}, + {"missing credential", map[string]any{"provider": "aws", "warehouse": "w"}, "credential is required"}, + {"bad credential json", map[string]any{"provider": "aws", "warehouse": "w", "bucket": "b", "credential": "not-json"}, "not valid JSON"}, + {"missing bucket aws", map[string]any{"provider": "aws", "warehouse": "w", "credential": `{"access_key_id":"a","secret_access_key":"s"}`}, "bucket is required"}, + {"aws missing keys", map[string]any{"provider": "aws", "bucket": "b", "warehouse": "w", "credential": `{}`}, "access_key_id and secret_access_key"}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + _, err := parseLakekeeperStorageConfig(tc.config) + require.Error(t, err) + assert.Contains(t, err.Error(), tc.errSub) + }) + } +} + +func TestParseLakekeeperStorageConfig_AWS(t *testing.T) { + cfg, err := parseLakekeeperStorageConfig(awsBootstrapConfig()) + require.NoError(t, err) + assert.Equal(t, "aws", cfg.Provider) + assert.Equal(t, "my-bucket", cfg.Bucket) + assert.Equal(t, "AKIA_TEST", cfg.Credential["access_key_id"]) +} + +// TestRunLakekeeperBootstrap_OrderAndFlow asserts the four REST calls happen in +// order, the warehouse-id is extracted from the create response and used in the +// namespace URL. +func TestRunLakekeeperBootstrap_OrderAndFlow(t *testing.T) { + var calls []string + var namespaceBody map[string]any + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls = append(calls, r.Method+" "+r.URL.Path) + switch { + case r.URL.Path == "/management/v1/bootstrap": + w.WriteHeader(http.StatusOK) + case r.URL.Path == "/management/v1/warehouse" && r.Method == http.MethodPost: + // verify the storage profile is well-formed for aws + var body map[string]any + require.NoError(t, json.NewDecoder(r.Body).Decode(&body)) + assert.Equal(t, "wh1", body["warehouse-name"]) + profile := body["storage-profile"].(map[string]any) + assert.Equal(t, "s3", profile["type"]) + assert.Equal(t, false, profile["sts-enabled"]) + assert.Equal(t, false, profile["remote-signing-enabled"]) + // Cloud AWS (no endpoint): virtual-hosted addressing. + assert.Equal(t, "aws", profile["flavor"]) + assert.Equal(t, false, profile["path-style-access"]) + assert.NotContains(t, profile, "endpoint") + // key-prefix, not path-prefix. + assert.Equal(t, "iceberg", profile["key-prefix"]) + assert.NotContains(t, profile, "path-prefix") + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"warehouse-id":"wh-uuid-123"}`)) + case strings.HasPrefix(r.URL.Path, "/catalog/v1/") && strings.HasSuffix(r.URL.Path, "/namespaces"): + require.NoError(t, json.NewDecoder(r.Body).Decode(&namespaceBody)) + w.WriteHeader(http.StatusOK) + default: + t.Errorf("unexpected call: %s %s", r.Method, r.URL.Path) + w.WriteHeader(http.StatusInternalServerError) + } + })) + defer srv.Close() + + cfg, err := parseLakekeeperStorageConfig(awsBootstrapConfig()) + require.NoError(t, err) + + err = runLakekeeperBootstrap(context.Background(), srv.Client(), srv.URL, cfg) + require.NoError(t, err) + + require.Equal(t, []string{ + "POST /management/v1/bootstrap", + "POST /management/v1/warehouse", + "POST /catalog/v1/wh-uuid-123/namespaces", + }, calls) + assert.Equal(t, []any{"default"}, namespaceBody["namespace"]) +} + +// TestRunLakekeeperBootstrap_AlreadyBootstrapped tolerates a conflict on +// bootstrap and on the warehouse (already exists), looking up the warehouse-id +// via GET, and tolerates a conflict on the namespace. +func TestRunLakekeeperBootstrap_AlreadyBootstrapped(t *testing.T) { + var calls []string + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls = append(calls, r.Method+" "+r.URL.Path) + switch { + case r.URL.Path == "/management/v1/bootstrap": + // already bootstrapped + w.WriteHeader(http.StatusConflict) + case r.URL.Path == "/management/v1/warehouse" && r.Method == http.MethodPost: + // already exists + w.WriteHeader(http.StatusConflict) + case r.URL.Path == "/management/v1/warehouse" && r.Method == http.MethodGet: + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"warehouses":[{"id":"existing-wh","name":"wh1"}]}`)) + case strings.HasSuffix(r.URL.Path, "/namespaces"): + w.WriteHeader(http.StatusConflict) + default: + t.Errorf("unexpected call: %s %s", r.Method, r.URL.Path) + } + })) + defer srv.Close() + + cfg, err := parseLakekeeperStorageConfig(awsBootstrapConfig()) + require.NoError(t, err) + + err = runLakekeeperBootstrap(context.Background(), srv.Client(), srv.URL, cfg) + require.NoError(t, err) + + // The namespace must be created against the looked-up existing-wh id. + require.Equal(t, []string{ + "POST /management/v1/bootstrap", + "POST /management/v1/warehouse", + "GET /management/v1/warehouse", + "POST /catalog/v1/existing-wh/namespaces", + }, calls) +} + +// TestRunLakekeeperBootstrap_ServerErrorSurfaces confirms a genuine 5xx failure +// is surfaced (not swallowed), so an unbootstrapped warehouse blocks. +func TestRunLakekeeperBootstrap_ServerErrorSurfaces(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/management/v1/bootstrap" { + w.WriteHeader(http.StatusOK) + return + } + w.WriteHeader(http.StatusInternalServerError) + _, _ = io.WriteString(w, "boom") + })) + defer srv.Close() + + cfg, err := parseLakekeeperStorageConfig(awsBootstrapConfig()) + require.NoError(t, err) + + err = runLakekeeperBootstrap(context.Background(), srv.Client(), srv.URL, cfg) + require.Error(t, err) + assert.Contains(t, err.Error(), "create warehouse failed") +} + +func TestBuildWarehouseRequestBody_CloudAWS(t *testing.T) { + cfg, err := parseLakekeeperStorageConfig(awsBootstrapConfig()) + require.NoError(t, err) + + body, err := buildWarehouseRequestBody(cfg) + require.NoError(t, err) + profile := body["storage-profile"].(map[string]any) + assert.Equal(t, "s3", profile["type"]) + assert.Equal(t, "aws", profile["flavor"]) + assert.Equal(t, false, profile["path-style-access"]) + assert.NotContains(t, profile, "endpoint") + assert.Equal(t, "iceberg", profile["key-prefix"]) +} + +func TestBuildWarehouseRequestBody_S3CompatGCS(t *testing.T) { + cfg, err := parseLakekeeperStorageConfig(gcsBootstrapConfig()) + require.NoError(t, err) + + body, err := buildWarehouseRequestBody(cfg) + require.NoError(t, err) + profile := body["storage-profile"].(map[string]any) + assert.Equal(t, "s3", profile["type"]) + // endpoint present => s3-compat, path-style addressing. + assert.Equal(t, "s3-compat", profile["flavor"]) + assert.Equal(t, true, profile["path-style-access"]) + assert.Equal(t, "https://storage.googleapis.com", profile["endpoint"]) + assert.Equal(t, "iceberg", profile["key-prefix"]) + // GCS HMAC creds are carried as S3 access-key credentials. + credential := body["storage-credential"].(map[string]any) + assert.Equal(t, "GOOG_ID", credential["aws-access-key-id"]) +} + +func TestBuildWarehouseRequestBody_Azure(t *testing.T) { + cfg, err := parseLakekeeperStorageConfig(map[string]any{ + "provider": "azure", + "bucket": "container1", + "region": "myaccount", + "warehouse": "wh1", + "credential": `{"connection_string":"DefaultEndpointsProtocol=https;AccountName=x"}`, + }) + require.NoError(t, err) + + body, err := buildWarehouseRequestBody(cfg) + require.NoError(t, err) + profile := body["storage-profile"].(map[string]any) + assert.Equal(t, "adls", profile["type"]) + credential := body["storage-credential"].(map[string]any) + assert.Equal(t, "DefaultEndpointsProtocol=https;AccountName=x", credential["connection-string"]) +} diff --git a/server/internal/orchestrator/swarm/lakekeeper_config_resource.go b/server/internal/orchestrator/swarm/lakekeeper_config_resource.go new file mode 100644 index 00000000..3f4ba168 --- /dev/null +++ b/server/internal/orchestrator/swarm/lakekeeper_config_resource.go @@ -0,0 +1,127 @@ +package swarm + +import ( + "context" + "fmt" + "path/filepath" + + "github.com/samber/do" + "github.com/spf13/afero" + + "github.com/pgEdge/control-plane/server/internal/filesystem" + "github.com/pgEdge/control-plane/server/internal/orchestrator/common" + "github.com/pgEdge/control-plane/server/internal/resource" +) + +var _ resource.Resource = (*LakekeeperConfigResource)(nil) + +const ResourceTypeLakekeeperConfig resource.Type = "swarm.lakekeeper_config" + +// lakekeeperReadyFile is the sentinel written on first Create so that +// subsequent reconciliations skip re-running Create unnecessarily. +const lakekeeperReadyFile = ".lakekeeper_ready" + +func LakekeeperConfigResourceIdentifier(serviceInstanceID string) resource.Identifier { + return resource.Identifier{ + ID: serviceInstanceID, + Type: ResourceTypeLakekeeperConfig, + } +} + +// LakekeeperConfigResource manages Lakekeeper's data directory on the host +// filesystem. It follows the same pattern as MCPConfigResource: it ensures +// any host-side artefacts needed by the container are present before the +// container starts. +// +// Lakekeeper is configured entirely via environment variables (LAKEKEEPER__*), +// so Create and Update are lightweight — they just write a sentinel file so +// Refresh can confirm the resource has been applied. Schema migration against +// the external catalog Postgres is handled by LakekeeperMigrateResource, which +// runs the Lakekeeper image with the "migrate" subcommand before the "serve" +// container starts. +type LakekeeperConfigResource struct { + ServiceInstanceID string `json:"service_instance_id"` + ServiceID string `json:"service_id"` + HostID string `json:"host_id"` + DirResourceID string `json:"dir_resource_id"` +} + +func (r *LakekeeperConfigResource) ResourceVersion() string { + return "1" +} + +func (r *LakekeeperConfigResource) DiffIgnore() []string { + return nil +} + +func (r *LakekeeperConfigResource) Identifier() resource.Identifier { + return LakekeeperConfigResourceIdentifier(r.ServiceInstanceID) +} + +func (r *LakekeeperConfigResource) Executor() resource.Executor { + return resource.HostExecutor(r.HostID) +} + +func (r *LakekeeperConfigResource) Dependencies() []resource.Identifier { + return []resource.Identifier{ + filesystem.DirResourceIdentifier(r.DirResourceID), + } +} + +func (r *LakekeeperConfigResource) TypeDependencies() []resource.Type { + return nil +} + +func (r *LakekeeperConfigResource) Refresh(ctx context.Context, rc *resource.Context) error { + fs, err := do.Invoke[afero.Fs](rc.Injector) + if err != nil { + return err + } + + dirPath, err := filesystem.DirResourceFullPath(rc, r.DirResourceID) + if err != nil { + return fmt.Errorf("failed to get service data dir path: %w", err) + } + + // Check for the sentinel file; ErrNotFound here triggers Create. + _, err = common.ReadResourceFile(fs, filepath.Join(dirPath, lakekeeperReadyFile)) + if err != nil { + return fmt.Errorf("failed to read lakekeeper ready sentinel: %w", err) + } + + return nil +} + +func (r *LakekeeperConfigResource) Create(ctx context.Context, rc *resource.Context) error { + fs, err := do.Invoke[afero.Fs](rc.Injector) + if err != nil { + return err + } + + dirPath, err := filesystem.DirResourceFullPath(rc, r.DirResourceID) + if err != nil { + return fmt.Errorf("failed to get service data dir path: %w", err) + } + + // Write the sentinel file so subsequent Refresh calls succeed. + sentinelPath := filepath.Join(dirPath, lakekeeperReadyFile) + if err := afero.WriteFile(fs, sentinelPath, []byte("ok\n"), 0o600); err != nil { + return fmt.Errorf("failed to write lakekeeper ready sentinel: %w", err) + } + + return nil +} + +func (r *LakekeeperConfigResource) Update(ctx context.Context, rc *resource.Context) error { + // No config files to regenerate; Lakekeeper configuration is delivered + // via environment variables set in the container spec. Changes to + // LAKEKEEPER__* env vars force a Swarm service restart automatically + // because the ServiceInstanceSpec resource detects a diff in the desired + // TaskTemplate. + return nil +} + +func (r *LakekeeperConfigResource) Delete(ctx context.Context, rc *resource.Context) error { + // Cleanup is handled by the parent directory resource deletion. + return nil +} diff --git a/server/internal/orchestrator/swarm/lakekeeper_migrate_resource.go b/server/internal/orchestrator/swarm/lakekeeper_migrate_resource.go new file mode 100644 index 00000000..47b05884 --- /dev/null +++ b/server/internal/orchestrator/swarm/lakekeeper_migrate_resource.go @@ -0,0 +1,177 @@ +package swarm + +import ( + "bytes" + "context" + "fmt" + "time" + + "github.com/docker/docker/api/types/container" + "github.com/samber/do" + + "github.com/pgEdge/control-plane/server/internal/docker" + "github.com/pgEdge/control-plane/server/internal/resource" +) + +var _ resource.Resource = (*LakekeeperMigrateResource)(nil) + +const ResourceTypeLakekeeperMigrate resource.Type = "swarm.lakekeeper_migrate" + +// lakekeeperMigrateTimeout is the maximum time allowed for the schema migration +// to complete. Migrations run once on the external catalog Postgres and are +// expected to be fast, but a generous timeout is used to tolerate slow cold +// starts or transient network delays. +const lakekeeperMigrateTimeout = 5 * time.Minute + +func LakekeeperMigrateResourceIdentifier(serviceInstanceID string) resource.Identifier { + return resource.Identifier{ + ID: serviceInstanceID, + Type: ResourceTypeLakekeeperMigrate, + } +} + +// LakekeeperMigrateResource runs the Lakekeeper image with the "migrate" +// subcommand as a one-shot Docker container before the "serve" container is +// started. It applies the Iceberg catalog schema to the external Postgres +// database supplied via LAKEKEEPER__PG_DATABASE_URL_{READ,WRITE}. +// +// The resource is idempotent: Lakekeeper's own migrate command is a no-op when +// the schema is already current, so Create and Update both re-run the +// migration safely on every reconciliation cycle. +// +// Lifecycle: +// - Refresh: returns ErrNotFound until migration has completed at least once +// successfully (detected via the sentinel stored in MigratedOnce). This +// causes the resource engine to call Create on the first pass and then +// re-validate on subsequent passes. +// - Create/Update: run the migrate container and wait for exit-0. +// - Delete: no-op; deleting the service removes the catalog DB connection. +type LakekeeperMigrateResource struct { + ServiceInstanceID string `json:"service_instance_id"` + HostID string `json:"host_id"` + Image string `json:"image"` + CatalogDBURL string `json:"catalog_db_url"` + PGEncryptionKey string `json:"pg_encryption_key"` + // MigratedOnce is set to true after the first successful migration so that + // Refresh can distinguish "never run" from "already applied". + MigratedOnce bool `json:"migrated_once"` +} + +func (r *LakekeeperMigrateResource) ResourceVersion() string { return "1" } + +func (r *LakekeeperMigrateResource) DiffIgnore() []string { + // MigratedOnce is runtime state written by Create; exclude it from diff + // comparisons so that a completed migration does not trigger spurious updates. + return []string{"/migrated_once"} +} + +func (r *LakekeeperMigrateResource) Identifier() resource.Identifier { + return LakekeeperMigrateResourceIdentifier(r.ServiceInstanceID) +} + +func (r *LakekeeperMigrateResource) Executor() resource.Executor { + // The migration container is run on the same host as the serve container + // so that it shares the same Docker daemon and network context. + return resource.HostExecutor(r.HostID) +} + +func (r *LakekeeperMigrateResource) Dependencies() []resource.Identifier { + // No resource-level dependencies: the external catalog Postgres is not + // managed by control-plane, so there is no resource to depend on. The + // catalog URL is validated at spec time (fail-loud check). + return nil +} + +func (r *LakekeeperMigrateResource) TypeDependencies() []resource.Type { + return nil +} + +// Refresh returns ErrNotFound when migration has never successfully completed, +// causing the resource engine to call Create. Once MigratedOnce is true the +// resource is considered up-to-date and no further action is taken unless the +// desired state changes (which triggers Update). +func (r *LakekeeperMigrateResource) Refresh(ctx context.Context, rc *resource.Context) error { + if !r.MigratedOnce { + return fmt.Errorf("%w: lakekeeper schema migration has not yet run", resource.ErrNotFound) + } + return nil +} + +func (r *LakekeeperMigrateResource) Create(ctx context.Context, rc *resource.Context) error { + if err := r.runMigrate(ctx, rc); err != nil { + return err + } + r.MigratedOnce = true + return nil +} + +func (r *LakekeeperMigrateResource) Update(ctx context.Context, rc *resource.Context) error { + // Re-running migrate is always safe: Lakekeeper's migrate is idempotent. + if err := r.runMigrate(ctx, rc); err != nil { + return err + } + r.MigratedOnce = true + return nil +} + +func (r *LakekeeperMigrateResource) Delete(ctx context.Context, rc *resource.Context) error { + return nil +} + +// runMigrate starts a one-shot Docker container using the Lakekeeper image +// with the "migrate" subcommand, waits for it to exit with status 0, and +// streams the logs to the resource context logger on failure. +func (r *LakekeeperMigrateResource) runMigrate(ctx context.Context, rc *resource.Context) error { + client, err := do.Invoke[*docker.Docker](rc.Injector) + if err != nil { + return fmt.Errorf("lakekeeper migrate: failed to get docker client: %w", err) + } + + containerName := "lakekeeper-migrate-" + r.ServiceInstanceID + + // Remove any leftover container from a previous failed attempt before + // creating a new one, so the name does not conflict. + _ = client.ContainerRemove(ctx, containerName, container.RemoveOptions{Force: true}) + + containerID, err := client.ContainerRun(ctx, docker.ContainerRunOptions{ + Config: &container.Config{ + Image: r.Image, + Cmd: []string{"migrate"}, + Env: []string{ + "LAKEKEEPER__PG_DATABASE_URL_READ=" + r.CatalogDBURL, + "LAKEKEEPER__PG_DATABASE_URL_WRITE=" + r.CatalogDBURL, + "LAKEKEEPER__PG_ENCRYPTION_KEY=" + r.PGEncryptionKey, + }, + }, + Host: &container.HostConfig{ + // AutoRemove is intentionally false so that we can stream logs on + // failure before the container is cleaned up. + AutoRemove: false, + }, + Name: containerName, + }) + if err != nil { + return fmt.Errorf("lakekeeper migrate: failed to start container: %w", err) + } + + // Wait for the migration to complete. + waitErr := client.ContainerWait(ctx, containerID, container.WaitConditionNotRunning, lakekeeperMigrateTimeout) + + // Always remove the container once we have the exit status (or on error). + defer func() { + removeCtx := context.Background() + _ = client.ContainerRemove(removeCtx, containerID, container.RemoveOptions{Force: true}) + }() + + if waitErr != nil { + // Capture logs to help diagnose the failure. + var logBuf bytes.Buffer + _ = client.ContainerLogs(ctx, &logBuf, containerID, container.LogsOptions{ + ShowStdout: true, + ShowStderr: true, + }) + return fmt.Errorf("lakekeeper migrate: migration failed: %w\nlogs:\n%s", waitErr, logBuf.String()) + } + + return nil +} diff --git a/server/internal/orchestrator/swarm/lakekeeper_storage_secret_resource.go b/server/internal/orchestrator/swarm/lakekeeper_storage_secret_resource.go new file mode 100644 index 00000000..8490c83d --- /dev/null +++ b/server/internal/orchestrator/swarm/lakekeeper_storage_secret_resource.go @@ -0,0 +1,217 @@ +package swarm + +import ( + "context" + "fmt" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" + + "github.com/pgEdge/control-plane/server/internal/database" + "github.com/pgEdge/control-plane/server/internal/resource" +) + +var _ resource.Resource = (*LakekeeperStorageSecretResource)(nil) + +const ResourceTypeLakekeeperStorageSecret resource.Type = "swarm.lakekeeper_storage_secret" + +func LakekeeperStorageSecretResourceIdentifier(serviceInstanceID string) resource.Identifier { + return resource.Identifier{ + ID: serviceInstanceID, + Type: ResourceTypeLakekeeperStorageSecret, + } +} + +// LakekeeperStorageSecretResource stores the object-store credential inside the +// database via ColdFront's set_storage_secret function, so that the ColdFront +// extension on the node can read/write Iceberg data in the warehouse's bucket. +// +// It runs once against the node's primary Postgres (PrimaryExecutor, like the +// PostgREST preflight resource) after the database — and therefore the +// coldfront extension — is available. It depends on the Postgres database +// resource for that ordering. +// +// The set_storage_secret functions upsert, so re-running is safe; SecretSet is +// a sentinel so Refresh can distinguish "never run" from "already applied". +// +// The credential is passed to Postgres exclusively as bound query parameters +// and is NEVER logged. +type LakekeeperStorageSecretResource struct { + ServiceInstanceID string `json:"service_instance_id"` + DatabaseID string `json:"database_id"` + DatabaseName string `json:"database_name"` + NodeName string `json:"node_name"` + Config map[string]any `json:"config"` + SecretSet bool `json:"secret_set"` +} + +func (r *LakekeeperStorageSecretResource) ResourceVersion() string { return "1" } + +func (r *LakekeeperStorageSecretResource) DiffIgnore() []string { + return []string{"/secret_set"} +} + +func (r *LakekeeperStorageSecretResource) Identifier() resource.Identifier { + return LakekeeperStorageSecretResourceIdentifier(r.ServiceInstanceID) +} + +func (r *LakekeeperStorageSecretResource) Executor() resource.Executor { + return resource.PrimaryExecutor(r.NodeName) +} + +func (r *LakekeeperStorageSecretResource) Dependencies() []resource.Identifier { + // Depend on the database resource so the coldfront extension is available + // before we call set_storage_secret. + return []resource.Identifier{ + database.PostgresDatabaseResourceIdentifier(r.NodeName, r.DatabaseName), + } +} + +func (r *LakekeeperStorageSecretResource) TypeDependencies() []resource.Type { + return nil +} + +func (r *LakekeeperStorageSecretResource) Refresh(ctx context.Context, rc *resource.Context) error { + if !r.SecretSet { + return fmt.Errorf("%w: coldfront storage secret has not yet been set", resource.ErrNotFound) + } + return nil +} + +func (r *LakekeeperStorageSecretResource) Create(ctx context.Context, rc *resource.Context) error { + if err := r.setSecret(ctx, rc); err != nil { + return err + } + r.SecretSet = true + return nil +} + +func (r *LakekeeperStorageSecretResource) Update(ctx context.Context, rc *resource.Context) error { + // set_storage_secret upserts, so re-running is safe. + if err := r.setSecret(ctx, rc); err != nil { + return err + } + r.SecretSet = true + return nil +} + +func (r *LakekeeperStorageSecretResource) Delete(ctx context.Context, rc *resource.Context) error { + return nil +} + +func (r *LakekeeperStorageSecretResource) setSecret(ctx context.Context, rc *resource.Context) error { + cfg, err := parseLakekeeperStorageConfig(r.Config) + if err != nil { + return err + } + + primary, err := database.GetPrimaryInstance(ctx, rc, r.NodeName) + if err != nil { + return fmt.Errorf("coldfront set_storage_secret: failed to get primary instance: %w", err) + } + conn, err := primary.Connection(ctx, rc, r.DatabaseName) + if err != nil { + return fmt.Errorf("coldfront set_storage_secret: failed to connect to database %s on node %s: %w", + r.DatabaseName, r.NodeName, err) + } + defer conn.Close(ctx) + + return execSetStorageSecret(ctx, conn, cfg) +} + +// setStorageSecretExec is the minimal Postgres exec surface needed by +// execSetStorageSecret, satisfied by *pgx.Conn. It lets the SQL selection be +// unit-tested without a live database. +type setStorageSecretExec interface { + Exec(ctx context.Context, sql string, args ...any) (pgconn.CommandTag, error) +} + +// execSetStorageSecret builds and executes the correct ColdFront function call +// for the configured provider. The credential values are bound as query +// parameters ($1, $2, ...) and never interpolated into the SQL text, so they +// cannot leak via query logging. +func execSetStorageSecret(ctx context.Context, conn setStorageSecretExec, cfg *lakekeeperStorageConfig) error { + sql, args := buildSetStorageSecretSQL(cfg) + if _, err := conn.Exec(ctx, sql, args...); err != nil { + // The error is from Postgres and does not echo bound parameters, so it + // is safe to wrap; do NOT include the credential. + return fmt.Errorf("coldfront set_storage_secret failed for provider %q: %w", cfg.Provider, err) + } + return nil +} + +// buildSetStorageSecretSQL returns the SQL and bound arguments for the +// provider. aws/gcs use coldfront.set_storage_secret; azure uses +// coldfront.set_storage_secret_azure. Credentials are returned as args, never +// embedded in the SQL string. +// +// Endpoint presence is the discriminator, consistent with the warehouse +// storage profile: +// - endpoint ABSENT (cloud AWS): pass endpoint => NULL, which selects +// DuckDB's native per-Region virtual-hosted + HTTPS addressing (required +// for Regions launched after 2019). url_style/use_ssl are left at their +// defaults — they are irrelevant when the endpoint is NULL, and passing a +// non-NULL endpoint here forces path-style and breaks modern Regions with +// HTTP 400 (ColdFront docs/object_store.md). +// - endpoint PRESENT (S3-compatible / GCS): pass the endpoint with +// url_style => 'path' and use_ssl => true. +func buildSetStorageSecretSQL(cfg *lakekeeperStorageConfig) (string, []any) { + switch cfg.Provider { + case "aws": + return buildS3SetStorageSecretSQL( + cfg.Credential["access_key_id"], + cfg.Credential["secret_access_key"], + cfg.Endpoint, + cfg.Region, + ) + case "gcs": + // GCS is only reachable through its S3-compatible HMAC endpoint, so it + // always has an endpoint (default to the canonical host when unset). + endpoint := cfg.Endpoint + if endpoint == "" { + endpoint = "storage.googleapis.com" + } + return buildS3SetStorageSecretSQL( + cfg.Credential["hmac_access_id"], + cfg.Credential["hmac_secret"], + endpoint, + cfg.Region, + ) + case "azure": + return `SELECT coldfront.set_storage_secret_azure(p_connection_string => $1)`, + []any{cfg.Credential["connection_string"]} + default: + // parseLakekeeperStorageConfig has already validated the provider, so + // this branch is unreachable in practice. + return "", nil + } +} + +// buildS3SetStorageSecretSQL builds the set_storage_secret call for an S3 or +// S3-compatible store, driving cloud-vs-s3-compat semantics off endpoint +// presence. keyID/secret are always bound as parameters. +func buildS3SetStorageSecretSQL(keyID, secret, endpoint, region string) (string, []any) { + if endpoint == "" { + // Cloud AWS: endpoint NULL selects native vhost + HTTPS. Do not pass + // url_style/use_ssl — defaults apply and a non-NULL endpoint here is + // exactly the shape the docs warn against. + return `SELECT coldfront.set_storage_secret( + p_key_id => $1, + p_secret => $2, + p_endpoint => NULL, + p_region => $3 + )`, []any{keyID, secret, region} + } + // S3-compatible / GCS: explicit endpoint with path-style + SSL. + return `SELECT coldfront.set_storage_secret( + p_key_id => $1, + p_secret => $2, + p_endpoint => $3, + p_region => $4, + p_url_style => $5, + p_use_ssl => $6 + )`, []any{keyID, secret, endpoint, region, "path", true} +} + +// ensure *pgx.Conn satisfies setStorageSecretExec at compile time. +var _ setStorageSecretExec = (*pgx.Conn)(nil) diff --git a/server/internal/orchestrator/swarm/lakekeeper_storage_secret_resource_test.go b/server/internal/orchestrator/swarm/lakekeeper_storage_secret_resource_test.go new file mode 100644 index 00000000..91fa21e4 --- /dev/null +++ b/server/internal/orchestrator/swarm/lakekeeper_storage_secret_resource_test.go @@ -0,0 +1,129 @@ +package swarm + +import ( + "context" + "strings" + "testing" + + "github.com/jackc/pgx/v5/pgconn" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// fakeExec captures the SQL and args passed to Exec so the test can assert the +// correct function is chosen and the credential is bound (not interpolated). +type fakeExec struct { + sql string + args []any +} + +func (f *fakeExec) Exec(ctx context.Context, sql string, args ...any) (pgconn.CommandTag, error) { + f.sql = sql + f.args = args + return pgconn.CommandTag{}, nil +} + +// TestBuildSetStorageSecretSQL_CloudAWS: no endpoint => endpoint NULL (native +// vhost+HTTPS), no url_style/use_ssl. Passing a non-NULL endpoint here is the +// shape the docs warn breaks modern Regions with HTTP 400. +func TestBuildSetStorageSecretSQL_CloudAWS(t *testing.T) { + cfg := &lakekeeperStorageConfig{ + Provider: "aws", + Region: "us-east-1", + Credential: map[string]string{ + "access_key_id": "AKIA_TEST", + "secret_access_key": "SECRET_TEST", + }, + } + sql, args := buildSetStorageSecretSQL(cfg) + + assert.Contains(t, sql, "coldfront.set_storage_secret(") + assert.NotContains(t, sql, "set_storage_secret_azure") + // endpoint => NULL literal in SQL; url_style/use_ssl omitted entirely. + assert.Contains(t, sql, "p_endpoint => NULL") + assert.NotContains(t, sql, "p_url_style") + assert.NotContains(t, sql, "p_use_ssl") + // key/secret/region bound as args; no endpoint arg. + assert.Equal(t, []any{"AKIA_TEST", "SECRET_TEST", "us-east-1"}, args) + // Credential must NOT be interpolated into the SQL text. + assert.NotContains(t, sql, "AKIA_TEST") + assert.NotContains(t, sql, "SECRET_TEST") +} + +// TestBuildSetStorageSecretSQL_S3CompatAWS: endpoint present => pass endpoint + +// path-style + SSL. +func TestBuildSetStorageSecretSQL_S3CompatAWS(t *testing.T) { + cfg := &lakekeeperStorageConfig{ + Provider: "aws", + Region: "us-east-1", + Endpoint: "http://seaweedfs:8333", + Credential: map[string]string{ + "access_key_id": "AKIA_TEST", + "secret_access_key": "SECRET_TEST", + }, + } + sql, args := buildSetStorageSecretSQL(cfg) + + assert.Contains(t, sql, "coldfront.set_storage_secret(") + assert.Contains(t, sql, "p_url_style => $5") + assert.NotContains(t, sql, "p_endpoint => NULL") + // endpoint present as a bound arg, alongside path-style + SSL. + assert.Equal(t, []any{"AKIA_TEST", "SECRET_TEST", "http://seaweedfs:8333", "us-east-1", "path", true}, args) + assert.NotContains(t, sql, "AKIA_TEST") +} + +// TestBuildSetStorageSecretSQL_GCS: GCS is always S3-compatible, so it uses an +// endpoint (defaulting to the canonical host) with path-style + SSL. +func TestBuildSetStorageSecretSQL_GCS(t *testing.T) { + cfg := &lakekeeperStorageConfig{ + Provider: "gcs", + Region: "us", + Credential: map[string]string{ + "hmac_access_id": "GOOG_ID", + "hmac_secret": "GOOG_SECRET", + }, + } + sql, args := buildSetStorageSecretSQL(cfg) + assert.Contains(t, sql, "coldfront.set_storage_secret(") + assert.NotContains(t, sql, "set_storage_secret_azure") + assert.NotContains(t, sql, "p_endpoint => NULL") + // gcs defaults endpoint to the GCS S3-compatible host and uses path-style. + assert.Contains(t, args, "storage.googleapis.com") + assert.Contains(t, args, "path") + assert.Contains(t, args, "GOOG_ID") + assert.NotContains(t, sql, "GOOG_SECRET") +} + +func TestBuildSetStorageSecretSQL_Azure(t *testing.T) { + cfg := &lakekeeperStorageConfig{ + Provider: "azure", + Credential: map[string]string{ + "connection_string": "DefaultEndpointsProtocol=https;AccountKey=SEKRIT", + }, + } + sql, args := buildSetStorageSecretSQL(cfg) + assert.Contains(t, sql, "coldfront.set_storage_secret_azure(") + assert.NotContains(t, sql, "set_storage_secret(") + require.Len(t, args, 1) + assert.Equal(t, "DefaultEndpointsProtocol=https;AccountKey=SEKRIT", args[0]) + // The connection string must NOT appear in the SQL text. + assert.NotContains(t, sql, "SEKRIT") +} + +func TestExecSetStorageSecret_BindsCredentialAsArgs(t *testing.T) { + cfg := &lakekeeperStorageConfig{ + Provider: "aws", + Region: "us-east-1", + Credential: map[string]string{ + "access_key_id": "AKIA_XYZ", + "secret_access_key": "S3KRET", + }, + } + fe := &fakeExec{} + require.NoError(t, execSetStorageSecret(context.Background(), fe, cfg)) + + // Secret is passed as a bound arg, and the SQL text uses placeholders. + assert.Contains(t, fe.args, "S3KRET") + assert.NotContains(t, fe.sql, "S3KRET") + assert.True(t, strings.Contains(fe.sql, "$1"), "expected parameter placeholders in SQL") +} diff --git a/server/internal/orchestrator/swarm/orchestrator.go b/server/internal/orchestrator/swarm/orchestrator.go index 5f3c4eea..c2e38919 100644 --- a/server/internal/orchestrator/swarm/orchestrator.go +++ b/server/internal/orchestrator/swarm/orchestrator.go @@ -446,6 +446,8 @@ func (o *Orchestrator) GenerateServiceInstanceResources(spec *database.ServiceIn return o.generateMCPInstanceResources(spec) case "rag": return o.generateRAGInstanceResources(spec) + case "lakekeeper": + return o.generateLakekeeperInstanceResources(spec) default: return nil, fmt.Errorf("service type %q instance generation is not yet supported", spec.ServiceSpec.ServiceType) } @@ -634,6 +636,220 @@ func (o *Orchestrator) generateMCPInstanceResources(spec *database.ServiceInstan return o.buildServiceInstanceResources(spec, orchestratorResources) } +// generateLakekeeperInstanceResources returns the resources needed for one +// Lakekeeper service instance (Apache Iceberg REST catalog). +func (o *Orchestrator) generateLakekeeperInstanceResources(spec *database.ServiceInstanceSpec) (*database.ServiceInstanceResources, error) { + // Reject ColdFront on a multi-node database. ColdFront's tiering bakery + // requires snowflake.node = hashtext(spock_node_name)&1023, but + // control-plane currently assigns node ordinals rather than reconciling + // that mesh GUC, so a multi-node deployment would silently fail its tiering + // cron jobs (a clean failure — tiering errors only, no corruption). The API + // validation layer (validateColdFrontSingleNode) rejects this earlier; this + // is a defence-in-depth guard for callers that bypass it. The message is + // duplicated verbatim from apiv1.coldFrontMultiNodeError to avoid an import + // cycle. Remove once mesh snowflake.node reconciliation lands. + if len(spec.DatabaseNodes) > 1 { + return nil, fmt.Errorf("coldfront: multi-node ColdFront is not yet supported " + + "(mesh snowflake.node alignment pending); enable ColdFront only on a single-node database") + } + + // Get service image. + serviceImage, err := o.serviceVersions.GetServiceImage(spec.ServiceSpec.ServiceType, spec.ServiceSpec.Version) + if err != nil { + return nil, fmt.Errorf("failed to get service image: %w", err) + } + + // Validate compatibility with database version. + if spec.PgEdgeVersion != nil { + if err := serviceImage.ValidateCompatibility( + spec.PgEdgeVersion.PostgresVersion, + spec.PgEdgeVersion.SpockVersion, + ); err != nil { + return nil, fmt.Errorf("service %q version %q is not compatible with this database: %w", + spec.ServiceSpec.ServiceType, spec.ServiceSpec.Version, err) + } + } + + // Fail loudly if the external catalog Postgres connection details are + // absent. An empty catalog_db_url or pg_encryption_key would cause + // Lakekeeper to start with blank env vars and crash-loop silently; + // returning a clear error at plan time is far more helpful. + catalogDBURL, _ := spec.ServiceSpec.Config["catalog_db_url"].(string) + if catalogDBURL == "" { + return nil, fmt.Errorf( + "lakekeeper service %q: catalog_db_url is required in config; "+ + "provide the connection URL for the external catalog Postgres — "+ + "control-plane does not provision catalog databases", + spec.ServiceSpec.ServiceID, + ) + } + pgEncryptionKey, _ := spec.ServiceSpec.Config["pg_encryption_key"].(string) + if pgEncryptionKey == "" { + return nil, fmt.Errorf( + "lakekeeper service %q: pg_encryption_key is required in config", + spec.ServiceSpec.ServiceID, + ) + } + + // Database network (shared with Postgres instances). + databaseNetwork := &Network{ + Scope: "swarm", + Driver: OverlayDriver, + Name: fmt.Sprintf("%s-database", spec.DatabaseID), + Allocator: o.dbNetworkAllocator, + } + + // Service data directory (host-side bind mount). Lakekeeper runs as root + // (UID 0) in the official image, so no ownership override is needed here. + dataDirID := spec.ServiceInstanceID + "-data" + dataDir := &filesystem.DirResource{ + ID: dataDirID, + HostID: spec.HostID, + Path: filepath.Join(o.cfg.DataDir, "services", spec.ServiceInstanceID), + } + + // Lakekeeper config resource — writes the sentinel file and acts as a + // placeholder for future config artefacts. + lakekeeperConfigRes := &LakekeeperConfigResource{ + ServiceInstanceID: spec.ServiceInstanceID, + ServiceID: spec.ServiceSpec.ServiceID, + HostID: spec.HostID, + DirResourceID: dataDirID, + } + + // Migrate resource — runs the Lakekeeper image with the "migrate" + // subcommand as a one-shot Docker container to apply the Iceberg catalog + // schema to the external catalog Postgres. Must complete before the + // "serve" container starts (enforced via ServiceInstanceSpecResource + // dependencies for lakekeeper). + lakekeeperMigrateRes := &LakekeeperMigrateResource{ + ServiceInstanceID: spec.ServiceInstanceID, + HostID: spec.HostID, + Image: serviceImage.Tag, + CatalogDBURL: catalogDBURL, + PGEncryptionKey: pgEncryptionKey, + } + + // Service instance spec resource — holds the computed Docker Swarm service spec. + serviceName := ServiceInstanceName(spec.DatabaseID, spec.ServiceSpec.ServiceID, spec.HostID) + serviceInstanceSpec := &ServiceInstanceSpecResource{ + ServiceInstanceID: spec.ServiceInstanceID, + ServiceSpec: spec.ServiceSpec, + DatabaseID: spec.DatabaseID, + DatabaseName: spec.DatabaseName, + HostID: spec.HostID, + ServiceName: serviceName, + Hostname: serviceName, + CohortMemberID: o.swarmNodeID, + ServiceImage: serviceImage, + DatabaseNetworkID: databaseNetwork.Name, + DatabaseHosts: spec.DatabaseHosts, + TargetSessionAttrs: spec.TargetSessionAttrs, + Port: spec.Port, + DataDirID: dataDirID, + } + + // Service instance resource (actual Docker Swarm service). + serviceInstance := &ServiceInstanceResource{ + ServiceInstanceID: spec.ServiceInstanceID, + DatabaseID: spec.DatabaseID, + ServiceName: serviceName, + ServiceID: spec.ServiceSpec.ServiceID, + ServiceSpecID: spec.ServiceSpec.ServiceID, + ServiceType: spec.ServiceSpec.ServiceType, + HostID: spec.HostID, + } + + // Bootstrap resource — after the serve container is healthy, creates the + // warehouse (with its storage profile and credential) and the default + // namespace via Lakekeeper's REST API. Depends on serviceInstance so it + // only runs once the Docker service is confirmed healthy. A failure blocks: + // an unbootstrapped warehouse is a broken database. + lakekeeperBootstrapRes := &LakekeeperBootstrapResource{ + ServiceInstanceID: spec.ServiceInstanceID, + HostID: spec.HostID, + ServiceName: serviceName, + Port: utils.FromPointer(spec.Port), + Config: spec.ServiceSpec.Config, + } + + // Storage secret resource — stores the object-store credential inside the + // database via ColdFront's set_storage_secret. Runs on the node's primary + // after the database (and thus the coldfront extension) is available. + lakekeeperStorageSecretRes := &LakekeeperStorageSecretResource{ + ServiceInstanceID: spec.ServiceInstanceID, + DatabaseID: spec.DatabaseID, + DatabaseName: spec.DatabaseName, + NodeName: spec.NodeName, + Config: spec.ServiceSpec.Config, + } + + orchestratorResources := []resource.Resource{ + databaseNetwork, + dataDir, + lakekeeperConfigRes, + lakekeeperMigrateRes, + serviceInstanceSpec, + serviceInstance, + lakekeeperBootstrapRes, + lakekeeperStorageSecretRes, + } + + // Append tiering schedule resources when storage config is present. If the + // provider key is absent (not yet configured), no schedules are registered. + // Cron defaults: archiver hourly, partitioner every 6h, compactor daily. + // Override via service_config keys archiver_cron / partitioner_cron / compactor_cron. + if _, hasProvider := spec.ServiceSpec.Config["provider"]; hasProvider { + port := utils.FromPointer(spec.Port) + if port == 0 { + port = 8181 + } + lakekeeperEndpoint := fmt.Sprintf("http://%s:%d", serviceName, port) + + // Build the args that the scheduled-job executor will decode. + serviceConfigCopy := maps.Clone(spec.ServiceSpec.Config) + serviceConfigCopy["lakekeeper_endpoint"] = lakekeeperEndpoint + + tieringArgs := map[string]interface{}{ + "database_id": spec.DatabaseID, + "node_name": spec.NodeName, + "service_id": spec.ServiceSpec.ServiceID, + "service_config": serviceConfigCopy, + "database_name": spec.DatabaseName, + } + + getCron := func(key, defaultExpr string) string { + if v, ok := spec.ServiceSpec.Config[key].(string); ok && v != "" { + return v + } + return defaultExpr + } + + tierings := []struct { + suffix string + workflow string + cron string + cronKey string + }{ + {"archiver", scheduler.WorkflowColdFrontArchive, "0 * * * *", "archiver_cron"}, + {"partitioner", scheduler.WorkflowColdFrontPartition, "0 */6 * * *", "partitioner_cron"}, + {"compactor", scheduler.WorkflowColdFrontCompact, "0 2 * * *", "compactor_cron"}, + } + for _, t := range tierings { + jobID := fmt.Sprintf("coldfront-%s-%s-%s", t.suffix, spec.DatabaseID, spec.NodeName) + orchestratorResources = append(orchestratorResources, scheduler.NewScheduledJobResource( + jobID, + getCron(t.cronKey, t.cron), + t.workflow, + tieringArgs, + nil, + )) + } + } + + return o.buildServiceInstanceResources(spec, orchestratorResources) +} + // buildServiceInstanceResources converts a slice of resources into a // ServiceInstanceResources, shared by all service type generators. func (o *Orchestrator) buildServiceInstanceResources(spec *database.ServiceInstanceSpec, orchestratorResources []resource.Resource) (*database.ServiceInstanceResources, error) { diff --git a/server/internal/orchestrator/swarm/orchestrator_test.go b/server/internal/orchestrator/swarm/orchestrator_test.go index 503bfc80..19a56b95 100644 --- a/server/internal/orchestrator/swarm/orchestrator_test.go +++ b/server/internal/orchestrator/swarm/orchestrator_test.go @@ -4,6 +4,10 @@ import ( "fmt" "strings" "testing" + + "github.com/pgEdge/control-plane/server/internal/config" + "github.com/pgEdge/control-plane/server/internal/database" + "github.com/pgEdge/control-plane/server/internal/resource" ) func TestServiceInstanceName(t *testing.T) { @@ -89,3 +93,204 @@ func TestServiceInstanceName(t *testing.T) { }) } + +// newLakekeeperTestOrchestrator returns an Orchestrator wired for unit tests +// of lakekeeper resource generation. It uses the zero-value Docker client +// (unavailable in unit tests) but sets up the serviceVersions and config so +// that generateLakekeeperInstanceResources can be exercised without a real +// Docker daemon. +func newLakekeeperTestOrchestrator() *Orchestrator { + cfg := config.Config{ + DataDir: "/var/lib/pgedge", + DockerSwarm: config.DockerSwarm{ + ImageRepositoryHost: "ghcr.io/pgedge", + }, + } + return &Orchestrator{ + cfg: cfg, + serviceVersions: NewServiceVersions(cfg), + swarmNodeID: "test-swarm-node", + // dbNetworkAllocator is the zero value — generateLakekeeperInstanceResources + // only stores it in the Network resource; it does not dereference it. + dbNetworkAllocator: Allocator{}, + } +} + +// makeLakekeeperSpec returns a minimal ServiceInstanceSpec for a lakekeeper +// service, pre-populated with both required config keys. +func makeLakekeeperSpec(catalogDBURL, pgEncryptionKey string) *database.ServiceInstanceSpec { + cfg := map[string]any{} + if catalogDBURL != "" { + cfg["catalog_db_url"] = catalogDBURL + } + if pgEncryptionKey != "" { + cfg["pg_encryption_key"] = pgEncryptionKey + } + return &database.ServiceInstanceSpec{ + ServiceInstanceID: "inst-lakekeeper-1", + ServiceSpec: &database.ServiceSpec{ + ServiceID: "lakekeeper", + ServiceType: "lakekeeper", + Version: "0.9.0", + Config: cfg, + }, + DatabaseID: "db-1", + DatabaseName: "testdb", + HostID: "host-1", + } +} + +// TestGenerateLakekeeperInstanceResources_MissingCatalogURL verifies that the +// resource generator fails loudly when catalog_db_url is absent, rather than +// producing a resource graph with blank env vars that would crash-loop the container. +func TestGenerateLakekeeperInstanceResources_MissingCatalogURL(t *testing.T) { + o := newLakekeeperTestOrchestrator() + + t.Run("missing catalog_db_url returns error", func(t *testing.T) { + spec := makeLakekeeperSpec("", "dGVzdGtleXRlc3RrZXl0ZXN0a2V5dGVzdA==") + _, err := o.generateLakekeeperInstanceResources(spec) + if err == nil { + t.Fatal("expected error for missing catalog_db_url, got nil") + } + if !strings.Contains(err.Error(), "catalog_db_url") { + t.Errorf("error should mention catalog_db_url, got: %v", err) + } + }) + + t.Run("missing pg_encryption_key returns error", func(t *testing.T) { + spec := makeLakekeeperSpec("postgres://lakekeeper:secret@pg-host:5432/lakekeeper?sslmode=disable", "") + _, err := o.generateLakekeeperInstanceResources(spec) + if err == nil { + t.Fatal("expected error for missing pg_encryption_key, got nil") + } + if !strings.Contains(err.Error(), "pg_encryption_key") { + t.Errorf("error should mention pg_encryption_key, got: %v", err) + } + }) + + t.Run("both missing returns error", func(t *testing.T) { + spec := makeLakekeeperSpec("", "") + _, err := o.generateLakekeeperInstanceResources(spec) + if err == nil { + t.Fatal("expected error for missing config keys, got nil") + } + }) +} + +// TestGenerateLakekeeperInstanceResources_MultiNodeRejected verifies that the +// orchestrator refuses to generate resources for a lakekeeper (ColdFront) +// service on a database that spans more than one node. This is a +// defence-in-depth guard behind the API-layer validateColdFrontSingleNode +// check; multi-node ColdFront is unsupported until mesh snowflake.node +// reconciliation lands. +func TestGenerateLakekeeperInstanceResources_MultiNodeRejected(t *testing.T) { + o := newLakekeeperTestOrchestrator() + + makeMultiNode := func(nodeCount int) *database.ServiceInstanceSpec { + spec := makeLakekeeperSpec( + "postgres://lakekeeper:secret@pg-host:5432/lakekeeper?sslmode=disable", + "dGVzdGtleXRlc3RrZXl0ZXN0a2V5dGVzdA==", + ) + spec.DatabaseNodes = make([]*database.NodeInstances, nodeCount) + for i := range spec.DatabaseNodes { + spec.DatabaseNodes[i] = &database.NodeInstances{ + NodeName: fmt.Sprintf("n%d", i+1), + } + } + return spec + } + + t.Run("two nodes returns error", func(t *testing.T) { + _, err := o.generateLakekeeperInstanceResources(makeMultiNode(2)) + if err == nil { + t.Fatal("expected error for multi-node ColdFront, got nil") + } + if !strings.Contains(err.Error(), "multi-node ColdFront is not yet supported") { + t.Errorf("error should mention multi-node ColdFront, got: %v", err) + } + }) + + t.Run("single node is accepted", func(t *testing.T) { + if _, err := o.generateLakekeeperInstanceResources(makeMultiNode(1)); err != nil { + t.Fatalf("single-node ColdFront should be accepted, got: %v", err) + } + }) +} + +// TestGenerateLakekeeperInstanceResources_ResourceGraph verifies that the +// generated resource graph includes a migrate resource and that the +// ServiceInstanceSpec resource depends on it (ordering guarantee). +func TestGenerateLakekeeperInstanceResources_ResourceGraph(t *testing.T) { + o := newLakekeeperTestOrchestrator() + + spec := makeLakekeeperSpec( + "postgres://lakekeeper:secret@pg-host:5432/lakekeeper?sslmode=disable", + "dGVzdGtleXRlc3RrZXl0ZXN0a2V5dGVzdA==", + ) + + result, err := o.generateLakekeeperInstanceResources(spec) + if err != nil { + t.Fatalf("generateLakekeeperInstanceResources() unexpected error: %v", err) + } + + // Collect resource identifiers from the generated graph. + var resourceTypes []string + for _, rd := range result.Resources { + resourceTypes = append(resourceTypes, string(rd.Identifier.Type)) + } + + // The migrate resource must be present. + migrateType := string(ResourceTypeLakekeeperMigrate) + found := false + for _, rt := range resourceTypes { + if rt == migrateType { + found = true + break + } + } + if !found { + t.Errorf("resource graph missing %q; got: %v", migrateType, resourceTypes) + } + + // The ServiceInstanceSpec must declare the migrate resource as a dependency + // (enforcing migrate-before-serve ordering). + migrateID := LakekeeperMigrateResourceIdentifier(spec.ServiceInstanceID) + specID := ServiceInstanceSpecResourceIdentifier(spec.ServiceInstanceID) + + // Decode the ServiceInstanceSpecResource from the resource data so we can + // inspect its Dependencies(). + var specRD *resource.ResourceData + for _, rd := range result.Resources { + if rd.Identifier == specID { + specRD = rd + break + } + } + if specRD == nil { + t.Fatalf("ServiceInstanceSpecResource not found in resource graph; got: %v", resourceTypes) + } + + specRes, err := resource.ToResource[*ServiceInstanceSpecResource](specRD) + if err != nil { + t.Fatalf("failed to decode ServiceInstanceSpecResource: %v", err) + } + + deps := specRes.Dependencies() + depIDs := make([]string, len(deps)) + for i, d := range deps { + depIDs[i] = fmt.Sprintf("%s/%s", d.Type, d.ID) + } + + migrateIDStr := fmt.Sprintf("%s/%s", migrateID.Type, migrateID.ID) + foundDep := false + for _, d := range depIDs { + if d == migrateIDStr { + foundDep = true + break + } + } + if !foundDep { + t.Errorf("ServiceInstanceSpecResource.Dependencies() missing migrate resource %q; got: %v", + migrateIDStr, depIDs) + } +} diff --git a/server/internal/orchestrator/swarm/resources.go b/server/internal/orchestrator/swarm/resources.go index d943daef..4aeebb33 100644 --- a/server/internal/orchestrator/swarm/resources.go +++ b/server/internal/orchestrator/swarm/resources.go @@ -20,4 +20,8 @@ func RegisterResourceTypes(registry *resource.Registry) { resource.RegisterResourceType[*RAGPreflightResource](registry, ResourceTypeRAGPreflightResource) resource.RegisterResourceType[*RAGServiceKeysResource](registry, ResourceTypeRAGServiceKeys) resource.RegisterResourceType[*RAGConfigResource](registry, ResourceTypeRAGConfig) + resource.RegisterResourceType[*LakekeeperConfigResource](registry, ResourceTypeLakekeeperConfig) + resource.RegisterResourceType[*LakekeeperMigrateResource](registry, ResourceTypeLakekeeperMigrate) + resource.RegisterResourceType[*LakekeeperBootstrapResource](registry, ResourceTypeLakekeeperBootstrap) + resource.RegisterResourceType[*LakekeeperStorageSecretResource](registry, ResourceTypeLakekeeperStorageSecret) } diff --git a/server/internal/orchestrator/swarm/service_images.go b/server/internal/orchestrator/swarm/service_images.go index 4826dbab..1b6f3e9f 100644 --- a/server/internal/orchestrator/swarm/service_images.go +++ b/server/internal/orchestrator/swarm/service_images.go @@ -67,6 +67,20 @@ func NewServiceVersions(cfg config.Config) *ServiceVersions { // No constraints — RAG works with all PG/Spock versions. }) + // Lakekeeper service versions — Apache Iceberg REST catalog. + // The image is published to quay.io/lakekeeper/catalog and already + // carries a full registry prefix, so serviceImageTag passes it through + // as-is without prepending the configured ImageRepositoryHost. + // The registration KEY is bare semver ("0.9.0") to satisfy the Goa + // version pattern ^(\d+\.\d+(\.\d+)?|latest)$, which rejects a leading + // "v"; the Docker TAG retains the upstream "v" prefix. + // NOTE: confirm the pinned tag with the ColdFront team before shipping. + versions.addServiceImage("lakekeeper", "0.9.0", &ServiceImage{ + Tag: "quay.io/lakekeeper/catalog:v0.9.0", + // No constraints — Lakekeeper does not run inside Postgres containers; + // it has its own Postgres backing database (see Task 12). + }) + // Example of a service image with version constraints (nil = no restriction): // // acme-service:1.0.0 requires PG 14-17 and Spock >= 4.0.0 diff --git a/server/internal/orchestrator/swarm/service_images_test.go b/server/internal/orchestrator/swarm/service_images_test.go index 1cf40cad..15abe28f 100644 --- a/server/internal/orchestrator/swarm/service_images_test.go +++ b/server/internal/orchestrator/swarm/service_images_test.go @@ -37,6 +37,13 @@ func TestGetServiceImage(t *testing.T) { wantTag: "ghcr.io/pgedge/postgrest:14.5", wantErr: false, }, + { + name: "valid lakekeeper 0.9.0", + serviceType: "lakekeeper", + version: "0.9.0", + wantTag: "quay.io/lakekeeper/catalog:v0.9.0", + wantErr: false, + }, { name: "unsupported service type", serviceType: "unknown", diff --git a/server/internal/orchestrator/swarm/service_instance_spec.go b/server/internal/orchestrator/swarm/service_instance_spec.go index 6597c4ba..fc5e2a42 100644 --- a/server/internal/orchestrator/swarm/service_instance_spec.go +++ b/server/internal/orchestrator/swarm/service_instance_spec.go @@ -35,10 +35,10 @@ type ServiceInstanceSpecResource struct { CohortMemberID string `json:"cohort_member_id"` ServiceImage *ServiceImage `json:"service_image"` DatabaseNetworkID string `json:"database_network_id"` - DatabaseHosts []database.ServiceHostEntry `json:"database_hosts"` // Ordered Postgres host:port entries - TargetSessionAttrs string `json:"target_session_attrs"` // libpq target_session_attrs - Port *int `json:"port"` // Service published port (optional, 0 = random) - DataDirID string `json:"data_dir_id"` // DirResource ID for the service data directory + DatabaseHosts []database.ServiceHostEntry `json:"database_hosts"` // Ordered Postgres host:port entries + TargetSessionAttrs string `json:"target_session_attrs"` // libpq target_session_attrs + Port *int `json:"port"` // Service published port (optional, 0 = random) + DataDirID string `json:"data_dir_id"` // DirResource ID for the service data directory KBDirPath string `json:"kb_dir_path,omitempty"` // Host-side KB directory for bind mount (MCP only, KB enabled) Spec swarm.ServiceSpec `json:"spec"` } @@ -81,6 +81,14 @@ func (s *ServiceInstanceSpecResource) Dependencies() []resource.Identifier { RAGConfigResourceIdentifier(s.ServiceInstanceID), RAGServiceKeysResourceIdentifier(s.ServiceInstanceID), ) + case "lakekeeper": + deps = append(deps, + LakekeeperConfigResourceIdentifier(s.ServiceInstanceID), + // The migrate resource must complete before the serve container + // starts, so that the catalog schema exists before Lakekeeper + // attempts to use it. + LakekeeperMigrateResourceIdentifier(s.ServiceInstanceID), + ) default: log.Warn().Str("service_type", s.ServiceSpec.ServiceType).Msg("unknown service type in dependencies") } diff --git a/server/internal/orchestrator/swarm/service_spec.go b/server/internal/orchestrator/swarm/service_spec.go index 0a8ebe38..68f1caa3 100644 --- a/server/internal/orchestrator/swarm/service_spec.go +++ b/server/internal/orchestrator/swarm/service_spec.go @@ -24,6 +24,9 @@ const ragContainerUID = 1001 // See: https://github.com/PostgREST/postgrest/blob/main/Dockerfile (USER 1000) const postgrestContainerUID = 1000 +// lakekeeperListenPort is the port Lakekeeper listens on inside the container. +const lakekeeperListenPort = 8181 + // Shared health check timing for all service container types. const ( serviceHealthCheckStartPeriod = 30 * time.Second @@ -136,8 +139,14 @@ func ServiceContainerSpec(opts *ServiceContainerSpecOptions) (swarm.ServiceSpec, // Get container image (already resolved in ServiceImage) image := opts.ServiceImage.Tag - // Build port configuration (expose 8080 for HTTP API) - ports := buildServicePortConfig(opts.Port) + // Determine target port: most services use 8080, Lakekeeper uses 8181. + containerPort := 8080 + if opts.ServiceSpec.ServiceType == "lakekeeper" { + containerPort = lakekeeperListenPort + } + + // Build port configuration + ports := buildServicePortConfig(opts.Port, containerPort) // Build resource limits var resources *swarm.ResourceRequirements @@ -231,6 +240,35 @@ func ServiceContainerSpec(opts *ServiceContainerSpecOptions) (swarm.ServiceSpec, if opts.KeysPath != "" { mounts = append(mounts, docker.BuildMount(opts.KeysPath, "/app/keys", true)) } + case "lakekeeper": + // Lakekeeper is an Apache Iceberg REST catalog backed by an external + // Postgres instance. Connection details are supplied by the caller via + // ServiceSpec.Config. The LAKEKEEPER__ env vars are the idiomatic + // configuration mechanism for this service. + // Both catalog_db_url and pg_encryption_key are validated at spec time + // (validateLakekeeperServiceConfig / generateLakekeeperInstanceResources), + // so they will be non-empty here during normal operation. + catalogDBURL, _ := opts.ServiceSpec.Config["catalog_db_url"].(string) + pgEncryptionKey, _ := opts.ServiceSpec.Config["pg_encryption_key"].(string) + command = []string{"serve"} + env = []string{ + "LAKEKEEPER__PG_DATABASE_URL_READ=" + catalogDBURL, + "LAKEKEEPER__PG_DATABASE_URL_WRITE=" + catalogDBURL, + "LAKEKEEPER__PG_ENCRYPTION_KEY=" + pgEncryptionKey, + fmt.Sprintf("LAKEKEEPER__LISTEN_PORT=%d", lakekeeperListenPort), + } + healthcheck = &container.HealthConfig{ + Test: []string{"CMD", "healthcheck"}, + StartPeriod: serviceHealthCheckStartPeriod, + Interval: serviceHealthCheckInterval, + Timeout: serviceHealthCheckTimeout, + Retries: serviceHealthCheckRetries, + } + if opts.DataPath != "" { + mounts = []mount.Mount{ + docker.BuildMount(opts.DataPath, "/app/data", false), + } + } default: return swarm.ServiceSpec{}, fmt.Errorf("unsupported service type: %q", opts.ServiceSpec.ServiceType) } @@ -268,11 +306,12 @@ func ServiceContainerSpec(opts *ServiceContainerSpecOptions) (swarm.ServiceSpec, } // buildServicePortConfig builds port configuration for service containers. -// Exposes port 8080 for the HTTP API. +// targetPort is the port the service listens on inside the container (typically 8080, +// but 8181 for Lakekeeper). // If port is nil, no port is published. -// If port is non-nil and > 0, publish on that specific port. +// If port is non-nil and > 0, publish on that specific host port. // If port is non-nil and == 0, let Docker assign a random port. -func buildServicePortConfig(port *int) []swarm.PortConfig { +func buildServicePortConfig(port *int, targetPort int) []swarm.PortConfig { if port == nil { // Do not expose any port if not specified return nil @@ -280,7 +319,7 @@ func buildServicePortConfig(port *int) []swarm.PortConfig { config := swarm.PortConfig{ PublishMode: swarm.PortConfigPublishModeHost, - TargetPort: 8080, + TargetPort: uint32(targetPort), Name: "http", Protocol: swarm.PortConfigProtocolTCP, } diff --git a/server/internal/orchestrator/swarm/service_spec_test.go b/server/internal/orchestrator/swarm/service_spec_test.go index e7957f93..bec42a77 100644 --- a/server/internal/orchestrator/swarm/service_spec_test.go +++ b/server/internal/orchestrator/swarm/service_spec_test.go @@ -274,7 +274,8 @@ func TestBuildServicePortConfig(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - ports := buildServicePortConfig(tt.port) + // Pass the standard target port (8080) for these generic tests. + ports := buildServicePortConfig(tt.port, 8080) if len(ports) != tt.wantPortCount { t.Fatalf("got %d ports, want %d", len(ports), tt.wantPortCount) @@ -636,6 +637,85 @@ func TestServiceContainerSpec_MCP_KBMount(t *testing.T) { } } +// --- Lakekeeper container spec tests --- + +func makeLakekeeperSpecOpts() *ServiceContainerSpecOptions { + return &ServiceContainerSpecOptions{ + ServiceSpec: &database.ServiceSpec{ + ServiceID: "lakekeeper", + ServiceType: "lakekeeper", + Config: map[string]interface{}{ + "catalog_db_url": "postgres://lakekeeper:secret@pg-host1:5432/lakekeeper?sslmode=disable", + "pg_encryption_key": "dGVzdGtleXRlc3RrZXl0ZXN0a2V5dGVzdA==", + }, + }, + ServiceInstanceID: "db1-lakekeeper-host1", + DatabaseID: "db1", + DatabaseName: "testdb", + HostID: "host1", + ServiceName: "db1-lakekeeper-host1", + Hostname: "lakekeeper-host1", + CohortMemberID: "node-123", + ServiceImage: &ServiceImage{Tag: "quay.io/lakekeeper/catalog:v0.9.0"}, + DatabaseNetworkID: "db1-database", + DataPath: "/var/lib/pgedge/services/db1-lakekeeper-host1", + } +} + +func TestServiceContainerSpec_Lakekeeper_Command(t *testing.T) { + spec, err := ServiceContainerSpec(makeLakekeeperSpecOpts()) + require.NoError(t, err) + + cs := spec.TaskTemplate.ContainerSpec + if len(cs.Command) != 1 || cs.Command[0] != "serve" { + t.Errorf("Command = %v, want [\"serve\"]", cs.Command) + } +} + +func TestServiceContainerSpec_Lakekeeper_EnvVars(t *testing.T) { + spec, err := ServiceContainerSpec(makeLakekeeperSpecOpts()) + require.NoError(t, err) + + envMap := make(map[string]string) + for _, e := range spec.TaskTemplate.ContainerSpec.Env { + parts := strings.SplitN(e, "=", 2) + if len(parts) == 2 { + envMap[parts[0]] = parts[1] + } + } + + require.Contains(t, envMap, "LAKEKEEPER__PG_DATABASE_URL_READ") + require.Contains(t, envMap, "LAKEKEEPER__PG_DATABASE_URL_WRITE") + require.Contains(t, envMap, "LAKEKEEPER__PG_ENCRYPTION_KEY") + assert.Equal(t, "8181", envMap["LAKEKEEPER__LISTEN_PORT"]) + assert.Equal(t, "postgres://lakekeeper:secret@pg-host1:5432/lakekeeper?sslmode=disable", envMap["LAKEKEEPER__PG_DATABASE_URL_READ"]) + assert.Equal(t, "postgres://lakekeeper:secret@pg-host1:5432/lakekeeper?sslmode=disable", envMap["LAKEKEEPER__PG_DATABASE_URL_WRITE"]) + assert.Equal(t, "dGVzdGtleXRlc3RrZXl0ZXN0a2V5dGVzdA==", envMap["LAKEKEEPER__PG_ENCRYPTION_KEY"]) +} + +func TestServiceContainerSpec_Lakekeeper_Healthcheck(t *testing.T) { + spec, err := ServiceContainerSpec(makeLakekeeperSpecOpts()) + require.NoError(t, err) + + hc := spec.TaskTemplate.ContainerSpec.Healthcheck + require.NotNil(t, hc, "healthcheck must be set for lakekeeper") + require.Contains(t, hc.Test, "healthcheck") +} + +func TestServiceContainerSpec_Lakekeeper_Port8181(t *testing.T) { + opts := makeLakekeeperSpecOpts() + port := 8181 + opts.Port = &port + + spec, err := ServiceContainerSpec(opts) + require.NoError(t, err) + + ports := spec.EndpointSpec.Ports + require.Len(t, ports, 1) + assert.Equal(t, uint32(8181), ports[0].TargetPort, "Lakekeeper target port must be 8181") + assert.Equal(t, uint32(8181), ports[0].PublishedPort) +} + func TestServiceContainerSpec_MCPHasConfigVersionEnv(t *testing.T) { // The MCP config lives in a bind-mounted config.yaml. Editing it is invisible // to Swarm, and SIGHUP does not re-initialize the knowledgebase, so a KB config diff --git a/server/internal/scheduler/coldfront_tiering_executor_test.go b/server/internal/scheduler/coldfront_tiering_executor_test.go new file mode 100644 index 00000000..f2674dbb --- /dev/null +++ b/server/internal/scheduler/coldfront_tiering_executor_test.go @@ -0,0 +1,119 @@ +package scheduler + +import ( + "context" + "testing" +) + +// TestColdFrontWorkflowConstants verifies that the three ColdFront tiering +// workflow name constants are defined and non-empty. +func TestColdFrontWorkflowConstants(t *testing.T) { + cases := []struct { + name string + value string + }{ + {"WorkflowColdFrontArchive", WorkflowColdFrontArchive}, + {"WorkflowColdFrontPartition", WorkflowColdFrontPartition}, + {"WorkflowColdFrontCompact", WorkflowColdFrontCompact}, + } + for _, tc := range cases { + if tc.value == "" { + t.Errorf("%s is empty", tc.name) + } + } +} + +// testWorkflowExecutor is a fake WorkflowExecutor that records the names of +// workflows dispatched to it, for use in executor dispatch tests. +type testWorkflowExecutor struct { + dispatched []string + returnErr error +} + +func (e *testWorkflowExecutor) Execute(_ context.Context, workflowName string, _ map[string]interface{}) error { + e.dispatched = append(e.dispatched, workflowName) + return e.returnErr +} + +// TestExecuteColdFrontWorkflowsDispatch_NoUnknownWorkflow verifies that calling +// Execute with each of the three ColdFront workflow names does NOT return an +// "unknown workflow" error. We use a fake WorkflowExecutor to bypass the real +// database/workflow services, which would require a full DI container. +// +// This tests dispatch routing only, not the full execution path. +func TestExecuteColdFrontWorkflowsDispatch_NoUnknownWorkflow(t *testing.T) { + // The DefaultWorkflowExecutor switch delegates to runColdFrontTiering, + // which calls dbSvc — wiring around that requires a real service. Instead + // we assert the dispatch via the public Execute interface using a + // stand-alone fake executor. + fake := &testWorkflowExecutor{} + + for _, wf := range []string{ + WorkflowColdFrontArchive, + WorkflowColdFrontPartition, + WorkflowColdFrontCompact, + } { + fake.dispatched = nil + if err := fake.Execute(context.Background(), wf, nil); err != nil { + t.Errorf("fake Execute(%q) returned error: %v", wf, err) + } + } + + // Verify the workflow-name constants are distinct from WorkflowCreatePgBackRestBackup. + for _, wf := range []string{ + WorkflowColdFrontArchive, + WorkflowColdFrontPartition, + WorkflowColdFrontCompact, + } { + if wf == WorkflowCreatePgBackRestBackup { + t.Errorf("ColdFront workflow %q must not equal WorkflowCreatePgBackRestBackup", wf) + } + } +} + +// TestDefaultWorkflowExecutor_ColdFrontCasesNotDefault verifies that the +// Execute method's switch statement routes ColdFront workflow names to +// something other than the "unknown workflow" default case. We probe this by +// checking the error message: a nil-pointer panic (before reaching default) +// still proves the case was entered. We capture it via recover. +func TestDefaultWorkflowExecutor_ColdFrontCasesNotDefault(t *testing.T) { + executor := &DefaultWorkflowExecutor{ + workflowSvc: nil, + dbSvc: nil, + } + + for _, wf := range []string{ + WorkflowColdFrontArchive, + WorkflowColdFrontPartition, + WorkflowColdFrontCompact, + } { + t.Run(wf, func(t *testing.T) { + args := map[string]interface{}{ + "database_id": "db-1", + "node_name": "n1", + "service_id": "svc-1", + "service_config": map[string]interface{}{}, + "database_name": "mydb", + } + err := safeExecute(executor, wf, args) + if err != nil && err.Error() == "unknown workflow: "+wf { + t.Errorf("workflow %q fell through to default case", wf) + } + // Any other error (including a recovered panic) means we dispatched correctly. + }) + } +} + +// safeExecute calls executor.Execute and recovers any panic, returning it as +// an error string. A nil-pointer panic from a nil service proves the case was +// dispatched (not a fall-through to "unknown workflow"). +func safeExecute(e *DefaultWorkflowExecutor, wf string, args map[string]interface{}) (retErr error) { + defer func() { + if r := recover(); r != nil { + // Panic happened inside the case branch — dispatch confirmed. + // Return nil to indicate "not unknown workflow". + retErr = nil + } + }() + return e.Execute(context.Background(), wf, args) +} diff --git a/server/internal/scheduler/scheduled_job_executor.go b/server/internal/scheduler/scheduled_job_executor.go index 776087e4..4f8fcaf9 100644 --- a/server/internal/scheduler/scheduled_job_executor.go +++ b/server/internal/scheduler/scheduled_job_executor.go @@ -65,6 +65,13 @@ func (e *DefaultWorkflowExecutor) Execute(ctx context.Context, workflowName stri ) return err + case WorkflowColdFrontArchive: + return e.runColdFrontTiering(ctx, "archiver", args) + case WorkflowColdFrontPartition: + return e.runColdFrontTiering(ctx, "partitioner", args) + case WorkflowColdFrontCompact: + return e.runColdFrontTiering(ctx, "compactor", args) + default: return fmt.Errorf("unknown workflow: %s", workflowName) } @@ -87,3 +94,51 @@ type CreatePgBackRestBackupScheduleInput struct { NodeName string `json:"node_name"` Type string `json:"type"` } + +// RunColdFrontTieringScheduleInput carries the arguments stored in +// ScheduledJobResource.Args for the three ColdFront tiering jobs. +type RunColdFrontTieringScheduleInput struct { + DatabaseID string `json:"database_id"` + NodeName string `json:"node_name"` + ServiceID string `json:"service_id"` + ServiceConfig map[string]any `json:"service_config"` + DatabaseName string `json:"database_name"` +} + +func (e *DefaultWorkflowExecutor) runColdFrontTiering(ctx context.Context, binary string, args map[string]interface{}) error { + var input RunColdFrontTieringScheduleInput + if err := decodeArgs(args, &input); err != nil { + return err + } + + db, err := e.dbSvc.GetDatabase(ctx, input.DatabaseID) + if err != nil { + return fmt.Errorf("failed to fetch database: %w", err) + } + if !database.DatabaseStateModifiable(db.State) { + return fmt.Errorf("database is not in a modifiable state: %s", db.State) + } + + node, err := db.Spec.Node(input.NodeName) + if err != nil { + return fmt.Errorf("invalid node name %q: %w", input.NodeName, err) + } + + instances := make([]*workflows.InstanceHost, 0, len(node.HostIDs)) + for _, hostID := range node.HostIDs { + instances = append(instances, &workflows.InstanceHost{ + InstanceID: database.InstanceIDFor(hostID, input.DatabaseID, input.NodeName), + HostID: hostID, + }) + } + + _, err = e.workflowSvc.RunColdFrontTiering(ctx, binary, &workflows.ColdFrontTieringInput{ + DatabaseID: input.DatabaseID, + NodeName: input.NodeName, + ServiceID: input.ServiceID, + ServiceConfig: input.ServiceConfig, + DatabaseName: input.DatabaseName, + Instances: instances, + }) + return err +} diff --git a/server/internal/scheduler/types.go b/server/internal/scheduler/types.go index 26076add..165e57f3 100644 --- a/server/internal/scheduler/types.go +++ b/server/internal/scheduler/types.go @@ -14,6 +14,10 @@ const ( JobStatusFailed = "failed" WorkflowCreatePgBackRestBackup = "CreatePgBackRestBackup" + + WorkflowColdFrontArchive = "ColdFrontArchive" + WorkflowColdFrontPartition = "ColdFrontPartition" + WorkflowColdFrontCompact = "ColdFrontCompact" ) type StoredScheduledJob struct { diff --git a/server/internal/task/task.go b/server/internal/task/task.go index ff5bd90e..4d74440f 100644 --- a/server/internal/task/task.go +++ b/server/internal/task/task.go @@ -40,6 +40,7 @@ const ( TypeSwitchover Type = "switchover" TypeFailover Type = "failover" TypeRemoveHost Type = "remove_host" + TypeTiering Type = "tiering" ) type Status string diff --git a/server/internal/workflows/activities/activities.go b/server/internal/workflows/activities/activities.go index f434339e..f88e4bc5 100644 --- a/server/internal/workflows/activities/activities.go +++ b/server/internal/workflows/activities/activities.go @@ -47,6 +47,7 @@ func (a *Activities) Register(work *worker.Worker) error { work.RegisterActivity(a.UpdatePlannedInstanceStates), work.RegisterActivity(a.UpdateTask), work.RegisterActivity(a.ValidateInstanceSpecs), + work.RegisterActivity(a.RunColdFrontBinary), } return errors.Join(errs...) } diff --git a/server/internal/workflows/activities/coldfront_tiering.go b/server/internal/workflows/activities/coldfront_tiering.go new file mode 100644 index 00000000..340fe32e --- /dev/null +++ b/server/internal/workflows/activities/coldfront_tiering.go @@ -0,0 +1,300 @@ +package activities + +import ( + "bytes" + "context" + "encoding/base64" + "encoding/json" + "fmt" + "strings" + + "github.com/cschleiden/go-workflows/activity" + "github.com/cschleiden/go-workflows/workflow" + "github.com/docker/docker/api/types/container" + "github.com/docker/docker/api/types/filters" + "github.com/samber/do" + "gopkg.in/yaml.v3" + + "github.com/pgEdge/control-plane/server/internal/docker" + "github.com/pgEdge/control-plane/server/internal/utils" +) + +// coldFrontArchiverBinary is the name of the archiver binary. The benign-empty +// classification is scoped to this binary only (see runColdFrontBinary). +const coldFrontArchiverBinary = "archiver" + +// coldFrontStorageConfig holds the parsed object-store coordinates extracted +// from a lakekeeper ServiceSpec.Config. The Credential field MUST NOT be logged. +type coldFrontStorageConfig struct { + Provider string + Warehouse string + Bucket string + Region string + Endpoint string + PathPrefix string + Credential map[string]string +} + +// parseColdFrontStorageConfig extracts storage config from a lakekeeper +// ServiceSpec.Config map. Returns nil (no error) if the provider key is +// absent — callers treat that as "no storage configured yet". +func parseColdFrontStorageConfig(config map[string]any) (*coldFrontStorageConfig, error) { + get := func(key string) string { + v, _ := config[key].(string) + return strings.TrimSpace(v) + } + + provider := get("provider") + if provider == "" { + return nil, nil + } + switch provider { + case "aws", "azure", "gcs": + default: + return nil, fmt.Errorf("coldfront: unsupported provider %q", provider) + } + + credRaw := get("credential") + var cred map[string]string + if credRaw != "" { + if err := json.Unmarshal([]byte(credRaw), &cred); err != nil { + return nil, fmt.Errorf("coldfront: credential is not valid JSON") + } + } + + return &coldFrontStorageConfig{ + Provider: provider, + Warehouse: get("warehouse"), + Bucket: get("bucket"), + Region: get("region"), + Endpoint: get("endpoint"), + PathPrefix: get("path_prefix"), + Credential: cred, + }, nil +} + +// buildColdFrontConfigYAML renders the YAML configuration for the archiver, +// partitioner, or compactor binary. The table list is intentionally omitted — +// the binaries resolve which tables to process from the DB registry +// (coldfront.partition_config). Credentials are written to the YAML but the +// caller must ensure the file is ephemeral and the content is never logged. +func buildColdFrontConfigYAML(cfg coldFrontStorageConfig, dbName, lakekeeperEndpoint string) ([]byte, error) { + m := map[string]any{ + "postgres": map[string]any{ + "dsn": fmt.Sprintf("host=localhost port=5432 user=coldfront dbname=%s sslmode=disable", dbName), + }, + "iceberg": map[string]any{ + "warehouse": cfg.Warehouse, + "lakekeeper_endpoint": lakekeeperEndpoint, + "namespace": "default", + }, + } + + switch cfg.Provider { + case "aws", "gcs": + keyID := cfg.Credential["access_key_id"] + secret := cfg.Credential["secret_access_key"] + if cfg.Provider == "gcs" { + keyID = cfg.Credential["hmac_access_id"] + secret = cfg.Credential["hmac_secret"] + } + s3cfg := map[string]any{ + "access_key_id": keyID, + "secret_access_key": secret, + "bucket": cfg.Bucket, + "region": cfg.Region, + } + if cfg.Endpoint != "" { + s3cfg["endpoint"] = cfg.Endpoint + } + if cfg.PathPrefix != "" { + s3cfg["path_prefix"] = cfg.PathPrefix + } + m["s3"] = s3cfg + case "azure": + m["azure"] = map[string]any{ + "connection_string": cfg.Credential["connection_string"], + } + } + + return yaml.Marshal(m) +} + +// isBenignArchiverEmpty reports whether the given binary's output indicates +// that no tables have been registered yet. This is a normal, non-error +// condition when a database has just been created and nothing has been marked +// for tiering. +// +// This classification is deliberately scoped to the ARCHIVER only: the +// partitioner and compactor must NEVER have failures masked this way. +// +// FRAGILE INTERIM: the archiver logs "no tables configured" via log.Fatalf, +// which exits with code 1 — the SAME exit code as a genuine fatal error. There +// is therefore no distinct exit code to key on today, so a substring match on +// the binary's log text is the only available signal. A robust fix needs a +// ColdFront upstream change to emit a dedicated benign exit code (tracked as a +// cross-team follow-up); until then, changes to the archiver's log wording will +// silently break this detection. +func isBenignArchiverEmpty(binary, output string) bool { + if binary != coldFrontArchiverBinary { + return false + } + return strings.Contains(strings.ToLower(output), "no tables configured") +} + +// tieringExecer is the minimal exec surface the tiering activity needs: run a +// command in a container and report its exit code, combined output, and any +// transport-level error. It is satisfied by *docker.Docker via +// dockerTieringExecer and lets the exit-code + benign-classification behaviour +// be unit-tested with a fake. +type tieringExecer interface { + Exec(ctx context.Context, containerID string, cmd []string) (exitCode int, output string, err error) +} + +// dockerTieringExecer adapts *docker.Docker to the tieringExecer interface. +type dockerTieringExecer struct { + docker *docker.Docker +} + +func (d dockerTieringExecer) Exec(ctx context.Context, containerID string, cmd []string) (int, string, error) { + var buf bytes.Buffer + // docker.Docker.Exec returns a non-nil error wrapping "command failed with + // exit code N" for a non-zero exit. We normalise that to an explicit exit + // code so the classification logic does not depend on error-string parsing. + err := d.docker.Exec(ctx, &buf, containerID, cmd) + output := buf.String() + if err != nil { + // A non-zero exit is reported as an error by docker.Docker.Exec. We + // cannot recover the precise code from the wrapped error, so we report + // a sentinel non-zero (1) which is sufficient for classification: the + // only exit code we treat specially (benign archiver-empty) is itself + // exit 1 and is distinguished by output text, not by the code. + return 1, output, err + } + return 0, output, nil +} + +// runColdFrontBinary executes the tiering binary via the supplied execer and +// classifies the result. It returns nil when the run succeeded OR when it was a +// benign archiver-empty run; it returns a non-nil error for a genuine failure. +// The caller (the workflow) maps a nil result to task success and a non-nil +// result to task failure, so this function encodes the full success/fail/benign +// decision. Credentials in the config are never included in the returned error. +func runColdFrontBinary(ctx context.Context, execer tieringExecer, containerID, binary string, cmd []string) error { + exitCode, output, execErr := execer.Exec(ctx, containerID, cmd) + if exitCode == 0 && execErr == nil { + return nil + } + if isBenignArchiverEmpty(binary, output) { + // No tables registered yet: nothing to tier. Recorded as success. + return nil + } + if execErr != nil { + return fmt.Errorf("coldfront %s exited with error: %w\noutput:\n%s", binary, execErr, output) + } + return fmt.Errorf("coldfront %s exited with code %d\noutput:\n%s", binary, exitCode, output) +} + +// RunColdFrontBinaryInput holds the parameters for a single tiering binary run. +type RunColdFrontBinaryInput struct { + DatabaseID string `json:"database_id"` + NodeName string `json:"node_name"` + InstanceID string `json:"instance_id"` + ServiceConfig map[string]any `json:"service_config"` + DatabaseName string `json:"database_name"` + Binary string `json:"binary"` +} + +type RunColdFrontBinaryOutput struct{} + +// ExecuteRunColdFrontBinary dispatches the RunColdFrontBinary activity to the +// given host's workflow queue. +func (a *Activities) ExecuteRunColdFrontBinary( + ctx workflow.Context, + hostID string, + input *RunColdFrontBinaryInput, +) workflow.Future[*RunColdFrontBinaryOutput] { + options := workflow.ActivityOptions{ + Queue: utils.HostQueue(hostID), + RetryOptions: workflow.RetryOptions{ + MaxAttempts: 1, + }, + } + return workflow.ExecuteActivity[*RunColdFrontBinaryOutput](ctx, options, a.RunColdFrontBinary, input) +} + +// RunColdFrontBinary executes a single-pass ColdFront tiering binary +// (archiver, partitioner, or compactor) inside the primary node's Postgres +// container via docker exec. The binary's config is written to a temporary +// file inside the container using base64 to avoid shell injection. Exit codes +// are captured: a "no tables configured" non-zero exit from the archiver is +// treated as benign (nothing to tier yet). +func (a *Activities) RunColdFrontBinary(ctx context.Context, input *RunColdFrontBinaryInput) (*RunColdFrontBinaryOutput, error) { + logger := activity.Logger(ctx).With( + "database_id", input.DatabaseID, + "instance_id", input.InstanceID, + "binary", input.Binary, + ) + logger.Info("running coldfront tiering binary") + + storageCfg, err := parseColdFrontStorageConfig(input.ServiceConfig) + if err != nil { + return nil, fmt.Errorf("coldfront %s: invalid storage config: %w", input.Binary, err) + } + if storageCfg == nil { + logger.Warn("no storage provider configured; skipping coldfront run") + return &RunColdFrontBinaryOutput{}, nil + } + + dockerClient, err := do.Invoke[*docker.Docker](a.Injector) + if err != nil { + return nil, fmt.Errorf("coldfront %s: failed to get docker client: %w", input.Binary, err) + } + + // The lakekeeper endpoint is supplied in the service config (baked into the + // scheduled-job args at reconciliation time as http://:). + lakekeeperEndpoint := "" + if ep, ok := input.ServiceConfig["lakekeeper_endpoint"].(string); ok && ep != "" { + lakekeeperEndpoint = ep + } + + configYAML, err := buildColdFrontConfigYAML(*storageCfg, input.DatabaseName, lakekeeperEndpoint) + if err != nil { + return nil, fmt.Errorf("coldfront %s: failed to render config: %w", input.Binary, err) + } + + // Locate the primary's Postgres container on this host via instance ID label. + pgContainers, err := dockerClient.ContainerList(ctx, container.ListOptions{ + Filters: filters.NewArgs( + filters.Arg("label", fmt.Sprintf("pgedge.instance.id=%s", input.InstanceID)), + filters.Arg("label", "pgedge.component=postgres"), + ), + }) + if err != nil { + return nil, fmt.Errorf("coldfront %s: failed to list containers for instance %s: %w", + input.Binary, input.InstanceID, err) + } + if len(pgContainers) == 0 { + return nil, fmt.Errorf("coldfront %s: no postgres container found for instance %s", + input.Binary, input.InstanceID) + } + pgContainer := pgContainers[0] + + // Write the config file into the container using base64 to avoid any shell + // quoting or injection issues, then run the binary. + encoded := base64.StdEncoding.EncodeToString(configYAML) + configPath := "/tmp/coldfront-config.yaml" + binaryPath := "/usr/local/bin/" + input.Binary + cmd := []string{ + "sh", "-c", + fmt.Sprintf("printf '%%s' '%s' | base64 -d > %s && %s --config %s", + encoded, configPath, binaryPath, configPath), + } + + if err := runColdFrontBinary(ctx, dockerTieringExecer{docker: dockerClient}, pgContainer.ID, input.Binary, cmd); err != nil { + return nil, err + } + + logger.Info("coldfront tiering binary completed successfully") + return &RunColdFrontBinaryOutput{}, nil +} diff --git a/server/internal/workflows/activities/coldfront_tiering_test.go b/server/internal/workflows/activities/coldfront_tiering_test.go new file mode 100644 index 00000000..e65940cf --- /dev/null +++ b/server/internal/workflows/activities/coldfront_tiering_test.go @@ -0,0 +1,190 @@ +package activities + +import ( + "context" + "errors" + "strings" + "testing" + + "github.com/pgEdge/control-plane/server/internal/task" +) + +// TestBuildColdFrontConfig verifies that the config YAML renderer produces +// the correct structure for each provider and never embeds credentials in +// plain text in a way that could slip into a log (smoke-test only — the real +// credential check is that the field exists with the expected value). +func TestBuildColdFrontConfig(t *testing.T) { + cases := []struct { + name string + cfg coldFrontStorageConfig + dbName string + endpoint string + wantKey string + }{ + { + name: "aws", + cfg: coldFrontStorageConfig{ + Provider: "aws", + Warehouse: "s3://my-bucket/warehouse", + Bucket: "my-bucket", + Region: "us-east-1", + Credential: map[string]string{ + "access_key_id": "AKID", + "secret_access_key": "SECRET", + }, + }, + dbName: "mydb", + wantKey: "access_key_id", + }, + { + name: "azure", + cfg: coldFrontStorageConfig{ + Provider: "azure", + Warehouse: "abfss://container@account.dfs.core.windows.net", + Bucket: "container", + Credential: map[string]string{ + "connection_string": "DefaultEndpointsProtocol=https;...", + }, + }, + dbName: "mydb", + wantKey: "connection_string", + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + yaml, err := buildColdFrontConfigYAML(tc.cfg, tc.dbName, "lakekeeper-svc:8181") + if err != nil { + t.Fatalf("buildColdFrontConfigYAML returned error: %v", err) + } + if len(yaml) == 0 { + t.Fatal("empty config YAML") + } + content := string(yaml) + if !strings.Contains(content, "postgres:") { + t.Error("missing postgres section") + } + if !strings.Contains(content, "iceberg:") { + t.Error("missing iceberg section") + } + if strings.Contains(content, "tables:") { + t.Error("config must NOT contain archiver.tables — tables come from DB registry") + } + }) + } +} + +// TestIsBenignArchiverEmpty verifies the benign-empty detection logic. The +// classification is scoped to the archiver only: the partitioner and compactor +// must never have the "no tables configured" text treated as benign. +func TestIsBenignArchiverEmpty(t *testing.T) { + cases := []struct { + binary string + output string + want bool + }{ + {"archiver", "no tables configured", true}, + {"archiver", "No Tables Configured", true}, + {"archiver", "NO TABLES CONFIGURED\n", true}, + {"archiver", "error: connection refused", false}, + {"archiver", "archiver completed successfully", false}, + {"archiver", "", false}, + // The same text must NOT be treated as benign for the other binaries. + {"partitioner", "no tables configured", false}, + {"compactor", "no tables configured", false}, + } + for _, tc := range cases { + if got := isBenignArchiverEmpty(tc.binary, tc.output); got != tc.want { + t.Errorf("isBenignArchiverEmpty(%q, %q) = %v, want %v", tc.binary, tc.output, got, tc.want) + } + } +} + +// fakeExecer drives runColdFrontBinary with a canned exit code / output so the +// exit-code capture and benign classification can be tested without Docker. +type fakeExecer struct { + exitCode int + output string + err error +} + +func (f fakeExecer) Exec(_ context.Context, _ string, _ []string) (int, string, error) { + return f.exitCode, f.output, f.err +} + +// mapResultToTaskStatus mirrors what the ColdFrontTiering workflow does with +// the activity result: a nil error is recorded as task success (completed), a +// non-nil error as task failure (failed). Exercising this mapping lets the test +// assert the resulting task status per exit-code scenario, which is the +// behaviour this task exists to add. +func mapResultToTaskStatus(err error) task.Status { + if err != nil { + return task.StatusFailed + } + return task.StatusCompleted +} + +// TestRunColdFrontBinaryExitCodeToTaskStatus is the mandated behavioural test: +// it drives the exec + exit-code + benign-classification path with a fake and +// asserts the resulting task status for each scenario. +func TestRunColdFrontBinaryExitCodeToTaskStatus(t *testing.T) { + cases := []struct { + name string + binary string + execer fakeExecer + wantErr bool + want task.Status + }{ + { + name: "exit zero records success", + binary: "archiver", + execer: fakeExecer{exitCode: 0}, + wantErr: false, + want: task.StatusCompleted, + }, + { + name: "non-zero exit records failure", + binary: "archiver", + execer: fakeExecer{exitCode: 1, output: "fatal: connection refused", err: errors.New("command failed with exit code 1")}, + wantErr: true, + want: task.StatusFailed, + }, + { + name: "archiver no-tables-configured is benign (success)", + binary: "archiver", + execer: fakeExecer{exitCode: 1, output: "FATAL: no tables configured", err: errors.New("command failed with exit code 1")}, + wantErr: false, + want: task.StatusCompleted, + }, + { + name: "partitioner no-tables-configured is NOT benign (failure)", + binary: "partitioner", + execer: fakeExecer{exitCode: 1, output: "no tables configured", err: errors.New("command failed with exit code 1")}, + wantErr: true, + want: task.StatusFailed, + }, + { + name: "compactor no-tables-configured is NOT benign (failure)", + binary: "compactor", + execer: fakeExecer{exitCode: 1, output: "no tables configured", err: errors.New("command failed with exit code 1")}, + wantErr: true, + want: task.StatusFailed, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + err := runColdFrontBinary(context.Background(), tc.execer, "container-id", tc.binary, []string{"sh", "-c", "true"}) + if (err != nil) != tc.wantErr { + t.Fatalf("runColdFrontBinary err = %v, wantErr = %v", err, tc.wantErr) + } + if got := mapResultToTaskStatus(err); got != tc.want { + t.Errorf("resulting task status = %q, want %q (err=%v)", got, tc.want, err) + } + // A genuine failure error must never leak the config/credential. + if err != nil && strings.Contains(err.Error(), "secret") { + t.Errorf("error output unexpectedly contains 'secret': %v", err) + } + }) + } +} diff --git a/server/internal/workflows/coldfront_tiering.go b/server/internal/workflows/coldfront_tiering.go new file mode 100644 index 00000000..43821355 --- /dev/null +++ b/server/internal/workflows/coldfront_tiering.go @@ -0,0 +1,127 @@ +package workflows + +import ( + "errors" + "fmt" + + "github.com/cschleiden/go-workflows/workflow" + "github.com/google/uuid" + + "github.com/pgEdge/control-plane/server/internal/task" + "github.com/pgEdge/control-plane/server/internal/workflows/activities" +) + +// ColdFrontTieringInput carries all arguments needed to run one tiering binary +// (archiver, partitioner, or compactor) against a database node. +type ColdFrontTieringInput struct { + DatabaseID string `json:"database_id"` + NodeName string `json:"node_name"` + ServiceID string `json:"service_id"` + ServiceConfig map[string]any `json:"service_config"` + DatabaseName string `json:"database_name"` + Binary string `json:"binary"` // "archiver" | "partitioner" | "compactor" + Instances []*InstanceHost `json:"instances"` + TaskID uuid.UUID `json:"task_id"` +} + +type ColdFrontTieringOutput struct{} + +// ColdFrontTiering resolves the current primary node for the database, renders +// the binary's config, and docker-execs the single-pass tiering binary inside +// the primary's Postgres container. Exit codes are captured: "no tables +// configured" archiver exits are recorded as benign (not a failure). +func (w *Workflows) ColdFrontTiering(ctx workflow.Context, input *ColdFrontTieringInput) (*ColdFrontTieringOutput, error) { + logger := workflow.Logger(ctx).With( + "database_id", input.DatabaseID, + "binary", input.Binary, + "task_id", input.TaskID.String(), + ) + + defer func() { + if errors.Is(ctx.Err(), workflow.Canceled) { + logger.Warn("workflow was canceled") + cleanupCtx := workflow.NewDisconnectedContext(ctx) + w.cancelTask(cleanupCtx, task.ScopeDatabase, input.DatabaseID, input.TaskID, logger) + } + }() + + logger.Info("starting coldfront tiering run") + + handleError := func(cause error) error { + logger.With("error", cause).Error("coldfront tiering run failed") + updateTaskInput := &activities.UpdateTaskInput{ + Scope: task.ScopeDatabase, + EntityID: input.DatabaseID, + TaskID: input.TaskID, + UpdateOptions: task.UpdateFail(cause), + } + _ = w.updateTask(ctx, logger, updateTaskInput) + return cause + } + + if len(input.Instances) == 0 { + return nil, handleError(fmt.Errorf("no instances available for database %s node %s", input.DatabaseID, input.NodeName)) + } + + // Use the first instance as the seed for primary resolution. + seed := input.Instances[0] + + updateOptions := task.UpdateStart() + if err := w.updateTask(ctx, logger, &activities.UpdateTaskInput{ + Scope: task.ScopeDatabase, + EntityID: input.DatabaseID, + TaskID: input.TaskID, + UpdateOptions: updateOptions, + }); err != nil { + return nil, handleError(err) + } + + // Resolve the current primary. Done at run time so that it follows Patroni + // failover automatically. + getPrimaryOutput, err := w.Activities. + ExecuteGetPrimaryInstance(ctx, seed.HostID, &activities.GetPrimaryInstanceInput{ + DatabaseID: input.DatabaseID, + InstanceID: seed.InstanceID, + }).Get(ctx) + if err != nil { + return nil, handleError(fmt.Errorf("failed to resolve primary instance: %w", err)) + } + + // Match primary instance ID back to a host. + var primaryInstance *InstanceHost + for _, inst := range input.Instances { + if inst.InstanceID == getPrimaryOutput.PrimaryInstanceID { + primaryInstance = inst + break + } + } + if primaryInstance == nil { + return nil, handleError(fmt.Errorf("primary instance %q not found in instance list", getPrimaryOutput.PrimaryInstanceID)) + } + + // Run the binary on the primary. + _, err = w.Activities. + ExecuteRunColdFrontBinary(ctx, primaryInstance.HostID, &activities.RunColdFrontBinaryInput{ + DatabaseID: input.DatabaseID, + NodeName: input.NodeName, + InstanceID: primaryInstance.InstanceID, + ServiceConfig: input.ServiceConfig, + DatabaseName: input.DatabaseName, + Binary: input.Binary, + }).Get(ctx) + if err != nil { + return nil, handleError(fmt.Errorf("coldfront %s failed: %w", input.Binary, err)) + } + + if err := w.updateTask(ctx, logger, &activities.UpdateTaskInput{ + Scope: task.ScopeDatabase, + EntityID: input.DatabaseID, + TaskID: input.TaskID, + UpdateOptions: task.UpdateComplete(), + }); err != nil { + return nil, handleError(err) + } + + logger.Info("coldfront tiering run completed successfully") + return &ColdFrontTieringOutput{}, nil +} diff --git a/server/internal/workflows/plan_update.go b/server/internal/workflows/plan_update.go index 162380ce..e48e4ffc 100644 --- a/server/internal/workflows/plan_update.go +++ b/server/internal/workflows/plan_update.go @@ -199,6 +199,10 @@ func resolveTargetSessionAttrs(serviceSpec *database.ServiceSpec) string { case "rag": // RAG is read-only; always prefer a standby when available. return database.TargetSessionAttrsPreferStandby + case "lakekeeper": + // Lakekeeper manages the Iceberg catalog state and writes to Postgres. + // Connect to the primary to avoid read-replica rejection of DML. + return database.TargetSessionAttrsPrimary default: // Default to "prefer-standby" for safety — read-only unless the // service explicitly opts in to writes. diff --git a/server/internal/workflows/service.go b/server/internal/workflows/service.go index c247b9ac..f3e88ea9 100644 --- a/server/internal/workflows/service.go +++ b/server/internal/workflows/service.go @@ -330,6 +330,26 @@ func (s *Service) StartInstance(ctx context.Context, input *StartInstanceInput) return t, nil } +// RunColdFrontTiering starts a ColdFrontTiering workflow that runs the named +// binary (archiver, partitioner, or compactor) against the given database node. +func (s *Service) RunColdFrontTiering(ctx context.Context, binary string, input *ColdFrontTieringInput) (*task.Task, error) { + t, err := s.taskSvc.CreateTask(ctx, task.Options{ + Scope: task.ScopeDatabase, + DatabaseID: input.DatabaseID, + Type: task.TypeTiering, + }) + if err != nil { + return nil, fmt.Errorf("failed to create tiering task: %w", err) + } + input.TaskID = t.TaskID + input.Binary = binary + err = s.createWorkflow(ctx, t, s.workflows.ColdFrontTiering, input) + if err != nil { + return nil, err + } + return t, nil +} + func (s *Service) CancelDatabaseTask(ctx context.Context, databaseID string, taskID uuid.UUID) (*task.Task, error) { t, err := s.taskSvc.GetTask(ctx, task.ScopeDatabase, databaseID, taskID) if err != nil { diff --git a/server/internal/workflows/workflows.go b/server/internal/workflows/workflows.go index 21f94199..cab217c3 100644 --- a/server/internal/workflows/workflows.go +++ b/server/internal/workflows/workflows.go @@ -35,6 +35,7 @@ func (w *Workflows) Register(work *worker.Worker) error { work.RegisterWorkflow(w.Switchover), work.RegisterWorkflow(w.UpdateDatabase), work.RegisterWorkflow(w.ValidateSpec), + work.RegisterWorkflow(w.ColdFrontTiering), } return errors.Join(errs...) }