From 1c6c72ce14ec0266f0956b20abf8deef6c888a08 Mon Sep 17 00:00:00 2001 From: Sean Rankine Date: Tue, 7 Jul 2026 20:08:30 +0100 Subject: [PATCH 1/7] Add per-environment assets bucket Rolling ECS deploys leave a window where old and new tasks serve traffic together. Requests for fingerprinted assets from a new release can be routed to an old task, which cannot serve them and returns 404. Add an S3 bucket to each environment to hold assets from every release, along with an origin access control so CloudFront can read from it. The bucket name is exported so the deploy pipelines can sync assets into it before deploying to ECS. --- .../deployments/forms/environment/outputs.tf | 4 ++ infra/modules/cloudfront/outputs.tf | 4 ++ infra/modules/cloudfront/s3.tf | 43 +++++++++++++++++++ infra/modules/environment/outputs.tf | 4 ++ 4 files changed, 55 insertions(+) diff --git a/infra/deployments/forms/environment/outputs.tf b/infra/deployments/forms/environment/outputs.tf index 3fc681044..7d93ff624 100644 --- a/infra/deployments/forms/environment/outputs.tf +++ b/infra/deployments/forms/environment/outputs.tf @@ -51,6 +51,10 @@ output "cloudfront_secret" { sensitive = true } +output "assets_bucket_name" { + value = module.environment.assets_bucket_name +} + output "eventbridge_dead_letter_queue_arn" { value = module.environment.eventbridge_dead_letter_queue_arn } diff --git a/infra/modules/cloudfront/outputs.tf b/infra/modules/cloudfront/outputs.tf index 7e86332fd..51a23f619 100644 --- a/infra/modules/cloudfront/outputs.tf +++ b/infra/modules/cloudfront/outputs.tf @@ -22,3 +22,7 @@ output "cloudfront_secret" { value = random_password.cloudfront_secret.result sensitive = true } + +output "assets_bucket_name" { + value = module.assets_bucket.name +} diff --git a/infra/modules/cloudfront/s3.tf b/infra/modules/cloudfront/s3.tf index 4944e9e47..e2c86a6ba 100644 --- a/infra/modules/cloudfront/s3.tf +++ b/infra/modules/cloudfront/s3.tf @@ -5,6 +5,49 @@ module "error_page_bucket" { name = "govuk-forms-${var.env_name}-error-page" } +locals { + assets_bucket_name = "govuk-forms-${var.env_name}-assets" +} + +module "assets_bucket" { + source = "../secure-bucket" + + name = local.assets_bucket_name + access_logging_enabled = true + send_access_logs_to_cyber = var.send_logs_to_cyber + + extra_bucket_policies = [data.aws_iam_policy_document.assets_bucket_cloudfront_read.json] +} + +data "aws_iam_policy_document" "assets_bucket_cloudfront_read" { + statement { + sid = "AllowCloudFrontToReadAssets" + principals { + type = "Service" + identifiers = ["cloudfront.amazonaws.com"] + } + actions = [ + "s3:GetObject", + ] + resources = [ + "arn:aws:s3:::${local.assets_bucket_name}/*" + ] + condition { + test = "StringEquals" + variable = "AWS:SourceArn" + values = [aws_cloudfront_distribution.main.arn] + } + } +} + +resource "aws_cloudfront_origin_access_control" "assets" { + name = "assets-${var.env_name}" + description = "Allow the CloudFront distribution to access the assets bucket" + origin_access_control_origin_type = "s3" + signing_behavior = "always" + signing_protocol = "sigv4" +} + locals { content_type_map = { "js" = "application/json" diff --git a/infra/modules/environment/outputs.tf b/infra/modules/environment/outputs.tf index 280410e44..3310df861 100644 --- a/infra/modules/environment/outputs.tf +++ b/infra/modules/environment/outputs.tf @@ -75,6 +75,10 @@ output "cloudfront_secret" { sensitive = true } +output "assets_bucket_name" { + value = module.cloudfront[0].assets_bucket_name +} + output "eventbridge_dead_letter_queue_arn" { value = aws_sqs_queue.event_bridge_dlq.arn } From 6f921633d6d08bd7ff0565d0837a85f6ed54ad06 Mon Sep 17 00:00:00 2001 From: Sean Rankine Date: Tue, 7 Jul 2026 20:09:23 +0100 Subject: [PATCH 2/7] Allow deployer to manage assets bucket and OAC The deployer role needs to create the per-environment assets bucket and CloudFront origin access control, and the deploy pipelines use the same role to sync assets into the bucket. --- infra/modules/deployer-access/policy.tf | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/infra/modules/deployer-access/policy.tf b/infra/modules/deployer-access/policy.tf index 95b40fc50..3068eeadf 100644 --- a/infra/modules/deployer-access/policy.tf +++ b/infra/modules/deployer-access/policy.tf @@ -187,6 +187,17 @@ data "aws_iam_policy_document" "cloudfront" { ] sid = "ManageCloudfrontDistribution" } + + statement { + actions = [ + "cloudfront:*OriginAccessControl*" + ] + effect = "Allow" + resources = [ + "arn:aws:cloudfront::${var.account_id}:origin-access-control/*" + ] + sid = "ManageCloudfrontOriginAccessControl" + } } data "aws_iam_policy_document" "cloudwatch" { @@ -928,7 +939,9 @@ data "aws_iam_policy_document" "s3" { "arn:aws:s3:::*-alb-access-logs-access-logs/*", "arn:aws:s3:::govuk-forms-alb-logs-${var.environment_name}*", - "arn:aws:s3:::govuk-forms-${var.environment_name}-error-page*" + "arn:aws:s3:::govuk-forms-${var.environment_name}-error-page*", + + "arn:aws:s3:::govuk-forms-${var.environment_name}-assets*" ] sid = "ManageS3" } From 1ab7dac85ea05768e432b1710c425abd4725c757 Mon Sep 17 00:00:00 2001 From: Sean Rankine Date: Tue, 7 Jul 2026 20:12:52 +0100 Subject: [PATCH 3/7] Sync assets to S3 during container deploys Copy the built assets out of the container image and upload them to the environment assets bucket before the new task definition is deployed to ECS. This makes the assets for a release available in S3 before any task serves pages referencing them. Assets from previous releases are left in the bucket so tasks which have not yet been replaced keep working during a deployment. --- .../buildspecs/sync-assets/sync-assets.yml | 30 +++++++++++++++++++ .../pipelines/deploy-forms-admin-container.tf | 21 +++++++++++++ .../deploy-forms-product-page-container.tf | 21 +++++++++++++ .../deploy-forms-runner-container.tf | 21 +++++++++++++ .../forms/pipelines/sync-assets.tf | 18 +++++++++++ 5 files changed, 111 insertions(+) create mode 100644 infra/deployments/forms/pipelines/buildspecs/sync-assets/sync-assets.yml create mode 100644 infra/deployments/forms/pipelines/sync-assets.tf diff --git a/infra/deployments/forms/pipelines/buildspecs/sync-assets/sync-assets.yml b/infra/deployments/forms/pipelines/buildspecs/sync-assets/sync-assets.yml new file mode 100644 index 000000000..3bdbd33db --- /dev/null +++ b/infra/deployments/forms/pipelines/buildspecs/sync-assets/sync-assets.yml @@ -0,0 +1,30 @@ +version: 0.2 +phases: + pre_build: + commands: + # Check Image URI is not the default pipeline value + - | + if [[ "${IMAGE_URI}" = "MUST_BE_SET" ]]; then + echo "The IMAGE_URI has not been set by the caller. The value of IMAGE_URI is \"${IMAGE_URI}\"" + exit 1 + fi + + - echo "Log in to Amazon ECR repository ${CONTAINER_REGISTRY}" + - aws ecr get-login-password --region eu-west-2 | docker login --username AWS --password-stdin "${CONTAINER_REGISTRY}" + build: + commands: + # Copy the built assets out of the container image + - docker pull "${IMAGE_URI}" + - CONTAINER_ID=$(docker create "${IMAGE_URI}") + - docker cp "${CONTAINER_ID}:/app/public/assets" ./assets + - docker rm "${CONTAINER_ID}" + + # Upload the assets so they are available before the new tasks + # serve any pages which reference them. Assets from previous + # releases are deliberately left in place so tasks which have not + # yet been replaced keep working during the deployment. + - | + aws s3 sync ./assets "s3://${ASSETS_BUCKET}/assets/" \ + --exclude ".vite/*" \ + --cache-control "public, max-age=300" \ + --no-progress diff --git a/infra/deployments/forms/pipelines/deploy-forms-admin-container.tf b/infra/deployments/forms/pipelines/deploy-forms-admin-container.tf index 468ce695e..d266f6f68 100644 --- a/infra/deployments/forms/pipelines/deploy-forms-admin-container.tf +++ b/infra/deployments/forms/pipelines/deploy-forms-admin-container.tf @@ -189,6 +189,27 @@ resource "aws_codepipeline" "deploy_admin_container" { } } + action { + name = "sync-assets-to-s3" + category = "Build" + owner = "AWS" + provider = "CodeBuild" + version = "1" + run_order = 1 + input_artifacts = ["buildspec_source"] + # we need an input according to AWS, even if we don't... so we'll use this one for now. + configuration = { + ProjectName = module.sync_assets.name + EnvironmentVariables = jsonencode([ + { + name = "IMAGE_URI" + value = "#{variables.container_image_uri}" + type = "PLAINTEXT" + } + ]) + } + } + action { name = "generate-image-definitions" namespace = "Build" diff --git a/infra/deployments/forms/pipelines/deploy-forms-product-page-container.tf b/infra/deployments/forms/pipelines/deploy-forms-product-page-container.tf index 12d881630..9a56ec318 100644 --- a/infra/deployments/forms/pipelines/deploy-forms-product-page-container.tf +++ b/infra/deployments/forms/pipelines/deploy-forms-product-page-container.tf @@ -146,6 +146,27 @@ resource "aws_codepipeline" "deploy_product_pages_container" { stage { name = "deploy-to-ecs" + action { + name = "sync-assets-to-s3" + category = "Build" + owner = "AWS" + provider = "CodeBuild" + version = "1" + run_order = 1 + input_artifacts = ["buildspec_source"] + # we need an input according to AWS, even if we don't... so we'll use this one for now. + configuration = { + ProjectName = module.sync_assets.name + EnvironmentVariables = jsonencode([ + { + name = "IMAGE_URI" + value = "#{variables.container_image_uri}" + type = "PLAINTEXT" + } + ]) + } + } + action { name = "generate-image-definitions" namespace = "Build" diff --git a/infra/deployments/forms/pipelines/deploy-forms-runner-container.tf b/infra/deployments/forms/pipelines/deploy-forms-runner-container.tf index 1b8fa6689..d80bf6c60 100644 --- a/infra/deployments/forms/pipelines/deploy-forms-runner-container.tf +++ b/infra/deployments/forms/pipelines/deploy-forms-runner-container.tf @@ -182,6 +182,27 @@ resource "aws_codepipeline" "deploy_runner_container" { } } + action { + name = "sync-assets-to-s3" + category = "Build" + owner = "AWS" + provider = "CodeBuild" + version = "1" + run_order = 1 + input_artifacts = ["buildspec_source"] + # we need an input according to AWS, even if we don't... so we'll use this one for now. + configuration = { + ProjectName = module.sync_assets.name + EnvironmentVariables = jsonencode([ + { + name = "IMAGE_URI" + value = "#{variables.container_image_uri}" + type = "PLAINTEXT" + } + ]) + } + } + action { name = "generate-image-definitions" namespace = "Build" diff --git a/infra/deployments/forms/pipelines/sync-assets.tf b/infra/deployments/forms/pipelines/sync-assets.tf new file mode 100644 index 000000000..b127d82f4 --- /dev/null +++ b/infra/deployments/forms/pipelines/sync-assets.tf @@ -0,0 +1,18 @@ +# A single project is shared by the app deploy pipelines. All of the +# apps store their assets in the same bucket: every file is fingerprinted +# with a content hash, so names cannot collide. +module "sync_assets" { + source = "../../../modules/code-build-build" + project_name = "sync_assets_${var.environment_name}" + project_description = "Sync assets from a container image to the assets bucket" + environment = var.environment_name + artifact_store_arn = module.artifact_bucket.arn + buildspec = file("${path.root}/buildspecs/sync-assets/sync-assets.yml") + log_group_name = "codebuild/sync_assets_${var.environment_name}" + codebuild_service_role_arn = data.aws_iam_role.deployer_role.arn + + environment_variables = { + ASSETS_BUCKET = data.terraform_remote_state.forms_environment.outputs.assets_bucket_name + CONTAINER_REGISTRY = var.container_registry + } +} From f64c4f81cb87a921d81d416cd61d265bd747da27 Mon Sep 17 00:00:00 2001 From: Sean Rankine Date: Tue, 7 Jul 2026 20:18:02 +0100 Subject: [PATCH 4/7] Add option to serve assets from S3 When enabled, CloudFront serves /assets/* from the assets bucket instead of the applications. The bucket holds assets from every release, so requests can no longer fail when a deployment leaves tasks from two releases serving traffic at once. The option is off by default and should only be enabled for an environment once its deploy pipelines have populated the bucket for all three applications. Serving from S3 cannot fall back to the ALB via an origin group: CloudFront must not forward the viewer Host header to an S3 origin, and without it the ALB listener rules cannot route the request. --- infra/deployments/forms/environment/main.tf | 2 ++ infra/deployments/forms/inputs.tf | 2 +- infra/modules/cloudfront/cloudfront.tf | 18 ++++++++++++++++-- infra/modules/cloudfront/variables.tf | 6 ++++++ infra/modules/environment/cloudfront.tf | 1 + infra/modules/environment/variables.tf | 6 ++++++ infra/modules/secure-bucket/outputs.tf | 4 ++++ 7 files changed, 36 insertions(+), 3 deletions(-) diff --git a/infra/deployments/forms/environment/main.tf b/infra/deployments/forms/environment/main.tf index bcb755f7f..ec311c03c 100644 --- a/infra/deployments/forms/environment/main.tf +++ b/infra/deployments/forms/environment/main.tf @@ -18,4 +18,6 @@ module "environment" { enable_shield_advanced_healthchecks = var.environmental_settings.enable_shield_advanced_healthchecks scheduled_smoke_tests_settings = var.scheduled_smoke_tests_settings + + serve_assets_from_s3 = var.environmental_settings.serve_assets_from_s3 } diff --git a/infra/deployments/forms/inputs.tf b/infra/deployments/forms/inputs.tf index 253917a76..c09c432ce 100644 --- a/infra/deployments/forms/inputs.tf +++ b/infra/deployments/forms/inputs.tf @@ -206,7 +206,7 @@ variable "environmental_settings" { redis_multi_az_enabled = bool enable_advanced_database_insights = bool rds_enhanced_monitoring_interval_seconds = number # Enables RDS enhanced monitoring if value is 1, 5, 10, 15, 30 or 60. Disabled if 0 - + serve_assets_from_s3 = optional(bool, false) }) validation { condition = contains([0, 1, 5, 10, 15, 30, 60], var.environmental_settings.rds_enhanced_monitoring_interval_seconds) diff --git a/infra/modules/cloudfront/cloudfront.tf b/infra/modules/cloudfront/cloudfront.tf index 611425b74..23d7b25d1 100644 --- a/infra/modules/cloudfront/cloudfront.tf +++ b/infra/modules/cloudfront/cloudfront.tf @@ -89,6 +89,12 @@ resource "aws_cloudfront_distribution" "main" { } } + origin { + domain_name = module.assets_bucket.regional_domain_name + origin_id = "assets_s3" + origin_access_control_id = aws_cloudfront_origin_access_control.assets.id + } + default_cache_behavior { viewer_protocol_policy = "redirect-to-https" allowed_methods = ["GET", "HEAD", "OPTIONS", "PUT", "POST", "PATCH", "DELETE"] @@ -143,15 +149,23 @@ resource "aws_cloudfront_distribution" "main" { origin_request_policy_id = data.aws_cloudfront_origin_request_policy.cors_s3_origin.id } + # The applications can serve their own assets, but doing so causes + # errors during deployments: old tasks cannot serve assets for a new + # release. The assets bucket holds the assets for every release, so + # prefer it once the deploy pipelines have started populating it. + # + # The all_viewer origin request policy must not be used with an S3 + # origin because forwarding the viewer Host header breaks the origin + # access control request signing. ordered_cache_behavior { path_pattern = "/assets/*" allowed_methods = ["GET", "HEAD", "OPTIONS"] cached_methods = ["GET", "HEAD"] - target_origin_id = "application_load_balancer" + target_origin_id = var.serve_assets_from_s3 ? "assets_s3" : "application_load_balancer" viewer_protocol_policy = "redirect-to-https" compress = true cache_policy_id = data.aws_cloudfront_cache_policy.caching_optimized.id - origin_request_policy_id = data.aws_cloudfront_origin_request_policy.all_viewer.id + origin_request_policy_id = var.serve_assets_from_s3 ? data.aws_cloudfront_origin_request_policy.cors_s3_origin.id : data.aws_cloudfront_origin_request_policy.all_viewer.id } } diff --git a/infra/modules/cloudfront/variables.tf b/infra/modules/cloudfront/variables.tf index 33bdeb70b..184ef0fed 100644 --- a/infra/modules/cloudfront/variables.tf +++ b/infra/modules/cloudfront/variables.tf @@ -50,3 +50,9 @@ variable "kinesis_subscription_role_arn" { description = "The arn of the role that is allowed to subscribe to the kinesis stream" type = string } + +variable "serve_assets_from_s3" { + description = "Whether to serve requests for /assets/* from the assets bucket rather than the applications. The deploy pipelines must have synced assets to the bucket before this is enabled." + type = bool + default = false +} diff --git a/infra/modules/environment/cloudfront.tf b/infra/modules/environment/cloudfront.tf index c69135987..02c8832e9 100644 --- a/infra/modules/environment/cloudfront.tf +++ b/infra/modules/environment/cloudfront.tf @@ -22,6 +22,7 @@ module "cloudfront" { subject_alternative_names = local.subject_alternative_names[var.env_name] kinesis_subscription_role_arn = var.kinesis_subscription_role_arn + serve_assets_from_s3 = var.serve_assets_from_s3 } resource "aws_ssm_parameter" "email_zendesk" { diff --git a/infra/modules/environment/variables.tf b/infra/modules/environment/variables.tf index dd92405e0..3744afbf7 100644 --- a/infra/modules/environment/variables.tf +++ b/infra/modules/environment/variables.tf @@ -32,6 +32,12 @@ variable "enable_cloudfront" { default = true } +variable "serve_assets_from_s3" { + type = bool + description = "Whether to serve requests for /assets/* from the assets bucket rather than the applications. The deploy pipelines must have synced assets to the bucket before this is enabled." + default = false +} + variable "enable_alert_actions" { type = bool description = "Whether any actions associated with CloudWatch alarms should be enabled" diff --git a/infra/modules/secure-bucket/outputs.tf b/infra/modules/secure-bucket/outputs.tf index 5099e3958..11633177d 100644 --- a/infra/modules/secure-bucket/outputs.tf +++ b/infra/modules/secure-bucket/outputs.tf @@ -5,3 +5,7 @@ output "name" { output "arn" { value = aws_s3_bucket.this.arn } + +output "regional_domain_name" { + value = aws_s3_bucket.this.bucket_regional_domain_name +} From 72e2cff7679ada66c5bd56da185459ee0423a31c Mon Sep 17 00:00:00 2001 From: Sean Rankine Date: Tue, 4 Aug 2026 14:29:59 +0100 Subject: [PATCH 5/7] Tolerate missing assets_bucket_name output on first apply The self-update-pipelines stage applies forms/pipelines before the environment stack has published the assets_bucket_name output, so the first apply of this branch would fail with an unsupported attribute error. Fall back to the derived bucket name until the output exists. --- infra/deployments/forms/pipelines/sync-assets.tf | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/infra/deployments/forms/pipelines/sync-assets.tf b/infra/deployments/forms/pipelines/sync-assets.tf index b127d82f4..72edc8a62 100644 --- a/infra/deployments/forms/pipelines/sync-assets.tf +++ b/infra/deployments/forms/pipelines/sync-assets.tf @@ -12,7 +12,9 @@ module "sync_assets" { codebuild_service_role_arn = data.aws_iam_role.deployer_role.arn environment_variables = { - ASSETS_BUCKET = data.terraform_remote_state.forms_environment.outputs.assets_bucket_name + # try() so that the first apply succeeds before the environment state + # has published the assets_bucket_name output + ASSETS_BUCKET = try(data.terraform_remote_state.forms_environment.outputs.assets_bucket_name, "govuk-forms-${var.environment_name}-assets") CONTAINER_REGISTRY = var.container_registry } } From f176dce2c37709fd19b45cdad039d330f109e3cf Mon Sep 17 00:00:00 2001 From: Sean Rankine Date: Tue, 4 Aug 2026 14:29:59 +0100 Subject: [PATCH 6/7] Grant OAC create and list actions on all resources CreateOriginAccessControl and ListOriginAccessControls do not support resource-level permissions, so scoping them to an origin-access-control ARN causes AccessDenied when the deployer creates the OAC. Move them to a statement with a wildcard resource; the other OAC actions stay ARN-scoped. --- infra/modules/deployer-access/policy.tf | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/infra/modules/deployer-access/policy.tf b/infra/modules/deployer-access/policy.tf index 3068eeadf..ec2e53282 100644 --- a/infra/modules/deployer-access/policy.tf +++ b/infra/modules/deployer-access/policy.tf @@ -190,7 +190,10 @@ data "aws_iam_policy_document" "cloudfront" { statement { actions = [ - "cloudfront:*OriginAccessControl*" + "cloudfront:DeleteOriginAccessControl", + "cloudfront:GetOriginAccessControl", + "cloudfront:GetOriginAccessControlConfig", + "cloudfront:UpdateOriginAccessControl" ] effect = "Allow" resources = [ @@ -198,6 +201,19 @@ data "aws_iam_policy_document" "cloudfront" { ] sid = "ManageCloudfrontOriginAccessControl" } + + # These actions do not support resource-level permissions + statement { + actions = [ + "cloudfront:CreateOriginAccessControl", + "cloudfront:ListOriginAccessControls" + ] + effect = "Allow" + resources = [ + "*" + ] + sid = "CreateCloudfrontOriginAccessControl" + } } data "aws_iam_policy_document" "cloudwatch" { From d9d077a6e805ca702e59990642ea839aa89f95c9 Mon Sep 17 00:00:00 2001 From: Sean Rankine Date: Tue, 11 Aug 2026 15:05:20 +0100 Subject: [PATCH 7/7] Pin generate-image-definitions to run order 1 --- .../deployments/forms/pipelines/deploy-forms-admin-container.tf | 1 + .../forms/pipelines/deploy-forms-product-page-container.tf | 1 + 2 files changed, 2 insertions(+) diff --git a/infra/deployments/forms/pipelines/deploy-forms-admin-container.tf b/infra/deployments/forms/pipelines/deploy-forms-admin-container.tf index d266f6f68..e16b3d002 100644 --- a/infra/deployments/forms/pipelines/deploy-forms-admin-container.tf +++ b/infra/deployments/forms/pipelines/deploy-forms-admin-container.tf @@ -217,6 +217,7 @@ resource "aws_codepipeline" "deploy_admin_container" { owner = "AWS" provider = "CodeBuild" version = "1" + run_order = 1 input_artifacts = ["buildspec_source"] output_artifacts = ["image-defs-json"] configuration = { diff --git a/infra/deployments/forms/pipelines/deploy-forms-product-page-container.tf b/infra/deployments/forms/pipelines/deploy-forms-product-page-container.tf index 9a56ec318..b5507f53a 100644 --- a/infra/deployments/forms/pipelines/deploy-forms-product-page-container.tf +++ b/infra/deployments/forms/pipelines/deploy-forms-product-page-container.tf @@ -174,6 +174,7 @@ resource "aws_codepipeline" "deploy_product_pages_container" { owner = "AWS" provider = "CodeBuild" version = "1" + run_order = 1 input_artifacts = ["buildspec_source"] output_artifacts = ["image-defs-json"] configuration = {