Serve app assets from S3

infra/deployments/forms/environment/main.tf

@@ -18,4 +18,6 @@ module "environment" {
 
   enable_shield_advanced_healthchecks = var.environmental_settings.enable_shield_advanced_healthchecks
   scheduled_smoke_tests_settings      = var.scheduled_smoke_tests_settings
+
+  serve_assets_from_s3 = var.environmental_settings.serve_assets_from_s3
 }

infra/deployments/forms/environment/outputs.tf

@@ -51,6 +51,10 @@ output "cloudfront_secret" {
   sensitive = true
 }
 
+output "assets_bucket_name" {
+  value = module.environment.assets_bucket_name
+}
+
 output "eventbridge_dead_letter_queue_arn" {
   value = module.environment.eventbridge_dead_letter_queue_arn
 }

infra/deployments/forms/inputs.tf

@@ -206,7 +206,7 @@ variable "environmental_settings" {
     redis_multi_az_enabled                   = bool
     enable_advanced_database_insights        = bool
     rds_enhanced_monitoring_interval_seconds = number # Enables RDS enhanced monitoring if value is 1, 5, 10, 15, 30 or 60. Disabled if 0
-
+    serve_assets_from_s3                     = optional(bool, false)
   })
   validation {
     condition     = contains([0, 1, 5, 10, 15, 30, 60], var.environmental_settings.rds_enhanced_monitoring_interval_seconds)

infra/deployments/forms/pipelines/buildspecs/sync-assets/sync-assets.yml

@@ -0,0 +1,30 @@
+version: 0.2
+phases:
+  pre_build:
+    commands:
+      # Check Image URI is not the default pipeline value
+      - |
+        if [[ "${IMAGE_URI}" = "MUST_BE_SET" ]]; then
+          echo "The IMAGE_URI has not been set by the caller. The value of IMAGE_URI is \"${IMAGE_URI}\""
+          exit 1
+        fi
+
+      - echo "Log in to Amazon ECR repository ${CONTAINER_REGISTRY}"
+      - aws ecr get-login-password --region eu-west-2 | docker login --username AWS --password-stdin "${CONTAINER_REGISTRY}"
+  build:
+    commands:
+      # Copy the built assets out of the container image
+      - docker pull "${IMAGE_URI}"
+      - CONTAINER_ID=$(docker create "${IMAGE_URI}")
+      - docker cp "${CONTAINER_ID}:/app/public/assets" ./assets
+      - docker rm "${CONTAINER_ID}"
+
+      # Upload the assets so they are available before the new tasks
+      # serve any pages which reference them. Assets from previous
+      # releases are deliberately left in place so tasks which have not
+      # yet been replaced keep working during the deployment.
+      - |
+        aws s3 sync ./assets "s3://${ASSETS_BUCKET}/assets/" \
+          --exclude ".vite/*" \
+          --cache-control "public, max-age=300" \
+          --no-progress

infra/deployments/forms/pipelines/deploy-forms-admin-container.tf

@@ -189,13 +189,35 @@ resource "aws_codepipeline" "deploy_admin_container" {
       }
     }
 
+    action {
+      name            = "sync-assets-to-s3"
+      category        = "Build"
+      owner           = "AWS"
+      provider        = "CodeBuild"
+      version         = "1"
+      run_order       = 1
+      input_artifacts = ["buildspec_source"]
+      # we need an input according to AWS, even if we don't... so we'll use this one for now.
+      configuration = {
+        ProjectName = module.sync_assets.name
+        EnvironmentVariables = jsonencode([
+          {
+            name  = "IMAGE_URI"
+            value = "#{variables.container_image_uri}"
+            type  = "PLAINTEXT"
+          }
+        ])
+      }
+    }
+
     action {
       name             = "generate-image-definitions"
       namespace        = "Build"
       category         = "Build"
       owner            = "AWS"
       provider         = "CodeBuild"
       version          = "1"
+      run_order        = 1
       input_artifacts  = ["buildspec_source"]
       output_artifacts = ["image-defs-json"]
       configuration = {

infra/deployments/forms/pipelines/deploy-forms-product-page-container.tf

@@ -146,13 +146,35 @@ resource "aws_codepipeline" "deploy_product_pages_container" {
   stage {
     name = "deploy-to-ecs"
 
+    action {
+      name            = "sync-assets-to-s3"
+      category        = "Build"
+      owner           = "AWS"
+      provider        = "CodeBuild"
+      version         = "1"
+      run_order       = 1
+      input_artifacts = ["buildspec_source"]
+      # we need an input according to AWS, even if we don't... so we'll use this one for now.
+      configuration = {
+        ProjectName = module.sync_assets.name
+        EnvironmentVariables = jsonencode([
+          {
+            name  = "IMAGE_URI"
+            value = "#{variables.container_image_uri}"
+            type  = "PLAINTEXT"
+          }
+        ])
+      }
+    }
+
     action {
       name             = "generate-image-definitions"
       namespace        = "Build"
       category         = "Build"
       owner            = "AWS"
       provider         = "CodeBuild"
       version          = "1"
+      run_order        = 1
       input_artifacts  = ["buildspec_source"]
       output_artifacts = ["image-defs-json"]
       configuration = {

infra/deployments/forms/pipelines/deploy-forms-runner-container.tf

@@ -182,6 +182,27 @@ resource "aws_codepipeline" "deploy_runner_container" {
       }
     }
 
+    action {
+      name            = "sync-assets-to-s3"
+      category        = "Build"
+      owner           = "AWS"
+      provider        = "CodeBuild"
+      version         = "1"
+      run_order       = 1
+      input_artifacts = ["buildspec_source"]
+      # we need an input according to AWS, even if we don't... so we'll use this one for now.
+      configuration = {
+        ProjectName = module.sync_assets.name
+        EnvironmentVariables = jsonencode([
+          {
+            name  = "IMAGE_URI"
+            value = "#{variables.container_image_uri}"
+            type  = "PLAINTEXT"
+          }
+        ])
+      }
+    }
+
     action {
       name             = "generate-image-definitions"
       namespace        = "Build"

infra/deployments/forms/pipelines/sync-assets.tf

@@ -0,0 +1,20 @@
+# A single project is shared by the app deploy pipelines. All of the
+# apps store their assets in the same bucket: every file is fingerprinted
+# with a content hash, so names cannot collide.
+module "sync_assets" {
+  source                     = "../../../modules/code-build-build"
+  project_name               = "sync_assets_${var.environment_name}"
+  project_description        = "Sync assets from a container image to the assets bucket"
+  environment                = var.environment_name
+  artifact_store_arn         = module.artifact_bucket.arn
+  buildspec                  = file("${path.root}/buildspecs/sync-assets/sync-assets.yml")
+  log_group_name             = "codebuild/sync_assets_${var.environment_name}"
+  codebuild_service_role_arn = data.aws_iam_role.deployer_role.arn
+
+  environment_variables = {
+    # try() so that the first apply succeeds before the environment state
+    # has published the assets_bucket_name output
+    ASSETS_BUCKET      = try(data.terraform_remote_state.forms_environment.outputs.assets_bucket_name, "govuk-forms-${var.environment_name}-assets")
+    CONTAINER_REGISTRY = var.container_registry
+  }
+}

infra/modules/cloudfront/cloudfront.tf

@@ -89,6 +89,12 @@ resource "aws_cloudfront_distribution" "main" {
     }
   }
 
+  origin {
+    domain_name              = module.assets_bucket.regional_domain_name
+    origin_id                = "assets_s3"
+    origin_access_control_id = aws_cloudfront_origin_access_control.assets.id
+  }
+
   default_cache_behavior {
     viewer_protocol_policy = "redirect-to-https"
     allowed_methods        = ["GET", "HEAD", "OPTIONS", "PUT", "POST", "PATCH", "DELETE"]
@@ -143,15 +149,23 @@ resource "aws_cloudfront_distribution" "main" {
     origin_request_policy_id = data.aws_cloudfront_origin_request_policy.cors_s3_origin.id
   }
 
+  # The applications can serve their own assets, but doing so causes
+  # errors during deployments: old tasks cannot serve assets for a new
+  # release. The assets bucket holds the assets for every release, so
+  # prefer it once the deploy pipelines have started populating it.
+  #
+  # The all_viewer origin request policy must not be used with an S3
+  # origin because forwarding the viewer Host header breaks the origin
+  # access control request signing.
   ordered_cache_behavior {
     path_pattern           = "/assets/*"
     allowed_methods        = ["GET", "HEAD", "OPTIONS"]
     cached_methods         = ["GET", "HEAD"]
-    target_origin_id       = "application_load_balancer"
+    target_origin_id       = var.serve_assets_from_s3 ? "assets_s3" : "application_load_balancer"
     viewer_protocol_policy = "redirect-to-https"
     compress               = true
 
     cache_policy_id          = data.aws_cloudfront_cache_policy.caching_optimized.id
-    origin_request_policy_id = data.aws_cloudfront_origin_request_policy.all_viewer.id
+    origin_request_policy_id = var.serve_assets_from_s3 ? data.aws_cloudfront_origin_request_policy.cors_s3_origin.id : data.aws_cloudfront_origin_request_policy.all_viewer.id
   }
 }

infra/modules/cloudfront/outputs.tf

@@ -22,3 +22,7 @@ output "cloudfront_secret" {
   value     = random_password.cloudfront_secret.result
   sensitive = true
 }
+
+output "assets_bucket_name" {
+  value = module.assets_bucket.name
+}

infra/modules/cloudfront/s3.tf

@@ -5,6 +5,49 @@ module "error_page_bucket" {
   name                   = "govuk-forms-${var.env_name}-error-page"
 }
 
+locals {
+  assets_bucket_name = "govuk-forms-${var.env_name}-assets"
+}
+
+module "assets_bucket" {
+  source = "../secure-bucket"
+
+  name                      = local.assets_bucket_name
+  access_logging_enabled    = true
+  send_access_logs_to_cyber = var.send_logs_to_cyber
+
+  extra_bucket_policies = [data.aws_iam_policy_document.assets_bucket_cloudfront_read.json]
+}
+
+data "aws_iam_policy_document" "assets_bucket_cloudfront_read" {
+  statement {
+    sid = "AllowCloudFrontToReadAssets"
+    principals {
+      type        = "Service"
+      identifiers = ["cloudfront.amazonaws.com"]
+    }
+    actions = [
+      "s3:GetObject",
+    ]
+    resources = [
+      "arn:aws:s3:::${local.assets_bucket_name}/*"
+    ]
+    condition {
+      test     = "StringEquals"
+      variable = "AWS:SourceArn"
+      values   = [aws_cloudfront_distribution.main.arn]
+    }
+  }
+}
+
+resource "aws_cloudfront_origin_access_control" "assets" {
+  name                              = "assets-${var.env_name}"
+  description                       = "Allow the CloudFront distribution to access the assets bucket"
+  origin_access_control_origin_type = "s3"
+  signing_behavior                  = "always"
+  signing_protocol                  = "sigv4"
+}
+
 locals {
   content_type_map = {
     "js"    = "application/json"

infra/modules/cloudfront/variables.tf

@@ -50,3 +50,9 @@ variable "kinesis_subscription_role_arn" {
   description = "The arn of the role that is allowed to subscribe to the kinesis stream"
   type        = string
 }
+
+variable "serve_assets_from_s3" {
+  description = "Whether to serve requests for /assets/* from the assets bucket rather than the applications. The deploy pipelines must have synced assets to the bucket before this is enabled."
+  type        = bool
+  default     = false
+}

infra/modules/deployer-access/policy.tf

@@ -187,6 +187,33 @@ data "aws_iam_policy_document" "cloudfront" {
     ]
     sid = "ManageCloudfrontDistribution"
   }
+
+  statement {
+    actions = [
+      "cloudfront:DeleteOriginAccessControl",
+      "cloudfront:GetOriginAccessControl",
+      "cloudfront:GetOriginAccessControlConfig",
+      "cloudfront:UpdateOriginAccessControl"
+    ]
+    effect = "Allow"
+    resources = [
+      "arn:aws:cloudfront::${var.account_id}:origin-access-control/*"
+    ]
+    sid = "ManageCloudfrontOriginAccessControl"
+  }
+
+  # These actions do not support resource-level permissions
+  statement {
+    actions = [
+      "cloudfront:CreateOriginAccessControl",
+      "cloudfront:ListOriginAccessControls"
+    ]
+    effect = "Allow"
+    resources = [
+      "*"
+    ]
+    sid = "CreateCloudfrontOriginAccessControl"
+  }
 }
 
 data "aws_iam_policy_document" "cloudwatch" {
@@ -928,7 +955,9 @@ data "aws_iam_policy_document" "s3" {
       "arn:aws:s3:::*-alb-access-logs-access-logs/*",
       "arn:aws:s3:::govuk-forms-alb-logs-${var.environment_name}*",
 
-      "arn:aws:s3:::govuk-forms-${var.environment_name}-error-page*"
+      "arn:aws:s3:::govuk-forms-${var.environment_name}-error-page*",
+
+      "arn:aws:s3:::govuk-forms-${var.environment_name}-assets*"
     ]
     sid = "ManageS3"
   }

infra/modules/environment/cloudfront.tf

@@ -22,6 +22,7 @@ module "cloudfront" {
   subject_alternative_names     = local.subject_alternative_names[var.env_name]
   kinesis_subscription_role_arn = var.kinesis_subscription_role_arn
 
+  serve_assets_from_s3 = var.serve_assets_from_s3
 }
 
 resource "aws_ssm_parameter" "email_zendesk" {

infra/modules/environment/outputs.tf

@@ -75,6 +75,10 @@ output "cloudfront_secret" {
   sensitive = true
 }
 
+output "assets_bucket_name" {
+  value = module.cloudfront[0].assets_bucket_name
+}
+
 output "eventbridge_dead_letter_queue_arn" {
   value = aws_sqs_queue.event_bridge_dlq.arn
 }

infra/modules/environment/variables.tf

@@ -32,6 +32,12 @@ variable "enable_cloudfront" {
   default     = true
 }
 
+variable "serve_assets_from_s3" {
+  type        = bool
+  description = "Whether to serve requests for /assets/* from the assets bucket rather than the applications. The deploy pipelines must have synced assets to the bucket before this is enabled."
+  default     = false
+}
+
 variable "enable_alert_actions" {
   type        = bool
   description = "Whether any actions associated with CloudWatch alarms should be enabled"

infra/modules/secure-bucket/outputs.tf

@@ -5,3 +5,7 @@ output "name" {
 output "arn" {
   value = aws_s3_bucket.this.arn
 }
+
+output "regional_domain_name" {
+  value = aws_s3_bucket.this.bucket_regional_domain_name
+}