diff --git a/.github/actions/acceptance-tests/action.yaml b/.github/actions/acceptance-tests/action.yaml index 7fe3c28a..6f6a44ba 100644 --- a/.github/actions/acceptance-tests/action.yaml +++ b/.github/actions/acceptance-tests/action.yaml @@ -49,6 +49,7 @@ runs: shell: bash env: PROJECT: nhs - COMPONENT: ${{ inputs.targetComponent }} + COMPONENT: cb + CLIENT_COMPONENT: cbc run: | make test-${{ inputs.testType }} diff --git a/.github/workflows/stage-2-test.yaml b/.github/workflows/stage-2-test.yaml index 77392e23..91d3ee45 100644 --- a/.github/workflows/stage-2-test.yaml +++ b/.github/workflows/stage-2-test.yaml @@ -125,6 +125,16 @@ jobs: - name: "Run linting" run: | make test-lint + test-lua-lint: + name: "Lua linting" + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - name: "Checkout code" + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - name: "Run luacheck" + run: | + make test-lua-lint test-typecheck: name: "Typecheck" runs-on: ubuntu-latest diff --git a/.gitignore b/.gitignore index e97bd341..d1b1e760 100644 --- a/.gitignore +++ b/.gitignore @@ -12,7 +12,7 @@ version.json # Please, add your custom content below! -# dependencies +# Dependencies node_modules .node-version */node_modules @@ -22,3 +22,4 @@ node_modules dist .DS_Store .reports +*~ diff --git a/.luarc.json b/.luarc.json new file mode 100644 index 00000000..09ef6a0c --- /dev/null +++ b/.luarc.json @@ -0,0 +1,12 @@ +{ + "diagnostics": { + "globals": [ + "KEYS", + "ARGV", + "redis", + "cjson", + "cmsgpack", + "bit" + ] + } +} diff --git a/AGENTS.md b/AGENTS.md index 982ca631..0ef373b1 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -23,25 +23,25 @@ Agents should look for a nested `AGENTS.md` in or near these areas before making ## Root package.json – role and usage -The root `package.json` is the orchestration manifestgit co for this repo. It does not ship application code; it wires up shared dev tooling and delegates to workspace-level projects. +The root `package.json` is the orchestration manifest for this repo. It does not ship application code; it wires up shared dev tooling and delegates to workspace-level projects. -- Workspaces: Declares the set of npm workspaces (e.g. under `lambdas/`, `utils/`, `tests/`, `scripts/`). Agents should add a new workspace path here when introducing a new npm project. -- Scripts: Provides top-level commands that fan out across workspaces using `--workspaces` (lint, typecheck, unit tests) and project-specific runners (e.g. `lambda-build`). +- Workspaces: Declares the set of pnpm workspaces (e.g. under `lambdas/`, `utils/`, `tests/`, `scripts/`). Agents should add a new workspace path here when introducing a new pnpm project. +- Scripts: Provides top-level commands that fan out across workspaces using `pnpm -r` (lint, typecheck, unit tests) and project-specific runners (e.g. `lambda-build`). - Dev tool dependencies: Centralises Jest, TypeScript, ESLint configurations and plugins to keep versions consistent across workspaces. Workspace projects should rely on these unless a local override is strictly needed. - Overrides/resolutions: Pins transitive dependencies (e.g. Jest/react-is) to avoid ecosystem conflicts. Agents must not remove overrides without verifying tests across all workspaces. Agent guidance: -- Before adding or removing a workspace, update the root `workspaces` array and ensure CI scripts still succeed with `npm run lint`, `npm run typecheck`, and `npm run test:unit` at the repo root. -- When adding repo-wide scripts, keep names consistent with existing patterns (e.g. `lint`, `lint:fix`, `typecheck`, `test:unit`, `lambda-build`) and prefer `--workspaces` fan-out. +- Before adding or removing a workspace, update the root `workspaces` array and ensure CI scripts still succeed with `pnpm run lint`, `pnpm run typecheck`, and `pnpm run test:unit` at the repo root. +- When adding repo-wide scripts, keep names consistent with existing patterns (e.g. `lint`, `lint:fix`, `typecheck`, `test:unit`, `lambda-build`) and prefer `pnpm -r` fan-out. - Do not publish from the root. If adding a new workspace intended for publication, mark that workspace package as `private: false` and keep the root as private. - Validate changes by running the repo pre-commit hooks: `make githooks-run`. Success criteria for changes affecting the root `package.json`: -- `npm run lint`, `npm run typecheck`, and `npm run test:unit` pass at the repo root. -- Workspace discovery is correct (new projects appear under `npm run typecheck --workspaces`). -- No regression in lambda build tooling (`npm run lambda-build`). +- `pnpm run lint`, `pnpm run typecheck`, and `pnpm run test:unit` pass at the repo root. +- Workspace discovery is correct (new projects appear under `pnpm run typecheck -r`). +- No regression in lambda build tooling (`pnpm run lambda-build`). ## What Agents Can / Can’t Do @@ -81,7 +81,7 @@ When proposing a change, agents should: to catch formatting and basic lint issues. Domain specific checks will be defined in appropriate nested AGENTS.md files. -- Suggest at least one extra validation step (for example `npm test:unit` in a lambda, or triggering a specific workflow). +- Suggest at least one extra validation step (for example `pnpm run test:unit` in a lambda, or triggering a specific workflow). - Any required follow up activites which fall outside of the current task's scope should be clearly marked with a 'TODO: CCM-12345' comment. The human user should be prompted to create and provide a JIRA ticket ID to be added to the comment. ## Security & Safety diff --git a/README.md b/README.md index 08fda19f..600c5621 100644 --- a/README.md +++ b/README.md @@ -4,14 +4,19 @@ Event-driven infrastructure for delivering NHS Notify callback notifications to ## Overview -The Client Callbacks infrastructure processes message and channel status events, applies client-specific subscription filters, and delivers callbacks to configured webhook endpoints. Events flow from the Shared Event Bus through an SQS queue, are transformed and filtered by a Lambda function, then routed to clients via per-client API Destination Target Rules. +The Client Callbacks infrastructure processes message and channel status events, applies client-specific subscription filters, and delivers callbacks to configured webhook endpoints. Events flow from the Shared Event Bus through an SQS queue, are transformed and filtered by a Lambda function, then routed to per-client delivery queues where dedicated HTTPS Client Lambdas handle webhook delivery with mTLS support, per-target rate limiting, and circuit breaking. ### Key Features -- **Event-Driven Architecture**: Consumes CloudEvents from the Shared Event Bus (`uk.nhs.notify...` namespace) +- **Event-Driven Architecture**: Consumes CloudEvents from the Shared Event Bus - **Client Subscription Filtering**: Applies per-client rules for message status and channel status event types -- **Webhook Delivery**: EventBridge API Destinations with per-client configuration and retry policies -- **Failure Handling**: Per-client Dead Letter Queues +- **mTLS Webhook Delivery**: Per-client HTTPS Client Lambdas with mutual TLS and optional certificate pinning +- **Per-Target Rate Limiting**: Token bucket algorithm with configurable delivery rates per client target +- **Circuit Breaking**: Automatic throttling of consistently failing endpoints +- **Retry and Backoff**: Exponential backoff with jitter, configurable retry windows +- **Client Isolation**: Dedicated queues, Lambdas, and DLQs per client — one client's issues do not affect others +- **Event Archive**: 7-day event archive on the Callbacks Event Bus for operational replay +- **Failure Handling**: Per-client Dead Letter Queues for permanently failed deliveries - **Backward Compatibility**: Maintains callback payload format compatibility with legacy Core domain implementation ## Table of Contents @@ -23,6 +28,7 @@ The Client Callbacks infrastructure processes message and channel status events, - [Architecture](#architecture) - [Components](#components) - [Event Flow](#event-flow) + - [Packages](#packages) - [Setup](#setup) - [Prerequisites](#prerequisites) - [Configuration](#configuration) @@ -37,23 +43,60 @@ The Client Callbacks infrastructure processes message and channel status events, ### Components - **Shared Event Bus**: Cross-domain EventBridge bus receiving events from Core, Routing, and other NHS Notify domains -- **Callback Event Queue**: SQS queue subscribed to `uk.nhs.notify...` events via EventBridge Target Rule +- **Callback Event Queue**: SQS queue subscribed to `uk.nhs.notify.message.status.PUBLISHED...` / `uk.nhs.notify.channel.status.PUBLISHED...` events via EventBridge Target Rule - **Transform & Filter Lambda**: Processes events, loads client configurations, applies subscription filters, and routes to Callbacks Event Bus -- **Callbacks Event Bus**: Domain-specific EventBridge bus for webhook orchestration -- **API Destination Target Rules**: Per-client rules invoking HTTPS endpoints with client-specific authentication -- **Client Config Storage**: S3 bucket storing client subscription configurations (status filters, webhook endpoints) -- **Per-Client Target DLQs**: SQS Dead Letter Queues for failed webhook deliveries (one per client target) +- **Callbacks Event Bus**: Domain-specific EventBridge bus for webhook orchestration, with a 7-day event archive for replay +- **Per-Client SQS Queues**: Dedicated queues per client, receiving events from the Callbacks Event Bus via per-subscription EventBridge rules +- **HTTPS Client Lambdas**: Per-client Lambda functions handling webhook delivery with mTLS, payload signing, rate limiting, and circuit breaking +- **Delivery State Store**: ElastiCache Serverless (Valkey) cluster storing per-target rate-limit token bucket and circuit-breaker state +- **Client Config Storage**: S3 bucket storing client subscription configurations (status filters, webhook endpoints, mTLS settings) +- **Per-Client DLQs**: SQS Dead Letter Queues for permanently failed webhook deliveries (one per client) ### Event Flow -1. Status change events published to Shared Event Bus in `uk.nhs.notify...` namespace +1. Status change events published to Shared Event Bus in `uk.nhs.notify.message.status.PUBLISHED...` / `uk.nhs.notify.channel.status.PUBLISHED...` namespace 2. SQS Target Rule routes events to Callback Event Queue 3. EventBridge Pipe invokes Transform & Filter Lambda with event batches 4. Lambda loads client subscription configs from S3 5. Lambda applies client-specific filters (message status, channel status) 6. Matching events published to Callbacks Event Bus -7. API Destination Target Rules deliver callbacks to client webhook endpoints -8. Failed deliveries moved to per-client DLQs after retry exhaustion +7. Per-subscription EventBridge rules route events to per-client SQS queues +8. HTTPS Client Lambda processes batches from the per-client queue, applying rate limiting and circuit-breaker checks, signing payloads, and delivering via HTTPS with optional mTLS +9. Temporary failures are retried with exponential backoff; permanent failures are sent to per-client DLQs + +## Packages + +The repository is organised as a pnpm workspace. Each package has its own `package.json`, build configuration, and tests. + +### Lambdas + +| Package | Path | Description | +| -------------------------------------------------- | ----------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------- | +| `nhs-notify-client-transform-filter-lambda` | `lambdas/client-transform-filter-lambda/` | Processes inbound events, applies client subscription filters, and publishes matching events to the Callbacks Event Bus | +| `@nhs-notify-client-callbacks/https-client-lambda` | `lambdas/https-client-lambda/` | Per-client delivery Lambda — signs payloads, delivers via HTTPS with mTLS, handles retries, rate limiting, and circuit breaking | +| `nhs-notify-mock-webhook-lambda` | `lambdas/mock-webhook-lambda/` | Mock webhook endpoint for integration testing | +| `nhs-notify-perf-runner-lambda` | `lambdas/perf-runner-lambda/` | Performance test runner | + +### Shared Libraries + +| Package | Path | Description | +| -------------------------------------------------------- | -------------------------------- | --------------------------------------------------------------------------------------------------------- | +| `@nhs-notify-client-callbacks/models` | `src/models/` | Shared TypeScript types and zod schemas for client configuration, callback targets, and delivery messages | +| `@nhs-notify-client-callbacks/logger` | `src/logger/` | Structured JSON logging utility | +| `@nhs-notify-client-callbacks/config-subscription-cache` | `src/config-subscription-cache/` | TTL-based in-memory cache for client subscription configs loaded from S3 | + +### Tools + +| Package | Path | Description | +| --------------------------------- | ---------------------------------------- | ------------------------------------------------------------------------------------------------- | +| `client-subscriptions-management` | `tools/client-subscriptions-management/` | CLI for managing client subscriptions, targets, mTLS certificates, and application mappings in S3 | + +### Tests + +| Package | Path | Description | +| ----------------------------------------------- | --------------------- | --------------------------------------------------------- | +| `nhs-notify-client-callbacks-integration-tests` | `tests/integration/` | Integration tests run against deployed AWS infrastructure | +| `@nhs-notify-client-callbacks/test-support` | `tests/test-support/` | Shared test helpers, fixtures, and mock configurations | ## Setup @@ -102,32 +145,44 @@ make config Run unit tests for Lambda functions: ```shell -npm test +pnpm test:unit ``` -## Infrastructure +### Linting -Infrastructure is managed with Terraform under `infrastructure/terraform/`: +```shell +pnpm lint +``` -- `components/`: Terraform components for different environments/accounts -- `modules/`: Reusable Terraform modules for callback infrastructure +### Type Checking -**Deploy infrastructure**: +```shell +pnpm typecheck +``` + +### Full Verification + +Run lint, typecheck, and unit tests together: ```shell -cd infrastructure/terraform/components/ -terraform init -terraform plan -terraform apply +pnpm verify ``` +## Infrastructure + +Infrastructure is managed with Terraform under `infrastructure/terraform/`: + +- `components/`: Terraform components for different environments/accounts +- `modules/`: Reusable Terraform modules for callback infrastructure + Key infrastructure modules: - **callback-event-queue**: SQS queue and EventBridge Target Rule for Shared Event Bus subscription - **transform-filter-lambda**: Lambda function with EventBridge Pipe trigger -- **callbacks-event-bus**: Domain-specific EventBridge bus -- **api-destinations**: Per-client API Destination Target Rules -- **client-config-storage**: S3 bucket for subscription configurations +- **callbacks-event-bus**: Domain-specific EventBridge bus with 7-day event archive +- **client-delivery**: Per-client SQS queue, HTTPS Client Lambda (VPC-attached), DLQ, and EventBridge rules +- **elasticache-delivery-state**: Shared ElastiCache Serverless (Valkey) cluster for rate-limit and circuit-breaker state +- **client-config-storage**: S3 bucket for subscription configurations (including mTLS and certificate pinning settings) ## Contributing diff --git a/docs/Makefile b/docs/Makefile index ea4bc005..cafd8ae7 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -8,7 +8,7 @@ h help: @egrep '^\S|^$$' Makefile install: - pnpm install + npm install bundle config set --local path vendor/bundle bundle install diff --git a/docs/test-standards.md b/docs/test-standards.md index c8ab86cf..4fef9318 100644 --- a/docs/test-standards.md +++ b/docs/test-standards.md @@ -104,7 +104,7 @@ AI must: - Verify mock return types match the actual function return types. 7. **The "Test Execution" Mandate**: - - After creating or modifying a test, you MUST run it using the repo's test command - e.g. npm run test:unit + - After creating or modifying a test, you MUST run it using the repo's test command - e.g. pnpm run test:unit - If the test fails due to incorrect imports, paths, or signatures, fix and re-run. - Only report completion when the test passes (exit code 0) and test coverage checks also pass. - See section 6.2 for the full self-correction loop requirements. @@ -192,7 +192,7 @@ AI must: When AI changes tests, it must: -- run all the tests in the npm workspace. +- run all the tests in the pnpm workspace. - report exactly what it ran and whether it passed. ### 6.2 AI Self-Correction Loop diff --git a/eslint.config.mjs b/eslint.config.mjs index eb59432b..9ea6c3e0 100644 --- a/eslint.config.mjs +++ b/eslint.config.mjs @@ -28,6 +28,7 @@ export default defineConfig([ "**/test-results", "**/playwright-report*", "eslint.config.mjs", + "**/lua-transform.js", ]), //imports @@ -200,7 +201,7 @@ export default defineConfig([ }, }, { - files: ["**/utils/**", "tests/test-team/**", "tests/performance/helpers/**", "lambdas/**/src/**"], + files: ["**/utils/**", "tests/test-team/**", "tests/performance/helpers/**", "lambdas/**/src/**", "src/**/src/**"], rules: { "import-x/prefer-default-export": 0, }, diff --git a/infrastructure/terraform/components/callbacks/README.md b/infrastructure/terraform/components/callbacks/README.md index b1587725..c3995c03 100644 --- a/infrastructure/terraform/components/callbacks/README.md +++ b/infrastructure/terraform/components/callbacks/README.md @@ -8,16 +8,23 @@ |------|---------| | [terraform](#requirement\_terraform) | >= 1.10.1 | | [aws](#requirement\_aws) | 6.13 | +| [external](#requirement\_external) | ~> 2.0 | | [random](#requirement\_random) | ~> 3.0 | +| [tls](#requirement\_tls) | ~> 4.0 | ## Inputs | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| -| [applications\_map\_parameter\_name](#input\_applications\_map\_parameter\_name) | SSM Parameter Store path for the clientId-to-applicationData map, where applicationData is currently only the applicationId | `string` | `null` | no | | [aws\_account\_id](#input\_aws\_account\_id) | The AWS Account ID (numeric) | `string` | n/a | yes | +| [cb\_cooldown\_period\_ms](#input\_cb\_cooldown\_period\_ms) | Full block duration after circuit opens, before half-open probes begin (ms) | `number` | `120000` | no | +| [cb\_recovery\_period\_ms](#input\_cb\_recovery\_period\_ms) | Linear ramp-up duration after circuit closes (ms) | `number` | `600000` | no | | [component](#input\_component) | The variable encapsulating the name of this component | `string` | `"callbacks"` | no | | [default\_tags](#input\_default\_tags) | A map of default tags to apply to all taggable resources within the component | `map(string)` | `{}` | no | +| [delivery\_lambda\_batch\_size](#input\_delivery\_lambda\_batch\_size) | Number of SQS messages per delivery Lambda invocation | `number` | `100` | no | +| [delivery\_lambda\_batching\_window\_sec](#input\_delivery\_lambda\_batching\_window\_sec) | Maximum time in seconds to wait for a full batch before invoking the delivery Lambda | `number` | `1` | no | | [deploy\_mock\_clients](#input\_deploy\_mock\_clients) | Flag to deploy mock webhook lambda for integration testing (test/dev environments only) | `bool` | `false` | no | +| [deploy\_perf\_runner](#input\_deploy\_perf\_runner) | Flag to deploy the perf-runner lambda for performance testing (test/dev environments only) | `bool` | `false` | no | +| [elasticache\_data\_storage\_maximum\_gb](#input\_elasticache\_data\_storage\_maximum\_gb) | Maximum data storage in GB for the ElastiCache Serverless delivery state cache | `number` | `1` | no | | [enable\_event\_anomaly\_detection](#input\_enable\_event\_anomaly\_detection) | Enable CloudWatch anomaly detection alarm for inbound event queue message reception | `bool` | `true` | no | | [enable\_xray\_tracing](#input\_enable\_xray\_tracing) | Enable AWS X-Ray active tracing for Lambda functions | `bool` | `false` | no | | [environment](#input\_environment) | The name of the tfscaffold environment | `string` | n/a | yes | @@ -30,25 +37,28 @@ | [log\_level](#input\_log\_level) | The log level to be used in lambda functions within the component. Any log with a lower severity than the configured value will not be logged: https://docs.python.org/3/library/logging.html#levels | `string` | `"INFO"` | no | | [log\_retention\_in\_days](#input\_log\_retention\_in\_days) | The retention period in days for the Cloudwatch Logs events to be retained, default of 0 is indefinite | `number` | `0` | no | | [message\_root\_uri](#input\_message\_root\_uri) | The root URI used for constructing message links in callback payloads | `string` | n/a | yes | +| [mtls\_ca\_s3\_key](#input\_mtls\_ca\_s3\_key) | S3 key for the CA certificate PEM bundle used for server verification | `string` | `""` | no | +| [mtls\_cert\_s3\_key](#input\_mtls\_cert\_s3\_key) | S3 key for the mTLS client certificate PEM bundle | `string` | `""` | no | | [parent\_acct\_environment](#input\_parent\_acct\_environment) | Name of the environment responsible for the acct resources used, affects things like DNS zone. Useful for named dev environments | `string` | `"main"` | no | | [pipe\_event\_patterns](#input\_pipe\_event\_patterns) | value | `list(string)` | `[]` | no | | [pipe\_log\_level](#input\_pipe\_log\_level) | Log level for the EventBridge Pipe. | `string` | `"ERROR"` | no | -| [pipe\_sqs\_input\_batch\_size](#input\_pipe\_sqs\_input\_batch\_size) | n/a | `number` | `1` | no | +| [pipe\_sqs\_input\_batch\_size](#input\_pipe\_sqs\_input\_batch\_size) | n/a | `number` | `10` | no | | [pipe\_sqs\_max\_batch\_window](#input\_pipe\_sqs\_max\_batch\_window) | n/a | `number` | `2` | no | | [project](#input\_project) | The name of the tfscaffold project | `string` | n/a | yes | | [region](#input\_region) | The AWS Region | `string` | n/a | yes | -| [s3\_enable\_force\_destroy](#input\_s3\_enable\_force\_destroy) | Whether to enable force destroy for the S3 buckets created in this module | `bool` | `false` | no | | [sqs\_inbound\_event\_max\_receive\_count](#input\_sqs\_inbound\_event\_max\_receive\_count) | n/a | `number` | `3` | no | | [sqs\_inbound\_event\_visibility\_timeout\_seconds](#input\_sqs\_inbound\_event\_visibility\_timeout\_seconds) | n/a | `number` | `60` | no | +| [token\_bucket\_burst\_capacity](#input\_token\_bucket\_burst\_capacity) | Token bucket burst capacity used by the rate limiter | `number` | `2250` | no | ## Modules | Name | Source | Version | |------|--------|---------| -| [client\_config\_bucket](#module\_client\_config\_bucket) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/3.0.7/terraform-s3bucket.zip | n/a | -| [client\_destination](#module\_client\_destination) | ../../modules/client-destination | n/a | +| [client\_delivery](#module\_client\_delivery) | ../../modules/client-delivery | n/a | | [client\_transform\_filter\_lambda](#module\_client\_transform\_filter\_lambda) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/3.0.7/terraform-lambda.zip | n/a | | [kms](#module\_kms) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/3.0.7/terraform-kms.zip | n/a | +| [mock\_webhook\_alb\_mtls](#module\_mock\_webhook\_alb\_mtls) | ../../modules/mock-webhook-alb-mtls | n/a | | [mock\_webhook\_lambda](#module\_mock\_webhook\_lambda) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/3.0.7/terraform-lambda.zip | n/a | +| [perf\_runner\_lambda](#module\_perf\_runner\_lambda) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/3.0.7/terraform-lambda.zip | n/a | | [sqs\_inbound\_event](#module\_sqs\_inbound\_event) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/3.0.7/terraform-sqs.zip | n/a | ## Outputs diff --git a/infrastructure/terraform/components/callbacks/cloudwatch_event_archive_main.tf b/infrastructure/terraform/components/callbacks/cloudwatch_event_archive_main.tf new file mode 100644 index 00000000..d23c38fc --- /dev/null +++ b/infrastructure/terraform/components/callbacks/cloudwatch_event_archive_main.tf @@ -0,0 +1,5 @@ +resource "aws_cloudwatch_event_archive" "main" { + name = "${local.csi}-archive" + event_source_arn = aws_cloudwatch_event_bus.main.arn + retention_days = 7 +} diff --git a/infrastructure/terraform/components/callbacks/cloudwatch_eventbus_main.tf b/infrastructure/terraform/components/callbacks/cloudwatch_event_bus_main.tf similarity index 100% rename from infrastructure/terraform/components/callbacks/cloudwatch_eventbus_main.tf rename to infrastructure/terraform/components/callbacks/cloudwatch_event_bus_main.tf diff --git a/infrastructure/terraform/components/callbacks/cloudwatch_metric_alarm_dlq_depth.tf b/infrastructure/terraform/components/callbacks/cloudwatch_metric_alarm_dlq_depth.tf deleted file mode 100644 index e6ed2d9d..00000000 --- a/infrastructure/terraform/components/callbacks/cloudwatch_metric_alarm_dlq_depth.tf +++ /dev/null @@ -1,31 +0,0 @@ -resource "aws_cloudwatch_metric_alarm" "client_dlq_depth" { - for_each = toset(keys(local.config_targets)) - - alarm_name = "${local.csi}-${each.key}-dlq-depth" - alarm_description = join(" ", [ - "RELIABILITY: Messages are in DLQ for ${each.key}.", - "Failed callback deliveries require operator attention.", - ]) - - comparison_operator = "GreaterThanThreshold" - evaluation_periods = 1 - metric_name = "ApproximateNumberOfMessagesVisible" - namespace = "AWS/SQS" - period = 300 - statistic = "Sum" - threshold = 0 - actions_enabled = true - treat_missing_data = "notBreaching" - - dimensions = { - QueueName = "${local.csi}-${each.key}-dlq-queue" - } - - tags = merge( - local.default_tags, - { - Name = "${local.csi}-${each.key}-dlq-depth" - Client = local.config_targets[each.key].client_id - }, - ) -} diff --git a/infrastructure/terraform/components/callbacks/cloudwatch_metric_alarm_elasticache_connections.tf b/infrastructure/terraform/components/callbacks/cloudwatch_metric_alarm_elasticache_connections.tf new file mode 100644 index 00000000..eb4b576c --- /dev/null +++ b/infrastructure/terraform/components/callbacks/cloudwatch_metric_alarm_elasticache_connections.tf @@ -0,0 +1,28 @@ +resource "aws_cloudwatch_metric_alarm" "elasticache_connections" { + alarm_name = "${local.csi}-elasticache-connections" + alarm_description = join(" ", [ + "RELIABILITY: ElastiCache connection count is high.", + "Review per-client Lambda connection pool sizing.", + ]) + + comparison_operator = "GreaterThanThreshold" + evaluation_periods = 2 + metric_name = "CurrConnections" + namespace = "AWS/ElastiCache" + period = 300 + statistic = "Maximum" + threshold = 500 + actions_enabled = true + treat_missing_data = "notBreaching" + + dimensions = { + CacheClusterId = aws_elasticache_serverless_cache.delivery_state.name + } + + tags = merge( + local.default_tags, + { + Name = "${local.csi}-elasticache-connections" + }, + ) +} diff --git a/infrastructure/terraform/components/callbacks/cloudwatch_metric_alarm_elasticache_ecpu_utilisation.tf b/infrastructure/terraform/components/callbacks/cloudwatch_metric_alarm_elasticache_ecpu_utilisation.tf new file mode 100644 index 00000000..c564bb85 --- /dev/null +++ b/infrastructure/terraform/components/callbacks/cloudwatch_metric_alarm_elasticache_ecpu_utilisation.tf @@ -0,0 +1,28 @@ +resource "aws_cloudwatch_metric_alarm" "elasticache_ecpu_utilisation" { + alarm_name = "${local.csi}-elasticache-ecpu-utilisation" + alarm_description = join(" ", [ + "PERFORMANCE: ElastiCache processing units utilisation is high.", + "Consider scaling up or optimising Redis commands.", + ]) + + comparison_operator = "GreaterThanThreshold" + evaluation_periods = 3 + metric_name = "ElastiCacheProcessingUnits" + namespace = "AWS/ElastiCache" + period = 300 + statistic = "Average" + threshold = 80 + actions_enabled = true + treat_missing_data = "notBreaching" + + dimensions = { + CacheClusterId = aws_elasticache_serverless_cache.delivery_state.name + } + + tags = merge( + local.default_tags, + { + Name = "${local.csi}-elasticache-ecpu-utilisation" + }, + ) +} diff --git a/infrastructure/terraform/components/callbacks/cloudwatch_metric_alarm_elasticache_storage_utilisation.tf b/infrastructure/terraform/components/callbacks/cloudwatch_metric_alarm_elasticache_storage_utilisation.tf new file mode 100644 index 00000000..88f3f092 --- /dev/null +++ b/infrastructure/terraform/components/callbacks/cloudwatch_metric_alarm_elasticache_storage_utilisation.tf @@ -0,0 +1,43 @@ +resource "aws_cloudwatch_metric_alarm" "elasticache_storage_utilisation" { + alarm_name = "${local.csi}-elasticache-storage-utilisation" + alarm_description = join(" ", [ + "CAPACITY: ElastiCache data storage utilisation exceeds 80%.", + "Review stored data or increase elasticache_data_storage_maximum_gb.", + ]) + + comparison_operator = "GreaterThanThreshold" + evaluation_periods = 2 + threshold = var.elasticache_data_storage_maximum_gb * 0.8 + actions_enabled = true + treat_missing_data = "notBreaching" + + metric_query { + id = "storage_used" + return_data = false + + metric { + metric_name = "BytesUsedForCache" + namespace = "AWS/ElastiCache" + period = 300 + stat = "Maximum" + + dimensions = { + CacheClusterId = aws_elasticache_serverless_cache.delivery_state.name + } + } + } + + metric_query { + id = "storage_used_gb" + expression = "storage_used / 1073741824" + label = "Storage Used (GB)" + return_data = true + } + + tags = merge( + local.default_tags, + { + Name = "${local.csi}-elasticache-storage-utilisation" + }, + ) +} diff --git a/infrastructure/terraform/components/callbacks/cloudwatch_metric_alarm_elasticache_throttled_ops.tf b/infrastructure/terraform/components/callbacks/cloudwatch_metric_alarm_elasticache_throttled_ops.tf new file mode 100644 index 00000000..9790a4b5 --- /dev/null +++ b/infrastructure/terraform/components/callbacks/cloudwatch_metric_alarm_elasticache_throttled_ops.tf @@ -0,0 +1,28 @@ +resource "aws_cloudwatch_metric_alarm" "elasticache_throttled_ops" { + alarm_name = "${local.csi}-elasticache-throttled-ops" + alarm_description = join(" ", [ + "PERFORMANCE: ElastiCache throttled operations detected.", + "Increase ECPU limit or reduce request rate.", + ]) + + comparison_operator = "GreaterThanThreshold" + evaluation_periods = 2 + metric_name = "ThrottledCmds" + namespace = "AWS/ElastiCache" + period = 300 + statistic = "Sum" + threshold = 0 + actions_enabled = true + treat_missing_data = "notBreaching" + + dimensions = { + CacheClusterId = aws_elasticache_serverless_cache.delivery_state.name + } + + tags = merge( + local.default_tags, + { + Name = "${local.csi}-elasticache-throttled-ops" + }, + ) +} diff --git a/infrastructure/terraform/components/callbacks/data_external_mock_server_spki_hash.tf b/infrastructure/terraform/components/callbacks/data_external_mock_server_spki_hash.tf new file mode 100644 index 00000000..a95b9d31 --- /dev/null +++ b/infrastructure/terraform/components/callbacks/data_external_mock_server_spki_hash.tf @@ -0,0 +1,16 @@ +data "external" "mock_server_spki_hash" { + count = var.deploy_mock_clients ? 1 : 0 + program = ["bash", "-c", <<-EOT + HASH=$(jq -r '.pem' \ + | openssl pkey -pubin -outform DER 2>/dev/null \ + | openssl dgst -sha256 -binary \ + | base64 \ + | tr -d '\n') + printf '{"hash":"%s"}' "$HASH" + EOT + ] + + query = { + pem = tls_private_key.mock_server[0].public_key_pem + } +} diff --git a/infrastructure/terraform/components/callbacks/elasticache_serverless_cache_delivery_state.tf b/infrastructure/terraform/components/callbacks/elasticache_serverless_cache_delivery_state.tf new file mode 100644 index 00000000..f63c2506 --- /dev/null +++ b/infrastructure/terraform/components/callbacks/elasticache_serverless_cache_delivery_state.tf @@ -0,0 +1,34 @@ +resource "aws_elasticache_serverless_cache" "delivery_state" { + name = "${local.csi}-delivery-state" + engine = "valkey" + major_engine_version = "8" + description = "Per-target rate limiting and circuit breaker state for callback delivery" + + snapshot_retention_limit = 0 + + user_group_id = aws_elasticache_user_group.delivery_state.user_group_id + + security_group_ids = [aws_security_group.elasticache_delivery_state.id] + subnet_ids = try(local.acct.private_subnets[local.bc_name], []) + + kms_key_id = module.kms.key_arn + + cache_usage_limits { + data_storage { + maximum = var.elasticache_data_storage_maximum_gb + unit = "GB" + } + + ecpu_per_second { + maximum = 1000 + } + } + + tags = merge( + local.default_tags, + { + Name = "${local.csi}-delivery-state" + Description = "Callback delivery rate limiter and circuit breaker state" + }, + ) +} diff --git a/infrastructure/terraform/components/callbacks/elasticache_user_delivery_state_default.tf b/infrastructure/terraform/components/callbacks/elasticache_user_delivery_state_default.tf new file mode 100644 index 00000000..bf92083f --- /dev/null +++ b/infrastructure/terraform/components/callbacks/elasticache_user_delivery_state_default.tf @@ -0,0 +1,13 @@ +resource "aws_elasticache_user" "delivery_state_default" { + user_id = "${local.csi}-valkey-default" + user_name = "default" + engine = "valkey" + access_string = "off -@all" + + authentication_mode { + type = "password" + passwords = [random_password.elasticache_default_user.result] + } + + tags = local.default_tags +} diff --git a/infrastructure/terraform/components/callbacks/elasticache_user_delivery_state_iam.tf b/infrastructure/terraform/components/callbacks/elasticache_user_delivery_state_iam.tf new file mode 100644 index 00000000..00a255ce --- /dev/null +++ b/infrastructure/terraform/components/callbacks/elasticache_user_delivery_state_iam.tf @@ -0,0 +1,12 @@ +resource "aws_elasticache_user" "delivery_state_iam" { + user_id = "${local.csi}-elasticache-user" + user_name = "${local.csi}-elasticache-user" + engine = "valkey" + access_string = "on ~* &* +@all" + + authentication_mode { + type = "iam" + } + + tags = local.default_tags +} diff --git a/infrastructure/terraform/components/callbacks/elasticache_user_group_delivery_state.tf b/infrastructure/terraform/components/callbacks/elasticache_user_group_delivery_state.tf new file mode 100644 index 00000000..92dcdc6a --- /dev/null +++ b/infrastructure/terraform/components/callbacks/elasticache_user_group_delivery_state.tf @@ -0,0 +1,11 @@ +resource "aws_elasticache_user_group" "delivery_state" { + engine = "valkey" + user_group_id = "${local.csi}-delivery-state" + + user_ids = [ + aws_elasticache_user.delivery_state_default.user_id, + aws_elasticache_user.delivery_state_iam.user_id, + ] + + tags = local.default_tags +} diff --git a/infrastructure/terraform/components/callbacks/locals.tf b/infrastructure/terraform/components/callbacks/locals.tf index f4707154..ec796beb 100644 --- a/infrastructure/terraform/components/callbacks/locals.tf +++ b/infrastructure/terraform/components/callbacks/locals.tf @@ -1,8 +1,18 @@ locals { + bc_name = "client-callbacks" + component = "cb" + client_csi = "${var.project}-${var.environment}-cbc" aws_lambda_functions_dir_path = "../../../../lambdas" log_destination_arn = "arn:aws:firehose:${var.region}:${var.aws_account_id}:deliverystream/nhs-main-obs-splunk-logs-firehose" - root_domain_name = "${var.environment}.${local.acct.route53_zone_names["client-callbacks"]}" # e.g. [main|dev|abxy0].smsnudge.[dev|nonprod|prod].nhsnotify.national.nhs.uk - root_domain_id = local.acct.route53_zone_ids["client-callbacks"] + root_domain_name = "${var.environment}.${local.acct.route53_zone_names[local.bc_name]}" # e.g. [main|dev|abxy0].client-callbacks.[dev|nonprod|prod].nhsnotify.national.nhs.uk + root_domain_id = local.acct.route53_zone_ids[local.bc_name] + + mtls_test_certs_s3_prefix = "${var.environment}/callbacks/mtls-test" + mtls_test_cert_s3_key = "${local.mtls_test_certs_s3_prefix}/client-bundle.pem" + mtls_test_ca_s3_key = "${local.mtls_test_certs_s3_prefix}/ca.pem" + mtls_cert_s3_bucket = local.acct.additional_s3_buckets["client-callbacks_certs"].name + mtls_cert_s3_key = var.deploy_mock_clients ? local.mtls_test_cert_s3_key : var.mtls_cert_s3_key # gitleaks:allow + mtls_ca_s3_key = var.deploy_mock_clients ? local.mtls_test_ca_s3_key : var.mtls_ca_s3_key # gitleaks:allow clients_dir_path = "${path.module}/../../modules/clients" @@ -12,6 +22,10 @@ locals { } ]...) + # SPKI hash of the mock webhook server certificate for cert-pinning enrichment. + # Computed via external data source because Terraform cannot SHA-256 hash raw binary (DER) data natively. + mock_server_spki_hash = var.deploy_mock_clients ? data.external.mock_server_spki_hash[0].result.hash : "" + # When deploying mock clients, replace sentinel placeholder values with the mock webhook URL and API key. # Only used for S3 object content — must not be used as a for_each source (contains apply-time values). enriched_mock_config_clients = var.deploy_mock_clients ? { @@ -20,47 +34,48 @@ locals { targets = [ for target in try(client.targets, []) : merge(target, { - invocationEndpoint = "${aws_lambda_function_url.mock_webhook[0].function_url}${target.targetId}" + invocationEndpoint = "https://${module.mock_webhook_alb_mtls[0].dns_name}/${target.targetId}" apiKey = merge(target.apiKey, { headerValue = random_password.mock_webhook_api_key[0].result }) + delivery = merge(try(target.delivery, {}), { + mtls = merge(try(target.delivery.mtls, {}), { + certPinning = merge(try(target.delivery.mtls.certPinning, {}), try(target.delivery.mtls.certPinning.enabled, false) ? { + spkiHash = local.mock_server_spki_hash + } : {}) + }) + }) }) ] }) } : local.config_clients - config_targets = merge([ - for client_id, data in local.config_clients : { - for target in try(data.targets, []) : target.targetId => { - client_id = client_id - target_id = target.targetId - invocation_endpoint = var.deploy_mock_clients ? "${aws_lambda_function_url.mock_webhook[0].function_url}${target.targetId}" : target.invocationEndpoint - invocation_rate_limit_per_second = target.invocationRateLimit - http_method = target.invocationMethod - header_name = target.apiKey.headerName - header_value = var.deploy_mock_clients ? random_password.mock_webhook_api_key[0].result : target.apiKey.headerValue - } - } - ]...) - - config_subscriptions = merge([ - for client_id, data in local.config_clients : { - for subscription in try(data.subscriptions, []) : subscription.subscriptionId => { - client_id = client_id + client_subscriptions = { + for client_id, data in local.config_clients : + client_id => { + for subscription in try(data.subscriptions, []) : + subscription.subscriptionId => { subscription_id = subscription.subscriptionId target_ids = try(subscription.targetIds, []) } } - ]...) + } - subscription_targets = merge([ - for subscription_id, subscription in local.config_subscriptions : { - for target_id in subscription.target_ids : - "${subscription_id}-${target_id}" => { - subscription_id = subscription_id - target_id = target_id + client_subscription_targets = { + for client_id, data in local.config_clients : + client_id => merge([ + for subscription in try(data.subscriptions, []) : { + for target_id in try(subscription.targetIds, []) : + "${subscription.subscriptionId}-${target_id}" => { + subscription_id = subscription.subscriptionId + target_id = target_id + } } - } - ]...) + ]...) + } + + applications_map_s3_key = "${var.environment}/applications-map.json" - applications_map_parameter_name = coalesce(var.applications_map_parameter_name, "/${var.project}/${var.environment}/${var.component}/applications-map") + client_config_s3_bucket = local.acct.additional_s3_buckets["client-callbacks_client-configs"].name + applications_map_s3_bucket = local.acct.additional_s3_buckets["client-callbacks_apps-map"].name + client_config_bucket_arn = local.acct.additional_s3_buckets["client-callbacks_client-configs"].arn } diff --git a/infrastructure/terraform/components/callbacks/locals_tfscaffold.tf b/infrastructure/terraform/components/callbacks/locals_tfscaffold.tf index b7cf3217..5206fd1a 100644 --- a/infrastructure/terraform/components/callbacks/locals_tfscaffold.tf +++ b/infrastructure/terraform/components/callbacks/locals_tfscaffold.tf @@ -11,7 +11,7 @@ locals { "%s-%s-%s", var.project, var.environment, - var.component, + local.component, ), "_", "", @@ -25,7 +25,7 @@ locals { var.aws_account_id, var.region, var.environment, - var.component, + local.component, ), "_", "", diff --git a/infrastructure/terraform/components/callbacks/module_client_delivery.tf b/infrastructure/terraform/components/callbacks/module_client_delivery.tf new file mode 100644 index 00000000..fb95f01d --- /dev/null +++ b/infrastructure/terraform/components/callbacks/module_client_delivery.tf @@ -0,0 +1,54 @@ +module "client_delivery" { + source = "../../modules/client-delivery" + for_each = local.config_clients + + project = var.project + aws_account_id = var.aws_account_id + region = var.region + component = "cbc" + environment = var.environment + group = var.group + + client_id = each.key + client_bus_name = aws_cloudwatch_event_bus.main.name + kms_key_arn = module.kms.key_arn + + subscriptions = local.client_subscriptions[each.key] + subscription_targets = local.client_subscription_targets[each.key] + + client_config_bucket = local.client_config_s3_bucket + client_config_bucket_arn = local.client_config_bucket_arn + client_config_key_prefix = "${var.environment}/client_subscriptions/" + + applications_map_s3_bucket = local.applications_map_s3_bucket + applications_map_s3_key = local.applications_map_s3_key + + delivery_lambda_s3_bucket = local.acct.s3_buckets["lambda_function_artefacts"]["id"] + delivery_lambda_code_base_path = local.aws_lambda_functions_dir_path + + force_lambda_code_deploy = var.force_lambda_code_deploy + log_level = var.log_level + log_retention_in_days = var.log_retention_in_days + enable_xray_tracing = var.enable_xray_tracing + + log_destination_arn = local.log_destination_arn + log_subscription_role_arn = local.acct.log_subscription_role_arn + + elasticache_endpoint = aws_elasticache_serverless_cache.delivery_state.endpoint[0].address + elasticache_cache_name = aws_elasticache_serverless_cache.delivery_state.name + elasticache_iam_username = "${var.project}-${var.environment}-${local.component}-elasticache-user" + + mtls_cert_s3_bucket = local.mtls_cert_s3_bucket + mtls_cert_s3_key = local.mtls_cert_s3_key # gitleaks:allow + mtls_ca_s3_key = local.mtls_ca_s3_key # gitleaks:allow + + token_bucket_burst_capacity = var.token_bucket_burst_capacity + cb_cooldown_period_ms = var.cb_cooldown_period_ms + cb_recovery_period_ms = var.cb_recovery_period_ms + + delivery_lambda_batch_size = var.delivery_lambda_batch_size + delivery_lambda_batching_window_sec = var.delivery_lambda_batching_window_sec + + vpc_subnet_ids = try(local.acct.private_subnets[local.bc_name], []) + delivery_lambda_security_group_id = aws_security_group.https_client_lambda.id +} diff --git a/infrastructure/terraform/components/callbacks/module_client_destination.tf b/infrastructure/terraform/components/callbacks/module_client_destination.tf deleted file mode 100644 index 21800e94..00000000 --- a/infrastructure/terraform/components/callbacks/module_client_destination.tf +++ /dev/null @@ -1,17 +0,0 @@ -module "client_destination" { - source = "../../modules/client-destination" - - project = var.project - aws_account_id = var.aws_account_id - region = var.region - component = var.component - environment = var.environment - client_bus_name = aws_cloudwatch_event_bus.main.name - - kms_key_arn = module.kms.key_arn - - targets = local.config_targets - subscriptions = local.config_subscriptions - subscription_targets = local.subscription_targets - -} diff --git a/infrastructure/terraform/components/callbacks/module_kms.tf b/infrastructure/terraform/components/callbacks/module_kms.tf index 327b5641..af8fddae 100644 --- a/infrastructure/terraform/components/callbacks/module_kms.tf +++ b/infrastructure/terraform/components/callbacks/module_kms.tf @@ -2,7 +2,7 @@ module "kms" { source = "https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/3.0.7/terraform-kms.zip" aws_account_id = var.aws_account_id - component = var.component + component = local.component environment = var.environment project = var.project region = var.region @@ -64,9 +64,10 @@ data "aws_iam_policy_document" "kms" { test = "ArnLike" variable = "kms:EncryptionContext:aws:sqs:arn" values = [ - "arn:aws:sqs:${var.region}:${var.aws_account_id}:${var.project}-${var.environment}-callbacks-inbound-event-queue", - "arn:aws:sqs:${var.region}:${var.aws_account_id}:${var.project}-${var.environment}-callbacks-inbound-event-dlq", - "arn:aws:sqs:${var.region}:${var.aws_account_id}:${var.project}-${var.environment}-callbacks-*-dlq-queue" #wildcard here so that DLQs for clients can also use this key + "arn:aws:sqs:${var.region}:${var.aws_account_id}:${var.project}-${var.environment}-${local.component}-inbound-event-queue", + "arn:aws:sqs:${var.region}:${var.aws_account_id}:${var.project}-${var.environment}-${local.component}-inbound-event-dlq", + "arn:aws:sqs:${var.region}:${var.aws_account_id}:${local.client_csi}-*-delivery-queue", + "arn:aws:sqs:${var.region}:${var.aws_account_id}:${local.client_csi}-*-delivery-dlq-queue", ] } } diff --git a/infrastructure/terraform/components/callbacks/module_mock_webhook_alb_mtls.tf b/infrastructure/terraform/components/callbacks/module_mock_webhook_alb_mtls.tf new file mode 100644 index 00000000..6287da23 --- /dev/null +++ b/infrastructure/terraform/components/callbacks/module_mock_webhook_alb_mtls.tf @@ -0,0 +1,18 @@ +module "mock_webhook_alb_mtls" { + count = var.deploy_mock_clients ? 1 : 0 + source = "../../modules/mock-webhook-alb-mtls" + + csi = local.csi + vpc_id = local.acct.vpc_ids[local.bc_name] + private_subnets = try(local.acct.private_subnets[local.bc_name], []) + + https_client_lambda_sg_id = aws_security_group.https_client_lambda.id + mock_webhook_lambda_function_name = module.mock_webhook_lambda[0].function_name + mock_webhook_lambda_function_arn = module.mock_webhook_lambda[0].function_arn + + server_cert_pem = tls_locally_signed_cert.mock_server[0].cert_pem + server_private_key_pem = tls_private_key.mock_server[0].private_key_pem + ca_cert_pem = tls_self_signed_cert.test_ca[0].cert_pem + + default_tags = local.default_tags +} diff --git a/infrastructure/terraform/components/callbacks/module_mock_webhook_lambda.tf b/infrastructure/terraform/components/callbacks/module_mock_webhook_lambda.tf index b951351e..6f2a5f9b 100644 --- a/infrastructure/terraform/components/callbacks/module_mock_webhook_lambda.tf +++ b/infrastructure/terraform/components/callbacks/module_mock_webhook_lambda.tf @@ -6,7 +6,7 @@ module "mock_webhook_lambda" { description = "Mock webhook endpoint for integration testing - logs received callbacks to CloudWatch" aws_account_id = var.aws_account_id - component = var.component + component = local.component environment = var.environment project = var.project region = var.region @@ -64,34 +64,3 @@ data "aws_iam_policy_document" "mock_webhook_lambda" { ] } } - -# Lambda Function URL for mock webhook (test/dev only) -resource "aws_lambda_function_url" "mock_webhook" { - count = var.deploy_mock_clients ? 1 : 0 - function_name = module.mock_webhook_lambda[0].function_name - authorization_type = "NONE" # Public endpoint for testing - - cors { - allow_origins = ["*"] - allow_methods = ["POST"] - allow_headers = ["*"] - max_age = 86400 - } -} - -resource "aws_lambda_permission" "mock_webhook_function_url" { - count = var.deploy_mock_clients ? 1 : 0 - statement_id_prefix = "FunctionURLAllowPublicAccess" - action = "lambda:InvokeFunctionUrl" - function_name = module.mock_webhook_lambda[0].function_name - principal = "*" - function_url_auth_type = "NONE" -} - -resource "aws_lambda_permission" "mock_webhook_function_invoke" { - count = var.deploy_mock_clients ? 1 : 0 - statement_id_prefix = "FunctionURLAllowInvokeAction" - action = "lambda:InvokeFunction" - function_name = module.mock_webhook_lambda[0].function_name - principal = "*" -} diff --git a/infrastructure/terraform/components/callbacks/module_perf_runner_lambda.tf b/infrastructure/terraform/components/callbacks/module_perf_runner_lambda.tf new file mode 100644 index 00000000..aee63a91 --- /dev/null +++ b/infrastructure/terraform/components/callbacks/module_perf_runner_lambda.tf @@ -0,0 +1,178 @@ +module "perf_runner_lambda" { + count = var.deploy_perf_runner ? 1 : 0 + source = "https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/3.0.7/terraform-lambda.zip" + + function_name = "perf-runner" + description = "Lambda function that executes performance tests against the client callbacks pipeline from within AWS" + + aws_account_id = var.aws_account_id + component = local.component + environment = var.environment + project = var.project + region = var.region + group = var.group + + log_retention_in_days = var.log_retention_in_days + kms_key_arn = module.kms.key_arn + + iam_policy_document = { + body = data.aws_iam_policy_document.perf_runner_lambda[0].json + } + + function_s3_bucket = local.acct.s3_buckets["lambda_function_artefacts"]["id"] + function_code_base_path = local.aws_lambda_functions_dir_path + function_code_dir = "perf-runner-lambda/dist" + handler_function_name = "handler" + runtime = "nodejs22.x" + memory = 512 + timeout = 900 + + log_level = var.log_level + force_lambda_code_deploy = var.force_lambda_code_deploy + enable_lambda_insights = false + enable_xray_tracing = false + + log_destination_arn = local.log_destination_arn + log_subscription_role_arn = local.acct.log_subscription_role_arn + + lambda_env_vars = { + ENVIRONMENT = var.environment + INBOUND_QUEUE_URL = module.sqs_inbound_event.sqs_queue_url + DELIVERY_QUEUE_URL_PREFIX = "https://sqs.${var.region}.amazonaws.com/${var.aws_account_id}/${local.client_csi}-" + TRANSFORM_FILTER_LOG_GROUP = module.client_transform_filter_lambda.cloudwatch_log_group_name + DELIVERY_LOG_GROUP_PREFIX = "/aws/lambda/${local.client_csi}-https-client-" + MOCK_WEBHOOK_LOG_GROUP = var.deploy_mock_clients ? module.mock_webhook_lambda[0].cloudwatch_log_group_name : "" + ELASTICACHE_ENDPOINT = aws_elasticache_serverless_cache.delivery_state.endpoint[0].address + ELASTICACHE_CACHE_NAME = aws_elasticache_serverless_cache.delivery_state.name + ELASTICACHE_IAM_USERNAME = "${var.project}-${var.environment}-${local.component}-elasticache-user" + } + + vpc_config = { + subnet_ids = try(local.acct.private_subnets[local.bc_name], []) + security_group_ids = [aws_security_group.https_client_lambda.id] + } +} + +data "aws_iam_policy_document" "perf_runner_lambda" { + count = var.deploy_perf_runner ? 1 : 0 + + statement { + sid = "KMSPermissions" + effect = "Allow" + + actions = [ + "kms:Decrypt", + "kms:GenerateDataKey", + ] + + resources = [ + module.kms.key_arn, + ] + } + + statement { + sid = "SQSSendMessage" + effect = "Allow" + + actions = [ + "sqs:SendMessage", + "sqs:SendMessageBatch", + ] + + resources = [ + module.sqs_inbound_event.sqs_queue_arn, + ] + } + + statement { + sid = "SQSPurgeQueue" + effect = "Allow" + + actions = [ + "sqs:PurgeQueue", + ] + + resources = [ + module.sqs_inbound_event.sqs_queue_arn, + module.sqs_inbound_event.sqs_dlq_arn, + "arn:aws:sqs:${var.region}:${var.aws_account_id}:${local.client_csi}-*-delivery-queue", + "arn:aws:sqs:${var.region}:${var.aws_account_id}:${local.client_csi}-*-delivery-dlq-queue", + ] + } + + statement { + sid = "SQSGetQueueAttributes" + effect = "Allow" + + actions = [ + "sqs:GetQueueAttributes", + ] + + resources = [ + module.sqs_inbound_event.sqs_queue_arn, + module.sqs_inbound_event.sqs_dlq_arn, + "arn:aws:sqs:${var.region}:${var.aws_account_id}:${local.client_csi}-*-delivery-queue", + "arn:aws:sqs:${var.region}:${var.aws_account_id}:${local.client_csi}-*-delivery-dlq-queue", + ] + } + + statement { + sid = "CloudWatchLogsInsightsQuery" + effect = "Allow" + + actions = [ + "logs:StartQuery", + "logs:StopQuery", + ] + + resources = concat( + [ + "arn:aws:logs:${var.region}:${var.aws_account_id}:log-group:${module.client_transform_filter_lambda.cloudwatch_log_group_name}:*", + "arn:aws:logs:${var.region}:${var.aws_account_id}:log-group:/aws/lambda/${local.client_csi}-https-client-*", + ], + var.deploy_mock_clients ? [ + "arn:aws:logs:${var.region}:${var.aws_account_id}:log-group:${module.mock_webhook_lambda[0].cloudwatch_log_group_name}:*", + ] : [], + ) + } + + statement { + sid = "CloudWatchLogsInsightsResults" + effect = "Allow" + + actions = [ + "logs:GetQueryResults", + ] + + resources = ["*"] + } + + statement { + sid = "ElastiCacheConnect" + effect = "Allow" + + actions = [ + "elasticache:Connect", + ] + + resources = [ + aws_elasticache_serverless_cache.delivery_state.arn, + aws_elasticache_user.delivery_state_iam.arn, + ] + } + + statement { + sid = "VPCNetworkInterfacePermissions" + effect = "Allow" + + actions = [ + "ec2:CreateNetworkInterface", + "ec2:DeleteNetworkInterface", + "ec2:DescribeNetworkInterfaces", + ] + + resources = [ + "*", + ] + } +} diff --git a/infrastructure/terraform/components/callbacks/module_sqs_inbound_event.tf b/infrastructure/terraform/components/callbacks/module_sqs_inbound_event.tf index 2e3080fe..2a15e357 100644 --- a/infrastructure/terraform/components/callbacks/module_sqs_inbound_event.tf +++ b/infrastructure/terraform/components/callbacks/module_sqs_inbound_event.tf @@ -2,7 +2,7 @@ module "sqs_inbound_event" { source = "https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/3.0.7/terraform-sqs.zip" aws_account_id = var.aws_account_id - component = var.component + component = local.component environment = var.environment project = var.project region = var.region @@ -33,7 +33,7 @@ data "aws_iam_policy_document" "sqs_inbound_event" { ] resources = [ - "arn:aws:sqs:${var.region}:${var.aws_account_id}:${var.project}-${var.environment}-${var.component}-inbound-event-queue" + "arn:aws:sqs:${var.region}:${var.aws_account_id}:${var.project}-${var.environment}-${local.component}-inbound-event-queue" ] condition { diff --git a/infrastructure/terraform/components/callbacks/module_transform_filter_lambda.tf b/infrastructure/terraform/components/callbacks/module_transform_filter_lambda.tf index fb1313f8..e6aeb5f6 100644 --- a/infrastructure/terraform/components/callbacks/module_transform_filter_lambda.tf +++ b/infrastructure/terraform/components/callbacks/module_transform_filter_lambda.tf @@ -5,7 +5,7 @@ module "client_transform_filter_lambda" { description = "Lambda function that transforms and filters events coming to through the eventpipe" aws_account_id = var.aws_account_id - component = var.component + component = local.component environment = var.environment project = var.project region = var.region @@ -37,12 +37,11 @@ module "client_transform_filter_lambda" { lambda_env_vars = { ENVIRONMENT = var.environment - METRICS_NAMESPACE = "nhs-notify-client-callbacks" - CLIENT_SUBSCRIPTION_CONFIG_BUCKET = module.client_config_bucket.bucket - CLIENT_SUBSCRIPTION_CONFIG_PREFIX = "client_subscriptions/" + METRICS_NAMESPACE = "nhs-notify-cb" + CLIENT_SUBSCRIPTION_CONFIG_BUCKET = local.client_config_s3_bucket + CLIENT_SUBSCRIPTION_CONFIG_PREFIX = "${var.environment}/client_subscriptions/" CLIENT_SUBSCRIPTION_CACHE_TTL_SECONDS = "60" MESSAGE_ROOT_URI = var.message_root_uri - APPLICATIONS_MAP_PARAMETER = local.applications_map_parameter_name } } @@ -70,7 +69,7 @@ data "aws_iam_policy_document" "client_transform_filter_lambda" { ] resources = [ - module.client_config_bucket.arn, + local.client_config_bucket_arn, ] } @@ -83,20 +82,7 @@ data "aws_iam_policy_document" "client_transform_filter_lambda" { ] resources = [ - "${module.client_config_bucket.arn}/*", - ] - } - - statement { - sid = "SSMApplicationsMapRead" - effect = "Allow" - - actions = [ - "ssm:GetParameter", - ] - - resources = [ - "arn:aws:ssm:${var.region}:${var.aws_account_id}:parameter${local.applications_map_parameter_name}", + "${local.client_config_bucket_arn}/*", ] } diff --git a/infrastructure/terraform/components/callbacks/pipes_pipe_main.tf b/infrastructure/terraform/components/callbacks/pipes_pipe_main.tf index 3fddfcca..ae914f4f 100644 --- a/infrastructure/terraform/components/callbacks/pipes_pipe_main.tf +++ b/infrastructure/terraform/components/callbacks/pipes_pipe_main.tf @@ -26,8 +26,7 @@ resource "aws_pipes_pipe" "main" { input_template = <, - "subscriptions": <$.subscriptions>, - "signatures": <$.signatures> + "subscriptions": <$.subscriptions> } EOF } diff --git a/infrastructure/terraform/components/callbacks/pre.sh b/infrastructure/terraform/components/callbacks/pre.sh index bba7c1b5..2fc13b77 100755 --- a/infrastructure/terraform/components/callbacks/pre.sh +++ b/infrastructure/terraform/components/callbacks/pre.sh @@ -6,17 +6,29 @@ script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # shellcheck source=_paths.sh source "${script_dir}/_paths.sh" -# Resolve deploy_mock_clients from tfvars; base_path/group/region/environment are in scope from terraform.sh +# Resolve tfvar overrides +tfvar_value() { + local key="$1" + local file="$2" + # Extract the value after '=', stripping surrounding whitespace and quotes + grep -E "^\s*${key}\s*=" "${file}" | tail -1 | sed 's/.*=\s*//;s/\s*$//;s/^"//;s/"$//' + return 0 +} + deploy_mock_clients="false" +deploy_perf_runner="false" for _tfvar_file in \ "${base_path}/etc/group_${group}.tfvars" \ "${base_path}/etc/env_${region}_${environment}.tfvars"; do if [[ -f "${_tfvar_file}" ]]; then - _val=$(grep -E '^\s*deploy_mock_clients\s*=' "${_tfvar_file}" | tail -1 | sed 's/.*=\s*//;s/\s*$//') + _val=$(tfvar_value deploy_mock_clients "${_tfvar_file}") [[ -n "${_val}" ]] && deploy_mock_clients="${_val}" + _val=$(tfvar_value deploy_perf_runner "${_tfvar_file}") + [[ -n "${_val}" ]] && deploy_perf_runner="${_val}" fi done echo "deploy_mock_clients resolved to: ${deploy_mock_clients}" +echo "deploy_perf_runner resolved to: ${deploy_perf_runner}" pnpm install --frozen-lockfile @@ -25,15 +37,13 @@ pnpm run generate-dependencies "${script_dir}/sync-client-config.sh" if [[ "${deploy_mock_clients}" == "true" ]]; then - shopt -s nullglob - existing_configs=("${clients_dir}"/*.json) - shopt -u nullglob - if [[ "${#existing_configs[@]}" -eq 0 ]]; then - cp "${bounded_context_root}/tests/integration/fixtures/subscriptions/"*.json "${clients_dir}/" - echo "Copied mock client subscription config fixtures into clients dir" - else - echo "Client configs already present from S3 sync; skipping fixture copy" - fi + cp "${bounded_context_root}/tests/integration/fixtures/subscriptions/"*.json "${clients_dir}/" + echo "Copied mock client subscription config fixtures into clients dir" +fi + +if [[ "${deploy_perf_runner}" == "true" ]]; then + cp "${bounded_context_root}/tests/performance/fixtures/subscriptions/"*.json "${clients_dir}/" + echo "Copied perf client subscription config fixtures into clients dir" fi pnpm run --recursive --if-present lambda-build diff --git a/infrastructure/terraform/components/callbacks/random_password_elasticache_default_user.tf b/infrastructure/terraform/components/callbacks/random_password_elasticache_default_user.tf new file mode 100644 index 00000000..109a3f74 --- /dev/null +++ b/infrastructure/terraform/components/callbacks/random_password_elasticache_default_user.tf @@ -0,0 +1,4 @@ +resource "random_password" "elasticache_default_user" { + length = 32 + special = false +} diff --git a/infrastructure/terraform/components/callbacks/s3_bucket_client_config.tf b/infrastructure/terraform/components/callbacks/s3_bucket_client_config.tf deleted file mode 100644 index 8bf25c83..00000000 --- a/infrastructure/terraform/components/callbacks/s3_bucket_client_config.tf +++ /dev/null @@ -1,104 +0,0 @@ -resource "aws_s3_object" "mock_client_config" { - for_each = var.deploy_mock_clients ? toset(keys(local.config_clients)) : toset([]) - - bucket = module.client_config_bucket.id - key = "client_subscriptions/${local.config_clients[each.key].clientId}.json" - content = jsonencode(local.enriched_mock_config_clients[each.key]) - - kms_key_id = module.kms.key_arn - server_side_encryption = "aws:kms" - - content_type = "application/json" -} - -module "client_config_bucket" { - source = "https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/3.0.7/terraform-s3bucket.zip" - - name = "subscription-config" - - aws_account_id = var.aws_account_id - component = var.component - environment = var.environment - project = var.project - region = var.region - - default_tags = merge( - local.default_tags, - { - Description = "Client subscription configuration storage" - } - ) - - kms_key_arn = module.kms.key_arn - force_destroy = var.s3_enable_force_destroy - versioning = true - object_ownership = "BucketOwnerPreferred" - bucket_key_enabled = true - - policy_documents = [ - data.aws_iam_policy_document.client_config_bucket.json - ] -} - -data "aws_iam_policy_document" "client_config_bucket" { - statement { - sid = "AllowLambdaListAccess" - effect = "Allow" - - principals { - type = "AWS" - identifiers = [module.client_transform_filter_lambda.iam_role_arn] - } - - actions = [ - "s3:ListBucket", - ] - - resources = [ - module.client_config_bucket.arn, - ] - } - - statement { - sid = "AllowLambdaReadAccess" - effect = "Allow" - - principals { - type = "AWS" - identifiers = [module.client_transform_filter_lambda.iam_role_arn] - } - - actions = [ - "s3:GetObject", - ] - - resources = [ - "${module.client_config_bucket.arn}/*", - ] - } - - statement { - sid = "DenyInsecureTransport" - effect = "Deny" - - principals { - type = "*" - identifiers = ["*"] - } - - actions = [ - "s3:*", - ] - - resources = [ - module.client_config_bucket.arn, - "${module.client_config_bucket.arn}/*" - ] - - condition { - test = "Bool" - variable = "aws:SecureTransport" - values = ["false"] - } - } -} diff --git a/infrastructure/terraform/components/callbacks/s3_object_applications_map.tf b/infrastructure/terraform/components/callbacks/s3_object_applications_map.tf new file mode 100644 index 00000000..9911ede3 --- /dev/null +++ b/infrastructure/terraform/components/callbacks/s3_object_applications_map.tf @@ -0,0 +1,13 @@ +resource "aws_s3_object" "applications_map" { + bucket = local.applications_map_s3_bucket + key = local.applications_map_s3_key + content = jsonencode(var.deploy_mock_clients ? { for client_id, client in local.config_clients : client_id => "${client_id}-app-id" } : {}) + content_type = "application/json" + kms_key_id = module.kms.key_arn + + server_side_encryption = "aws:kms" + + lifecycle { + ignore_changes = [content] + } +} diff --git a/infrastructure/terraform/components/callbacks/s3_object_client_config.tf b/infrastructure/terraform/components/callbacks/s3_object_client_config.tf new file mode 100644 index 00000000..1f0a6c74 --- /dev/null +++ b/infrastructure/terraform/components/callbacks/s3_object_client_config.tf @@ -0,0 +1,12 @@ +resource "aws_s3_object" "mock_client_config" { + for_each = var.deploy_mock_clients ? toset(keys(local.config_clients)) : toset([]) + + bucket = local.client_config_s3_bucket + key = "${var.environment}/client_subscriptions/${local.config_clients[each.key].clientId}.json" + content = jsonencode(local.enriched_mock_config_clients[each.key]) + + kms_key_id = module.kms.key_arn + server_side_encryption = "aws:kms" + + content_type = "application/json" +} diff --git a/infrastructure/terraform/components/callbacks/s3_object_mtls_test_ca.tf b/infrastructure/terraform/components/callbacks/s3_object_mtls_test_ca.tf new file mode 100644 index 00000000..46452b2b --- /dev/null +++ b/infrastructure/terraform/components/callbacks/s3_object_mtls_test_ca.tf @@ -0,0 +1,10 @@ +resource "aws_s3_object" "mtls_test_ca" { + count = var.deploy_mock_clients ? 1 : 0 + bucket = local.mtls_cert_s3_bucket + key = local.mtls_test_ca_s3_key # gitleaks:allow + content = tls_self_signed_cert.test_ca[0].cert_pem + + kms_key_id = module.kms.key_arn + server_side_encryption = "aws:kms" + content_type = "application/x-pem-file" +} diff --git a/infrastructure/terraform/components/callbacks/s3_object_mtls_test_client_bundle.tf b/infrastructure/terraform/components/callbacks/s3_object_mtls_test_client_bundle.tf new file mode 100644 index 00000000..fd491c2f --- /dev/null +++ b/infrastructure/terraform/components/callbacks/s3_object_mtls_test_client_bundle.tf @@ -0,0 +1,10 @@ +resource "aws_s3_object" "mtls_test_client_bundle" { + count = var.deploy_mock_clients ? 1 : 0 + bucket = local.mtls_cert_s3_bucket + key = local.mtls_test_cert_s3_key # gitleaks:allow + content = "${tls_locally_signed_cert.test_client[0].cert_pem}${tls_private_key.test_client[0].private_key_pem}" + + kms_key_id = module.kms.key_arn + server_side_encryption = "aws:kms" + content_type = "application/x-pem-file" +} diff --git a/infrastructure/terraform/components/callbacks/security_group_elasticache_delivery_state.tf b/infrastructure/terraform/components/callbacks/security_group_elasticache_delivery_state.tf new file mode 100644 index 00000000..08bed00b --- /dev/null +++ b/infrastructure/terraform/components/callbacks/security_group_elasticache_delivery_state.tf @@ -0,0 +1,12 @@ +resource "aws_security_group" "elasticache_delivery_state" { + name = "${local.csi}-elasticache-delivery-state" + description = "Security group for ElastiCache delivery state cluster" + vpc_id = local.acct.vpc_ids[local.bc_name] + + tags = merge( + local.default_tags, + { + Name = "${local.csi}-elasticache-delivery-state" + }, + ) +} diff --git a/infrastructure/terraform/components/callbacks/security_group_https_client_lambda.tf b/infrastructure/terraform/components/callbacks/security_group_https_client_lambda.tf new file mode 100644 index 00000000..3c187dfa --- /dev/null +++ b/infrastructure/terraform/components/callbacks/security_group_https_client_lambda.tf @@ -0,0 +1,12 @@ +resource "aws_security_group" "https_client_lambda" { + name = "${local.csi}-https-client-lambda" + description = "Security group for per-client HTTPS Client Lambda functions" + vpc_id = local.acct.vpc_ids[local.bc_name] + + tags = merge( + local.default_tags, + { + Name = "${local.csi}-https-client-lambda" + }, + ) +} diff --git a/infrastructure/terraform/components/callbacks/ssm_parameter_applications_map.tf b/infrastructure/terraform/components/callbacks/ssm_parameter_applications_map.tf deleted file mode 100644 index 567647d1..00000000 --- a/infrastructure/terraform/components/callbacks/ssm_parameter_applications_map.tf +++ /dev/null @@ -1,19 +0,0 @@ -resource "random_password" "mock_application_id" { - for_each = var.deploy_mock_clients ? toset(keys(local.config_clients)) : toset([]) - length = 24 - special = false -} - -resource "aws_ssm_parameter" "applications_map" { - name = local.applications_map_parameter_name - type = "SecureString" - key_id = module.kms.key_arn - - value = var.deploy_mock_clients ? jsonencode({ - for id in keys(local.config_clients) : local.config_clients[id].clientId => random_password.mock_application_id[id].result - }) : jsonencode({}) - - lifecycle { - ignore_changes = [value] - } -} diff --git a/infrastructure/terraform/components/callbacks/sync-client-config.sh b/infrastructure/terraform/components/callbacks/sync-client-config.sh index 2c2a3ecb..9226cf01 100755 --- a/infrastructure/terraform/components/callbacks/sync-client-config.sh +++ b/infrastructure/terraform/components/callbacks/sync-client-config.sh @@ -18,9 +18,10 @@ cd "${repo_root}" rm -f "${clients_dir}"/*.json -bucket_name="nhs-${AWS_ACCOUNT_ID}-${AWS_REGION}-${ENVIRONMENT}-callbacks-subscription-config" -s3_prefix="client_subscriptions/" +bucket_name="nhs-${AWS_ACCOUNT_ID}-${AWS_REGION}-main-acct-clie-client-configs" + +s3_prefix="${ENVIRONMENT}/client_subscriptions/" echo "Seeding client configs from s3://${bucket_name}/${s3_prefix} for ${ENVIRONMENT}/${AWS_REGION}" diff --git a/infrastructure/terraform/components/callbacks/tls_cert_request_mock_server.tf b/infrastructure/terraform/components/callbacks/tls_cert_request_mock_server.tf new file mode 100644 index 00000000..a577a4c8 --- /dev/null +++ b/infrastructure/terraform/components/callbacks/tls_cert_request_mock_server.tf @@ -0,0 +1,12 @@ +resource "tls_cert_request" "mock_server" { + count = var.deploy_mock_clients ? 1 : 0 + private_key_pem = tls_private_key.mock_server[0].private_key_pem + + subject { + common_name = "NHS Notify Mock Webhook Server" + organization = "NHS Notify" + country = "GB" + } + + dns_names = ["*.eu-west-2.elb.amazonaws.com"] +} diff --git a/infrastructure/terraform/components/callbacks/tls_cert_request_test_client.tf b/infrastructure/terraform/components/callbacks/tls_cert_request_test_client.tf new file mode 100644 index 00000000..2b3eb4bc --- /dev/null +++ b/infrastructure/terraform/components/callbacks/tls_cert_request_test_client.tf @@ -0,0 +1,10 @@ +resource "tls_cert_request" "test_client" { + count = var.deploy_mock_clients ? 1 : 0 + private_key_pem = tls_private_key.test_client[0].private_key_pem + + subject { + common_name = "NHS Notify Callbacks Test Client" + organization = "NHS Notify" + country = "GB" + } +} diff --git a/infrastructure/terraform/components/callbacks/tls_locally_signed_cert_mock_server.tf b/infrastructure/terraform/components/callbacks/tls_locally_signed_cert_mock_server.tf new file mode 100644 index 00000000..23c36c9e --- /dev/null +++ b/infrastructure/terraform/components/callbacks/tls_locally_signed_cert_mock_server.tf @@ -0,0 +1,13 @@ +resource "tls_locally_signed_cert" "mock_server" { + count = var.deploy_mock_clients ? 1 : 0 + cert_request_pem = tls_cert_request.mock_server[0].cert_request_pem + ca_private_key_pem = tls_private_key.test_ca[0].private_key_pem + ca_cert_pem = tls_self_signed_cert.test_ca[0].cert_pem + validity_period_hours = 87600 + + allowed_uses = [ + "digital_signature", + "key_encipherment", + "server_auth", + ] +} diff --git a/infrastructure/terraform/components/callbacks/tls_locally_signed_cert_test_client.tf b/infrastructure/terraform/components/callbacks/tls_locally_signed_cert_test_client.tf new file mode 100644 index 00000000..866dbf7d --- /dev/null +++ b/infrastructure/terraform/components/callbacks/tls_locally_signed_cert_test_client.tf @@ -0,0 +1,12 @@ +resource "tls_locally_signed_cert" "test_client" { + count = var.deploy_mock_clients ? 1 : 0 + cert_request_pem = tls_cert_request.test_client[0].cert_request_pem + ca_private_key_pem = tls_private_key.test_ca[0].private_key_pem + ca_cert_pem = tls_self_signed_cert.test_ca[0].cert_pem + validity_period_hours = 87600 + + allowed_uses = [ + "digital_signature", + "client_auth", + ] +} diff --git a/infrastructure/terraform/components/callbacks/tls_private_key_mock_server.tf b/infrastructure/terraform/components/callbacks/tls_private_key_mock_server.tf new file mode 100644 index 00000000..188e9a94 --- /dev/null +++ b/infrastructure/terraform/components/callbacks/tls_private_key_mock_server.tf @@ -0,0 +1,5 @@ +resource "tls_private_key" "mock_server" { + count = var.deploy_mock_clients ? 1 : 0 + algorithm = "ECDSA" + ecdsa_curve = "P256" +} diff --git a/infrastructure/terraform/components/callbacks/tls_private_key_test_ca.tf b/infrastructure/terraform/components/callbacks/tls_private_key_test_ca.tf new file mode 100644 index 00000000..bcef9a8e --- /dev/null +++ b/infrastructure/terraform/components/callbacks/tls_private_key_test_ca.tf @@ -0,0 +1,5 @@ +resource "tls_private_key" "test_ca" { + count = var.deploy_mock_clients ? 1 : 0 + algorithm = "ECDSA" + ecdsa_curve = "P256" +} diff --git a/infrastructure/terraform/components/callbacks/tls_private_key_test_client.tf b/infrastructure/terraform/components/callbacks/tls_private_key_test_client.tf new file mode 100644 index 00000000..881fabac --- /dev/null +++ b/infrastructure/terraform/components/callbacks/tls_private_key_test_client.tf @@ -0,0 +1,5 @@ +resource "tls_private_key" "test_client" { + count = var.deploy_mock_clients ? 1 : 0 + algorithm = "ECDSA" + ecdsa_curve = "P256" +} diff --git a/infrastructure/terraform/components/callbacks/tls_self_signed_cert_test_ca.tf b/infrastructure/terraform/components/callbacks/tls_self_signed_cert_test_ca.tf new file mode 100644 index 00000000..af59b336 --- /dev/null +++ b/infrastructure/terraform/components/callbacks/tls_self_signed_cert_test_ca.tf @@ -0,0 +1,16 @@ +resource "tls_self_signed_cert" "test_ca" { + count = var.deploy_mock_clients ? 1 : 0 + private_key_pem = tls_private_key.test_ca[0].private_key_pem + is_ca_certificate = true + validity_period_hours = 87600 + + subject { + common_name = "NHS Notify Test CA" + organization = "NHS Notify" + country = "GB" + } + + allowed_uses = [ + "cert_signing", + ] +} diff --git a/infrastructure/terraform/components/callbacks/variables.tf b/infrastructure/terraform/components/callbacks/variables.tf index 74a72d24..2dbb9efb 100644 --- a/infrastructure/terraform/components/callbacks/variables.tf +++ b/infrastructure/terraform/components/callbacks/variables.tf @@ -102,7 +102,7 @@ variable "pipe_log_level" { variable "pipe_sqs_input_batch_size" { type = number - default = 1 + default = 10 } variable "pipe_sqs_max_batch_window" { @@ -155,6 +155,12 @@ variable "deploy_mock_clients" { default = false } +variable "deploy_perf_runner" { + type = bool + description = "Flag to deploy the perf-runner lambda for performance testing (test/dev environments only)" + default = false +} + variable "enable_xray_tracing" { type = bool description = "Enable AWS X-Ray active tracing for Lambda functions" @@ -166,14 +172,50 @@ variable "message_root_uri" { description = "The root URI used for constructing message links in callback payloads" } -variable "applications_map_parameter_name" { +variable "mtls_cert_s3_key" { type = string - default = null - description = "SSM Parameter Store path for the clientId-to-applicationData map, where applicationData is currently only the applicationId" + description = "S3 key for the mTLS client certificate PEM bundle" + default = "" } -variable "s3_enable_force_destroy" { - type = bool - description = "Whether to enable force destroy for the S3 buckets created in this module" - default = false +variable "mtls_ca_s3_key" { + type = string + description = "S3 key for the CA certificate PEM bundle used for server verification" + default = "" +} + +variable "elasticache_data_storage_maximum_gb" { + type = number + description = "Maximum data storage in GB for the ElastiCache Serverless delivery state cache" + default = 1 +} + +variable "token_bucket_burst_capacity" { + type = number + description = "Token bucket burst capacity used by the rate limiter" + default = 2250 +} + +variable "cb_cooldown_period_ms" { + type = number + description = "Full block duration after circuit opens, before half-open probes begin (ms)" + default = 120000 +} + +variable "cb_recovery_period_ms" { + type = number + description = "Linear ramp-up duration after circuit closes (ms)" + default = 600000 +} + +variable "delivery_lambda_batch_size" { + type = number + description = "Number of SQS messages per delivery Lambda invocation" + default = 100 +} + +variable "delivery_lambda_batching_window_sec" { + type = number + description = "Maximum time in seconds to wait for a full batch before invoking the delivery Lambda" + default = 1 } diff --git a/infrastructure/terraform/components/callbacks/versions.tf b/infrastructure/terraform/components/callbacks/versions.tf index 55552749..d91998a2 100644 --- a/infrastructure/terraform/components/callbacks/versions.tf +++ b/infrastructure/terraform/components/callbacks/versions.tf @@ -4,10 +4,18 @@ terraform { source = "hashicorp/aws" version = "6.13" } + external = { + source = "hashicorp/external" + version = "~> 2.0" + } random = { source = "hashicorp/random" version = "~> 3.0" } + tls = { + source = "hashicorp/tls" + version = "~> 4.0" + } } required_version = ">= 1.10.1" diff --git a/infrastructure/terraform/components/callbacks/vpc_security_group_egress_rule_lambda_to_elasticache.tf b/infrastructure/terraform/components/callbacks/vpc_security_group_egress_rule_lambda_to_elasticache.tf new file mode 100644 index 00000000..6e78bd17 --- /dev/null +++ b/infrastructure/terraform/components/callbacks/vpc_security_group_egress_rule_lambda_to_elasticache.tf @@ -0,0 +1,10 @@ +resource "aws_vpc_security_group_egress_rule" "lambda_to_elasticache" { + security_group_id = aws_security_group.https_client_lambda.id + referenced_security_group_id = aws_security_group.elasticache_delivery_state.id + from_port = 6379 + to_port = 6379 + ip_protocol = "tcp" + description = "Allow Lambda to connect to ElastiCache" + + tags = local.default_tags +} diff --git a/infrastructure/terraform/components/callbacks/vpc_security_group_egress_rule_lambda_to_https.tf b/infrastructure/terraform/components/callbacks/vpc_security_group_egress_rule_lambda_to_https.tf new file mode 100644 index 00000000..f4084256 --- /dev/null +++ b/infrastructure/terraform/components/callbacks/vpc_security_group_egress_rule_lambda_to_https.tf @@ -0,0 +1,10 @@ +resource "aws_vpc_security_group_egress_rule" "lambda_to_https" { + security_group_id = aws_security_group.https_client_lambda.id + cidr_ipv4 = "0.0.0.0/0" + from_port = 443 + to_port = 443 + ip_protocol = "tcp" + description = "Allow Lambda outbound HTTPS for webhook delivery" + + tags = local.default_tags +} diff --git a/infrastructure/terraform/components/callbacks/vpc_security_group_ingress_rule_elasticache_from_lambda.tf b/infrastructure/terraform/components/callbacks/vpc_security_group_ingress_rule_elasticache_from_lambda.tf new file mode 100644 index 00000000..db65c614 --- /dev/null +++ b/infrastructure/terraform/components/callbacks/vpc_security_group_ingress_rule_elasticache_from_lambda.tf @@ -0,0 +1,10 @@ +resource "aws_vpc_security_group_ingress_rule" "elasticache_from_lambda" { + security_group_id = aws_security_group.elasticache_delivery_state.id + referenced_security_group_id = aws_security_group.https_client_lambda.id + from_port = 6379 + to_port = 6379 + ip_protocol = "tcp" + description = "Allow HTTPS Client Lambda to connect to ElastiCache" + + tags = local.default_tags +} diff --git a/infrastructure/terraform/modules/client-delivery/README.md b/infrastructure/terraform/modules/client-delivery/README.md new file mode 100644 index 00000000..6acbeb80 --- /dev/null +++ b/infrastructure/terraform/modules/client-delivery/README.md @@ -0,0 +1,67 @@ + + + + +## Requirements + +No requirements. +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [applications\_map\_s3\_bucket](#input\_applications\_map\_s3\_bucket) | S3 bucket containing the applications map JSON | `string` | n/a | yes | +| [applications\_map\_s3\_key](#input\_applications\_map\_s3\_key) | S3 key for the applications map JSON file | `string` | n/a | yes | +| [aws\_account\_id](#input\_aws\_account\_id) | Account ID | `string` | n/a | yes | +| [cb\_cooldown\_period\_ms](#input\_cb\_cooldown\_period\_ms) | Full block duration after circuit opens, before half-open probes begin (ms) | `number` | `120000` | no | +| [cb\_recovery\_period\_ms](#input\_cb\_recovery\_period\_ms) | Linear ramp-up duration after circuit closes (ms) | `number` | `600000` | no | +| [client\_bus\_name](#input\_client\_bus\_name) | EventBridge bus name for subscription rules | `string` | n/a | yes | +| [client\_config\_bucket](#input\_client\_config\_bucket) | S3 bucket name containing client subscription configuration | `string` | n/a | yes | +| [client\_config\_bucket\_arn](#input\_client\_config\_bucket\_arn) | S3 bucket ARN containing client subscription configuration | `string` | n/a | yes | +| [client\_config\_key\_prefix](#input\_client\_config\_key\_prefix) | S3 key prefix for client subscription configuration files | `string` | `"client_subscriptions/"` | no | +| [client\_id](#input\_client\_id) | Unique identifier for this client | `string` | n/a | yes | +| [component](#input\_component) | Component name | `string` | n/a | yes | +| [delivery\_lambda\_batch\_size](#input\_delivery\_lambda\_batch\_size) | Number of SQS messages per Lambda invocation | `number` | `100` | no | +| [delivery\_lambda\_batching\_window\_sec](#input\_delivery\_lambda\_batching\_window\_sec) | Maximum time in seconds to wait for a full batch before invoking Lambda. Allows the delivery queue to fill to batch\_size, improving Lambda concurrency utilisation. | `number` | `1` | no | +| [delivery\_lambda\_code\_base\_path](#input\_delivery\_lambda\_code\_base\_path) | Base path to Lambda source code directories | `string` | n/a | yes | +| [delivery\_lambda\_max\_concurrency](#input\_delivery\_lambda\_max\_concurrency) | Maximum concurrent Lambda invocations for the SQS event source mapping | `number` | `200` | no | +| [delivery\_lambda\_memory](#input\_delivery\_lambda\_memory) | Lambda memory allocation in MB | `number` | `256` | no | +| [delivery\_lambda\_s3\_bucket](#input\_delivery\_lambda\_s3\_bucket) | S3 bucket for Lambda function artefacts | `string` | n/a | yes | +| [delivery\_lambda\_security\_group\_id](#input\_delivery\_lambda\_security\_group\_id) | Security group ID for the Lambda function | `string` | `""` | no | +| [delivery\_lambda\_timeout](#input\_delivery\_lambda\_timeout) | Lambda timeout in seconds | `number` | `30` | no | +| [elasticache\_cache\_name](#input\_elasticache\_cache\_name) | ElastiCache cache name for SigV4 token presigning | `string` | `""` | no | +| [elasticache\_endpoint](#input\_elasticache\_endpoint) | ElastiCache Serverless endpoint URL | `string` | `""` | no | +| [elasticache\_iam\_username](#input\_elasticache\_iam\_username) | IAM username for ElastiCache authentication | `string` | `""` | no | +| [enable\_xray\_tracing](#input\_enable\_xray\_tracing) | Enable AWS X-Ray active tracing for the Lambda function | `bool` | `false` | no | +| [environment](#input\_environment) | The name of the tfscaffold environment | `string` | n/a | yes | +| [force\_lambda\_code\_deploy](#input\_force\_lambda\_code\_deploy) | Force Lambda code redeployment even when commit tag matches | `bool` | `false` | no | +| [group](#input\_group) | The name of the tfscaffold group | `string` | `null` | no | +| [kms\_key\_arn](#input\_kms\_key\_arn) | KMS Key ARN for encryption at rest | `string` | n/a | yes | +| [log\_destination\_arn](#input\_log\_destination\_arn) | Firehose destination ARN for log forwarding | `string` | `""` | no | +| [log\_level](#input\_log\_level) | Log level for the Lambda function | `string` | `"INFO"` | no | +| [log\_retention\_in\_days](#input\_log\_retention\_in\_days) | CloudWatch log retention period in days | `number` | `0` | no | +| [log\_subscription\_role\_arn](#input\_log\_subscription\_role\_arn) | IAM role ARN for CloudWatch log subscription | `string` | `""` | no | +| [max\_retry\_duration\_seconds](#input\_max\_retry\_duration\_seconds) | Maximum retry window before messages are sent to DLQ | `number` | `7200` | no | +| [mtls\_ca\_s3\_key](#input\_mtls\_ca\_s3\_key) | S3 key for the CA certificate PEM bundle used for server verification | `string` | `""` | no | +| [mtls\_cert\_s3\_bucket](#input\_mtls\_cert\_s3\_bucket) | S3 bucket containing the mTLS client certificate bundle | `string` | `""` | no | +| [mtls\_cert\_s3\_key](#input\_mtls\_cert\_s3\_key) | S3 key for the mTLS client certificate PEM bundle | `string` | `""` | no | +| [project](#input\_project) | The name of the tfscaffold project | `string` | n/a | yes | +| [region](#input\_region) | AWS Region | `string` | n/a | yes | +| [sqs\_max\_receive\_count](#input\_sqs\_max\_receive\_count) | Safety-net maximum receive count before a message moves to DLQ. Supplements the time-based retry window for cases where the Lambda fails before reaching the window check. | `number` | `100` | no | +| [sqs\_visibility\_timeout\_seconds](#input\_sqs\_visibility\_timeout\_seconds) | Visibility timeout for the per-client delivery queue | `number` | `60` | no | +| [subscription\_targets](#input\_subscription\_targets) | Flattened subscription-target fanout map keyed by subscription-target composite key |
map(object({
subscription_id = string
target_id = string
}))
| n/a | yes | +| [subscriptions](#input\_subscriptions) | Subscription definitions for this client, keyed by subscription\_id |
map(object({
subscription_id = string
target_ids = list(string)
}))
| n/a | yes | +| [token\_bucket\_burst\_capacity](#input\_token\_bucket\_burst\_capacity) | Token bucket burst capacity used by the rate limiter | `number` | `2250` | no | +| [vpc\_subnet\_ids](#input\_vpc\_subnet\_ids) | VPC subnet IDs for Lambda execution | `list(string)` | `[]` | no | +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [dlq\_delivery](#module\_dlq\_delivery) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/3.0.7/terraform-sqs.zip | n/a | +| [https\_client\_lambda](#module\_https\_client\_lambda) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/3.0.7/terraform-lambda.zip | n/a | +| [sqs\_delivery](#module\_sqs\_delivery) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/3.0.9/terraform-sqs.zip | n/a | +## Outputs + +No outputs. + + + diff --git a/infrastructure/terraform/modules/client-delivery/cloudwatch_event_rule_per_subscription.tf b/infrastructure/terraform/modules/client-delivery/cloudwatch_event_rule_per_subscription.tf new file mode 100644 index 00000000..88303431 --- /dev/null +++ b/infrastructure/terraform/modules/client-delivery/cloudwatch_event_rule_per_subscription.tf @@ -0,0 +1,17 @@ +resource "aws_cloudwatch_event_rule" "per_subscription" { + for_each = var.subscriptions + + name = "${local.csi}-${each.key}" + description = "Client Callbacks event rule for client ${var.client_id} subscription ${each.key}" + event_bus_name = var.client_bus_name + + event_pattern = jsonencode({ + "detail" : { + "subscriptions" : [each.value.subscription_id] + } + }) + + tags = merge(local.default_tags, { + SubscriptionId = each.value.subscription_id + }) +} diff --git a/infrastructure/terraform/modules/client-delivery/cloudwatch_event_target_per_subscription.tf b/infrastructure/terraform/modules/client-delivery/cloudwatch_event_target_per_subscription.tf new file mode 100644 index 00000000..84a62e0a --- /dev/null +++ b/infrastructure/terraform/modules/client-delivery/cloudwatch_event_target_per_subscription.tf @@ -0,0 +1,30 @@ +resource "aws_cloudwatch_event_target" "per_subscription_target" { + for_each = var.subscription_targets + + rule = aws_cloudwatch_event_rule.per_subscription[each.value.subscription_id].name + arn = module.sqs_delivery.sqs_queue_arn + target_id = "${local.csi}-${each.value.target_id}" + event_bus_name = var.client_bus_name + role_arn = aws_iam_role.eventbridge_sqs_target.arn + + sqs_target { + message_group_id = null + } + + input_transformer { + input_paths = { + payload = "$.detail.payload" + } + + input_template = "{\"payload\": , \"subscriptionId\": \"${each.value.subscription_id}\", \"targetId\": \"${each.value.target_id}\"}" + } + + dead_letter_config { + arn = module.dlq_delivery.sqs_queue_arn + } + + retry_policy { + maximum_retry_attempts = 0 + maximum_event_age_in_seconds = 60 + } +} diff --git a/infrastructure/terraform/modules/client-delivery/iam_role_eventbridge_sqs_target.tf b/infrastructure/terraform/modules/client-delivery/iam_role_eventbridge_sqs_target.tf new file mode 100644 index 00000000..7703a39d --- /dev/null +++ b/infrastructure/terraform/modules/client-delivery/iam_role_eventbridge_sqs_target.tf @@ -0,0 +1,18 @@ +resource "aws_iam_role" "eventbridge_sqs_target" { + name = "${local.client_prefix}-eb-sqs-role" + description = "Role for EventBridge to send messages to per-client SQS queue" + assume_role_policy = data.aws_iam_policy_document.eventbridge_sqs_assume.json + + tags = local.default_tags +} + +data "aws_iam_policy_document" "eventbridge_sqs_assume" { + statement { + actions = ["sts:AssumeRole"] + + principals { + type = "Service" + identifiers = ["events.amazonaws.com"] + } + } +} diff --git a/infrastructure/terraform/modules/client-delivery/iam_role_https_client_lambda.tf b/infrastructure/terraform/modules/client-delivery/iam_role_https_client_lambda.tf new file mode 100644 index 00000000..f9abf363 --- /dev/null +++ b/infrastructure/terraform/modules/client-delivery/iam_role_https_client_lambda.tf @@ -0,0 +1,135 @@ +data "aws_iam_policy_document" "https_client_lambda" { + statement { + sid = "KMSPermissions" + effect = "Allow" + + actions = [ + "kms:Decrypt", + "kms:GenerateDataKey", + ] + + resources = [ + var.kms_key_arn, + ] + } + + statement { + sid = "SQSDeliveryQueueConsume" + effect = "Allow" + + actions = [ + "sqs:ReceiveMessage", + "sqs:DeleteMessage", + "sqs:GetQueueAttributes", + "sqs:ChangeMessageVisibility", + ] + + resources = [ + module.sqs_delivery.sqs_queue_arn, + ] + } + + statement { + sid = "SQSDLQSend" + effect = "Allow" + + actions = [ + "sqs:SendMessage", + ] + + resources = [ + module.dlq_delivery.sqs_queue_arn, + ] + } + + statement { + sid = "S3ApplicationsMapReadAccess" + effect = "Allow" + + actions = [ + "s3:GetObject", + ] + + resources = [ + "arn:aws:s3:::${var.applications_map_s3_bucket}/${var.applications_map_s3_key}", + ] + } + + statement { + sid = "S3ClientConfigReadAccess" + effect = "Allow" + + actions = [ + "s3:GetObject", + ] + + resources = [ + "${var.client_config_bucket_arn}/${var.client_config_key_prefix}*", + ] + } + + statement { + sid = "S3ClientConfigListAccess" + effect = "Allow" + + actions = [ + "s3:ListBucket", + ] + + resources = [ + var.client_config_bucket_arn, + ] + } + + dynamic "statement" { + for_each = var.delivery_lambda_security_group_id != "" ? [1] : [] + content { + sid = "VPCNetworkInterfacePermissions" + effect = "Allow" + + actions = [ + "ec2:CreateNetworkInterface", + "ec2:DeleteNetworkInterface", + "ec2:DescribeNetworkInterfaces", + ] + + resources = [ + "*", + ] + } + } + + dynamic "statement" { + for_each = var.mtls_cert_s3_bucket != "" ? [1] : [] + content { + sid = "S3MTLSCertReadAccess" + effect = "Allow" + + actions = [ + "s3:GetObject", + ] + + resources = [ + "arn:aws:s3:::${var.mtls_cert_s3_bucket}/${var.mtls_cert_s3_key}", + "arn:aws:s3:::${var.mtls_cert_s3_bucket}/${var.mtls_ca_s3_key}", + ] + } + } + + dynamic "statement" { + for_each = var.elasticache_endpoint != "" ? [1] : [] + content { + sid = "ElastiCacheConnect" + effect = "Allow" + + actions = [ + "elasticache:Connect", + ] + + resources = [ + "arn:aws:elasticache:${var.region}:${var.aws_account_id}:serverlesscache:${var.elasticache_cache_name}", + "arn:aws:elasticache:${var.region}:${var.aws_account_id}:user:${var.elasticache_iam_username}", + ] + } + } +} diff --git a/infrastructure/terraform/modules/client-delivery/iam_role_policy_eventbridge_sqs_send.tf b/infrastructure/terraform/modules/client-delivery/iam_role_policy_eventbridge_sqs_send.tf new file mode 100644 index 00000000..85a1e80f --- /dev/null +++ b/infrastructure/terraform/modules/client-delivery/iam_role_policy_eventbridge_sqs_send.tf @@ -0,0 +1,35 @@ +resource "aws_iam_role_policy" "eventbridge_sqs_send" { + name = "sqs-send" + role = aws_iam_role.eventbridge_sqs_target.id + policy = data.aws_iam_policy_document.eventbridge_sqs_send.json +} + +data "aws_iam_policy_document" "eventbridge_sqs_send" { + statement { + sid = "AllowSQSSendMessage" + effect = "Allow" + + actions = [ + "sqs:SendMessage", + ] + + resources = [ + module.sqs_delivery.sqs_queue_arn, + module.dlq_delivery.sqs_queue_arn, + ] + } + + statement { + sid = "AllowKMSForSQS" + effect = "Allow" + + actions = [ + "kms:Decrypt", + "kms:GenerateDataKey", + ] + + resources = [ + var.kms_key_arn, + ] + } +} diff --git a/infrastructure/terraform/modules/client-delivery/locals.tf b/infrastructure/terraform/modules/client-delivery/locals.tf new file mode 100644 index 00000000..6ca35137 --- /dev/null +++ b/infrastructure/terraform/modules/client-delivery/locals.tf @@ -0,0 +1,21 @@ +locals { + csi = replace( + format( + "%s-%s-%s", + var.project, + var.environment, + var.component, + ), + "_", + "", + ) + + client_prefix = "${local.csi}-${var.client_id}" + + default_tags = { + Project = var.project + Environment = var.environment + Component = var.component + Client = var.client_id + } +} diff --git a/infrastructure/terraform/modules/client-delivery/module_dlq_per_client.tf b/infrastructure/terraform/modules/client-delivery/module_dlq_per_client.tf new file mode 100644 index 00000000..84c410dd --- /dev/null +++ b/infrastructure/terraform/modules/client-delivery/module_dlq_per_client.tf @@ -0,0 +1,43 @@ +module "dlq_delivery" { + source = "https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/3.0.7/terraform-sqs.zip" + + aws_account_id = var.aws_account_id + component = var.component + environment = var.environment + project = var.project + region = var.region + name = "${var.client_id}-delivery-dlq" + + sqs_kms_key_arn = var.kms_key_arn + + create_dlq = false +} + +resource "aws_cloudwatch_metric_alarm" "dlq_depth" { + alarm_name = "${local.client_prefix}-dlq-depth" + alarm_description = join(" ", [ + "RELIABILITY: Messages are in DLQ for client ${var.client_id}.", + "Failed callback deliveries require operator attention.", + ]) + + comparison_operator = "GreaterThanThreshold" + evaluation_periods = 1 + metric_name = "ApproximateNumberOfMessagesVisible" + namespace = "AWS/SQS" + period = 300 + statistic = "Sum" + threshold = 0 + actions_enabled = true + treat_missing_data = "notBreaching" + + dimensions = { + QueueName = "${local.client_prefix}-delivery-dlq-queue" + } + + tags = merge( + local.default_tags, + { + Name = "${local.client_prefix}-dlq-depth" + }, + ) +} diff --git a/infrastructure/terraform/modules/client-delivery/module_https_client_lambda.tf b/infrastructure/terraform/modules/client-delivery/module_https_client_lambda.tf new file mode 100644 index 00000000..abf75968 --- /dev/null +++ b/infrastructure/terraform/modules/client-delivery/module_https_client_lambda.tf @@ -0,0 +1,79 @@ +module "https_client_lambda" { + source = "https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/3.0.7/terraform-lambda.zip" + + function_name = "https-client-${var.client_id}" + description = "HTTPS delivery Lambda for client ${var.client_id}" + + aws_account_id = var.aws_account_id + component = var.component + environment = var.environment + project = var.project + region = var.region + group = var.group + + log_retention_in_days = var.log_retention_in_days + kms_key_arn = var.kms_key_arn + + iam_policy_document = { + body = data.aws_iam_policy_document.https_client_lambda.json + } + + function_s3_bucket = var.delivery_lambda_s3_bucket + function_code_base_path = var.delivery_lambda_code_base_path + function_code_dir = "https-client-lambda/dist" + function_include_common = true + handler_function_name = "handler" + runtime = "nodejs22.x" + memory = var.delivery_lambda_memory + timeout = var.delivery_lambda_timeout + log_level = var.log_level + + force_lambda_code_deploy = var.force_lambda_code_deploy + enable_lambda_insights = false + enable_xray_tracing = var.enable_xray_tracing + + log_destination_arn = var.log_destination_arn + log_subscription_role_arn = var.log_subscription_role_arn + + lambda_env_vars = { + APPLICATIONS_MAP_S3_BUCKET = var.applications_map_s3_bucket + APPLICATIONS_MAP_S3_KEY = var.applications_map_s3_key + CLIENT_ID = var.client_id + CLIENT_SUBSCRIPTION_CACHE_TTL_SECONDS = "60" + CLIENT_SUBSCRIPTION_CONFIG_BUCKET = var.client_config_bucket + CLIENT_SUBSCRIPTION_CONFIG_PREFIX = var.client_config_key_prefix + DLQ_URL = module.dlq_delivery.sqs_queue_url + ELASTICACHE_CACHE_NAME = var.elasticache_cache_name + ELASTICACHE_ENDPOINT = var.elasticache_endpoint + ELASTICACHE_IAM_USERNAME = var.elasticache_iam_username + ENVIRONMENT = var.environment + MAX_RETRY_DURATION_SECONDS = tostring(var.max_retry_duration_seconds) + METRICS_NAMESPACE = "nhs-notify-cb" + MTLS_CA_S3_KEY = var.mtls_ca_s3_key # gitleaks:allow + MTLS_CERT_S3_BUCKET = var.mtls_cert_s3_bucket + MTLS_CERT_S3_KEY = var.mtls_cert_s3_key # gitleaks:allow + QUEUE_URL = module.sqs_delivery.sqs_queue_url + TOKEN_BUCKET_BURST_CAPACITY = tostring(var.token_bucket_burst_capacity) + CB_COOLDOWN_PERIOD_MS = tostring(var.cb_cooldown_period_ms) + CB_RECOVERY_PERIOD_MS = tostring(var.cb_recovery_period_ms) + } + + vpc_config = var.delivery_lambda_security_group_id != "" ? { + subnet_ids = var.vpc_subnet_ids + security_group_ids = [var.delivery_lambda_security_group_id] + } : null +} + +resource "aws_lambda_event_source_mapping" "sqs_delivery" { + event_source_arn = module.sqs_delivery.sqs_queue_arn + function_name = module.https_client_lambda.function_arn + batch_size = var.delivery_lambda_batch_size + maximum_batching_window_in_seconds = var.delivery_lambda_batching_window_sec + enabled = true + + scaling_config { + maximum_concurrency = var.delivery_lambda_max_concurrency + } + + function_response_types = ["ReportBatchItemFailures"] +} diff --git a/infrastructure/terraform/modules/client-delivery/module_sqs_per_client.tf b/infrastructure/terraform/modules/client-delivery/module_sqs_per_client.tf new file mode 100644 index 00000000..0fad559b --- /dev/null +++ b/infrastructure/terraform/modules/client-delivery/module_sqs_per_client.tf @@ -0,0 +1,47 @@ +module "sqs_delivery" { + source = "https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/3.0.9/terraform-sqs.zip" + + aws_account_id = var.aws_account_id + component = var.component + environment = var.environment + project = var.project + region = var.region + name = "${var.client_id}-delivery" + + sqs_kms_key_arn = var.kms_key_arn + + visibility_timeout_seconds = var.sqs_visibility_timeout_seconds + max_receive_count = var.sqs_max_receive_count + + create_dlq = false + + sqs_policy_overload = data.aws_iam_policy_document.sqs_delivery.json +} + +resource "aws_sqs_queue_redrive_policy" "delivery" { + queue_url = module.sqs_delivery.sqs_queue_url + redrive_policy = jsonencode({ + deadLetterTargetArn = module.dlq_delivery.sqs_queue_arn + maxReceiveCount = var.sqs_max_receive_count + }) +} + +data "aws_iam_policy_document" "sqs_delivery" { + statement { + sid = "AllowEventBridgeToSendMessage" + effect = "Allow" + + principals { + type = "Service" + identifiers = ["events.amazonaws.com"] + } + + actions = [ + "sqs:SendMessage", + ] + + resources = [ + "arn:aws:sqs:${var.region}:${var.aws_account_id}:${local.csi}-${var.client_id}-delivery-queue", + ] + } +} diff --git a/infrastructure/terraform/modules/client-delivery/variables.tf b/infrastructure/terraform/modules/client-delivery/variables.tf new file mode 100644 index 00000000..6729cbc1 --- /dev/null +++ b/infrastructure/terraform/modules/client-delivery/variables.tf @@ -0,0 +1,247 @@ +variable "project" { + type = string + description = "The name of the tfscaffold project" +} + +variable "environment" { + type = string + description = "The name of the tfscaffold environment" +} + +variable "component" { + type = string + description = "Component name" +} + +variable "aws_account_id" { + type = string + description = "Account ID" +} + +variable "region" { + type = string + description = "AWS Region" +} + +variable "group" { + type = string + description = "The name of the tfscaffold group" + default = null +} + +variable "client_id" { + type = string + description = "Unique identifier for this client" +} + +variable "kms_key_arn" { + type = string + description = "KMS Key ARN for encryption at rest" +} + +variable "client_bus_name" { + type = string + description = "EventBridge bus name for subscription rules" +} + +variable "subscriptions" { + type = map(object({ + subscription_id = string + target_ids = list(string) + })) + description = "Subscription definitions for this client, keyed by subscription_id" +} + +variable "subscription_targets" { + type = map(object({ + subscription_id = string + target_id = string + })) + description = "Flattened subscription-target fanout map keyed by subscription-target composite key" +} + +variable "client_config_bucket" { + type = string + description = "S3 bucket name containing client subscription configuration" +} + +variable "client_config_bucket_arn" { + type = string + description = "S3 bucket ARN containing client subscription configuration" +} + +variable "client_config_key_prefix" { + type = string + description = "S3 key prefix for client subscription configuration files" + default = "client_subscriptions/" +} + +variable "applications_map_s3_bucket" { + type = string + description = "S3 bucket containing the applications map JSON" +} + +variable "applications_map_s3_key" { + type = string + description = "S3 key for the applications map JSON file" +} + +variable "delivery_lambda_s3_bucket" { + type = string + description = "S3 bucket for Lambda function artefacts" +} + +variable "delivery_lambda_code_base_path" { + type = string + description = "Base path to Lambda source code directories" +} + +variable "force_lambda_code_deploy" { + type = bool + description = "Force Lambda code redeployment even when commit tag matches" + default = false +} + +variable "log_level" { + type = string + description = "Log level for the Lambda function" + default = "INFO" +} + +variable "log_retention_in_days" { + type = number + description = "CloudWatch log retention period in days" + default = 0 +} + +variable "log_destination_arn" { + type = string + description = "Firehose destination ARN for log forwarding" + default = "" +} + +variable "log_subscription_role_arn" { + type = string + description = "IAM role ARN for CloudWatch log subscription" + default = "" +} + +variable "delivery_lambda_batch_size" { + type = number + description = "Number of SQS messages per Lambda invocation" + default = 100 +} + +variable "delivery_lambda_batching_window_sec" { + type = number + description = "Maximum time in seconds to wait for a full batch before invoking Lambda. Allows the delivery queue to fill to batch_size, improving Lambda concurrency utilisation." + default = 1 +} + +variable "delivery_lambda_max_concurrency" { + type = number + description = "Maximum concurrent Lambda invocations for the SQS event source mapping" + default = 200 +} + +variable "delivery_lambda_memory" { + type = number + description = "Lambda memory allocation in MB" + default = 256 +} + +variable "delivery_lambda_timeout" { + type = number + description = "Lambda timeout in seconds" + default = 30 +} + +variable "max_retry_duration_seconds" { + type = number + description = "Maximum retry window before messages are sent to DLQ" + default = 7200 +} + +variable "sqs_visibility_timeout_seconds" { + type = number + description = "Visibility timeout for the per-client delivery queue" + default = 60 +} + +variable "sqs_max_receive_count" { + type = number + description = "Safety-net maximum receive count before a message moves to DLQ. Supplements the time-based retry window for cases where the Lambda fails before reaching the window check." + default = 100 +} + +variable "enable_xray_tracing" { + type = bool + description = "Enable AWS X-Ray active tracing for the Lambda function" + default = false +} + +variable "mtls_cert_s3_bucket" { + type = string + description = "S3 bucket containing the mTLS client certificate bundle" + default = "" +} + +variable "mtls_cert_s3_key" { + type = string + description = "S3 key for the mTLS client certificate PEM bundle" + default = "" +} + +variable "mtls_ca_s3_key" { + type = string + description = "S3 key for the CA certificate PEM bundle used for server verification" + default = "" +} + +variable "token_bucket_burst_capacity" { + type = number + description = "Token bucket burst capacity used by the rate limiter" + default = 2250 +} + +variable "elasticache_endpoint" { + type = string + description = "ElastiCache Serverless endpoint URL" + default = "" +} + +variable "elasticache_cache_name" { + type = string + description = "ElastiCache cache name for SigV4 token presigning" + default = "" +} + +variable "elasticache_iam_username" { + type = string + description = "IAM username for ElastiCache authentication" + default = "" +} + +variable "vpc_subnet_ids" { + type = list(string) + description = "VPC subnet IDs for Lambda execution" + default = [] +} + +variable "delivery_lambda_security_group_id" { + type = string + description = "Security group ID for the Lambda function" + default = "" +} + +variable "cb_cooldown_period_ms" { + type = number + description = "Full block duration after circuit opens, before half-open probes begin (ms)" + default = 120000 +} + +variable "cb_recovery_period_ms" { + type = number + description = "Linear ramp-up duration after circuit closes (ms)" + default = 600000 +} diff --git a/infrastructure/terraform/modules/client-destination/README.md b/infrastructure/terraform/modules/client-destination/README.md deleted file mode 100644 index 11b689c3..00000000 --- a/infrastructure/terraform/modules/client-destination/README.md +++ /dev/null @@ -1,32 +0,0 @@ - - - - -## Requirements - -No requirements. -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [aws\_account\_id](#input\_aws\_account\_id) | Account ID | `string` | n/a | yes | -| [client\_bus\_name](#input\_client\_bus\_name) | EventBus name where you create the rule | `string` | n/a | yes | -| [component](#input\_component) | Component name | `string` | n/a | yes | -| [environment](#input\_environment) | The name of the tfscaffold environment | `string` | n/a | yes | -| [kms\_key\_arn](#input\_kms\_key\_arn) | KMS Key ARN | `string` | n/a | yes | -| [project](#input\_project) | The name of the tfscaffold project | `string` | n/a | yes | -| [region](#input\_region) | AWS Region | `string` | n/a | yes | -| [subscription\_targets](#input\_subscription\_targets) | Flattened subscription-target fanout map keyed by subscription-target composite key |
map(object({
subscription_id = string
target_id = string
}))
| n/a | yes | -| [subscriptions](#input\_subscriptions) | Flattened subscription definitions keyed by subscription\_id |
map(object({
client_id = string
subscription_id = string
target_ids = list(string)
}))
| n/a | yes | -| [targets](#input\_targets) | Flattened target definitions keyed by target\_id |
map(object({
client_id = string
target_id = string
invocation_endpoint = string
invocation_rate_limit_per_second = number
http_method = string
header_name = string
header_value = string
}))
| n/a | yes | -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [target\_dlq](#module\_target\_dlq) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/3.0.7/terraform-sqs.zip | n/a | -## Outputs - -No outputs. - - - diff --git a/infrastructure/terraform/modules/client-destination/cloudwatch_event_api_destination_this.tf b/infrastructure/terraform/modules/client-destination/cloudwatch_event_api_destination_this.tf deleted file mode 100644 index 4bec92cc..00000000 --- a/infrastructure/terraform/modules/client-destination/cloudwatch_event_api_destination_this.tf +++ /dev/null @@ -1,10 +0,0 @@ -resource "aws_cloudwatch_event_api_destination" "per_target" { - for_each = var.targets - - name = "${local.csi}-${each.key}" - description = "API Destination for ${each.key}" - invocation_endpoint = each.value.invocation_endpoint - http_method = each.value.http_method - invocation_rate_limit_per_second = each.value.invocation_rate_limit_per_second - connection_arn = aws_cloudwatch_event_connection.per_target[each.key].arn -} diff --git a/infrastructure/terraform/modules/client-destination/cloudwatch_event_connection_main.tf b/infrastructure/terraform/modules/client-destination/cloudwatch_event_connection_main.tf deleted file mode 100644 index 7546d666..00000000 --- a/infrastructure/terraform/modules/client-destination/cloudwatch_event_connection_main.tf +++ /dev/null @@ -1,14 +0,0 @@ -resource "aws_cloudwatch_event_connection" "per_target" { - for_each = var.targets - - name = "${local.csi}-${each.key}" - description = "Event Connection which would be used by API Destination ${each.key}" - authorization_type = "API_KEY" - - auth_parameters { - api_key { - key = each.value.header_name - value = each.value.header_value - } - } -} diff --git a/infrastructure/terraform/modules/client-destination/cloudwatch_event_rule_main.tf b/infrastructure/terraform/modules/client-destination/cloudwatch_event_rule_main.tf deleted file mode 100644 index bdf7ea47..00000000 --- a/infrastructure/terraform/modules/client-destination/cloudwatch_event_rule_main.tf +++ /dev/null @@ -1,46 +0,0 @@ -resource "aws_cloudwatch_event_rule" "per_subscription" { - for_each = var.subscriptions - - name = "${local.csi}-${each.key}" - description = "Client Callbacks event rule for subscription ${each.key}" - event_bus_name = var.client_bus_name - - event_pattern = jsonencode({ - "detail" : { - "subscriptions" : [each.value.subscription_id] - } - }) -} - -resource "aws_cloudwatch_event_target" "per_subscription_target" { - for_each = var.subscription_targets - - rule = aws_cloudwatch_event_rule.per_subscription[each.value.subscription_id].name - arn = aws_cloudwatch_event_api_destination.per_target[each.value.target_id].arn - target_id = "${local.csi}-${each.value.target_id}" - role_arn = aws_iam_role.api_target_role.arn - event_bus_name = var.client_bus_name - - dead_letter_config { - arn = module.target_dlq[each.value.target_id].sqs_queue_arn - } - - input_transformer { - input_paths = { - data = "$.detail.payload.data" - } - - input_template = "{\"data\": }" - } - - http_target { - header_parameters = { - "x-hmac-sha256-signature" = "$.detail.signatures.${replace(each.value.target_id, "-", "_")}" - } - } - - retry_policy { - maximum_retry_attempts = 3 - maximum_event_age_in_seconds = 3600 - } -} diff --git a/infrastructure/terraform/modules/client-destination/iam_role_api_target_role.tf b/infrastructure/terraform/modules/client-destination/iam_role_api_target_role.tf deleted file mode 100644 index 1158a2b2..00000000 --- a/infrastructure/terraform/modules/client-destination/iam_role_api_target_role.tf +++ /dev/null @@ -1,83 +0,0 @@ -resource "aws_iam_role" "api_target_role" { - name = "${local.csi}-api-target-target-role" - description = "Role for client target rule" - assume_role_policy = data.aws_iam_policy_document.api_target_role_assume_role_policy.json -} - -data "aws_iam_policy_document" "api_target_role_assume_role_policy" { - statement { - actions = [ - "sts:AssumeRole" - ] - - principals { - type = "Service" - identifiers = ["events.amazonaws.com"] - } - } -} - -resource "aws_iam_role_policy_attachment" "api_target_role" { - role = aws_iam_role.api_target_role.id - policy_arn = aws_iam_policy.api_target_role.arn -} - -resource "aws_iam_policy" "api_target_role" { - name = "${local.csi}-api-target-role-policy" - description = "IAM Policy for the client target role" - path = "/" - policy = data.aws_iam_policy_document.api_target_role.json -} - -data "aws_iam_policy_document" "api_target_role" { - dynamic "statement" { - for_each = length(aws_cloudwatch_event_api_destination.per_target) > 0 ? [1] : [] - content { - sid = "AllowAPIDestinationAccess" - effect = "Allow" - - actions = [ - "events:InvokeApiDestination", - ] - - resources = [ - for destination in aws_cloudwatch_event_api_destination.per_target : - destination.arn - ] - } - } - - dynamic "statement" { - for_each = length(module.target_dlq) > 0 ? [1] : [] - content { - sid = "AllowSQSSendMessageForDLQ" - effect = "Allow" - - actions = [ - "sqs:SendMessage", - ] - - resources = [ - for dlq in module.target_dlq : - dlq.sqs_queue_arn - ] - } - } - - statement { - sid = "AllowKMSForDLQ" - effect = "Allow" - - actions = [ - "kms:ReEncrypt*", - "kms:GenerateDataKey*", - "kms:Encrypt", - "kms:DescribeKey", - "kms:Decrypt" - ] - - resources = [ - var.kms_key_arn, - ] - } -} diff --git a/infrastructure/terraform/modules/client-destination/locals.tf b/infrastructure/terraform/modules/client-destination/locals.tf deleted file mode 100644 index fe672990..00000000 --- a/infrastructure/terraform/modules/client-destination/locals.tf +++ /dev/null @@ -1,12 +0,0 @@ -locals { - csi = replace( - format( - "%s-%s-%s", - var.project, - var.environment, - var.component, - ), - "_", - "", - ) -} diff --git a/infrastructure/terraform/modules/client-destination/module_target_dlq.tf b/infrastructure/terraform/modules/client-destination/module_target_dlq.tf deleted file mode 100644 index 36c4c277..00000000 --- a/infrastructure/terraform/modules/client-destination/module_target_dlq.tf +++ /dev/null @@ -1,41 +0,0 @@ -module "target_dlq" { - source = "https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/3.0.7/terraform-sqs.zip" - for_each = var.targets - - aws_account_id = var.aws_account_id - component = var.component - environment = var.environment - project = var.project - region = var.region - name = "${each.key}-dlq" - - sqs_kms_key_arn = var.kms_key_arn - - visibility_timeout_seconds = 60 - - create_dlq = false - - sqs_policy_overload = data.aws_iam_policy_document.target_dlq[each.key].json -} - -data "aws_iam_policy_document" "target_dlq" { - for_each = var.targets - - statement { - sid = "AllowEventBridgeToSendMessage" - effect = "Allow" - - principals { - type = "Service" - identifiers = ["events.amazonaws.com"] - } - - actions = [ - "sqs:SendMessage" - ] - - resources = [ - "arn:aws:sqs:${var.region}:${var.aws_account_id}:${var.project}-${var.environment}-${var.component}-${each.key}-dlq-queue" - ] - } -} diff --git a/infrastructure/terraform/modules/client-destination/variables.tf b/infrastructure/terraform/modules/client-destination/variables.tf deleted file mode 100644 index 2b9a0ceb..00000000 --- a/infrastructure/terraform/modules/client-destination/variables.tf +++ /dev/null @@ -1,67 +0,0 @@ -variable "project" { - type = string - description = "The name of the tfscaffold project" -} - -variable "environment" { - type = string - description = "The name of the tfscaffold environment" -} - -variable "component" { - type = string - description = "Component name" -} - -variable "aws_account_id" { - type = string - description = "Account ID" -} - -variable "region" { - type = string - description = "AWS Region" -} - -variable "targets" { - type = map(object({ - client_id = string - target_id = string - invocation_endpoint = string - invocation_rate_limit_per_second = number - http_method = string - header_name = string - header_value = string - })) - - description = "Flattened target definitions keyed by target_id" -} - -variable "subscriptions" { - type = map(object({ - client_id = string - subscription_id = string - target_ids = list(string) - })) - - description = "Flattened subscription definitions keyed by subscription_id" -} - -variable "subscription_targets" { - type = map(object({ - subscription_id = string - target_id = string - })) - - description = "Flattened subscription-target fanout map keyed by subscription-target composite key" -} - -variable "client_bus_name" { - type = string - description = "EventBus name where you create the rule" -} - -variable "kms_key_arn" { - type = string - description = "KMS Key ARN" -} diff --git a/infrastructure/terraform/modules/mock-webhook-alb-mtls/README.md b/infrastructure/terraform/modules/mock-webhook-alb-mtls/README.md new file mode 100644 index 00000000..c992b7e1 --- /dev/null +++ b/infrastructure/terraform/modules/mock-webhook-alb-mtls/README.md @@ -0,0 +1,32 @@ + + + + +## Requirements + +No requirements. +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [ca\_cert\_pem](#input\_ca\_cert\_pem) | PEM-encoded CA certificate chain for the ACM certificate | `string` | n/a | yes | +| [csi](#input\_csi) | Compound service identifier for resource naming | `string` | n/a | yes | +| [default\_tags](#input\_default\_tags) | Default tags to apply to all resources | `map(string)` | `{}` | no | +| [https\_client\_lambda\_sg\_id](#input\_https\_client\_lambda\_sg\_id) | Security group ID of the HTTPS Client Lambda | `string` | n/a | yes | +| [mock\_webhook\_lambda\_function\_arn](#input\_mock\_webhook\_lambda\_function\_arn) | Function ARN of the mock webhook Lambda | `string` | n/a | yes | +| [mock\_webhook\_lambda\_function\_name](#input\_mock\_webhook\_lambda\_function\_name) | Function name of the mock webhook Lambda | `string` | n/a | yes | +| [private\_subnets](#input\_private\_subnets) | Private subnet IDs for the ALB | `list(string)` | n/a | yes | +| [server\_cert\_pem](#input\_server\_cert\_pem) | PEM-encoded server certificate for the ALB HTTPS listener | `string` | n/a | yes | +| [server\_private\_key\_pem](#input\_server\_private\_key\_pem) | PEM-encoded private key for the server certificate | `string` | n/a | yes | +| [vpc\_id](#input\_vpc\_id) | VPC ID for the security group | `string` | n/a | yes | +## Modules + +No modules. +## Outputs + +| Name | Description | +|------|-------------| +| [dns\_name](#output\_dns\_name) | DNS name of the mock webhook mTLS ALB | + + + diff --git a/infrastructure/terraform/modules/mock-webhook-alb-mtls/acm_certificate_mock_webhook_server.tf b/infrastructure/terraform/modules/mock-webhook-alb-mtls/acm_certificate_mock_webhook_server.tf new file mode 100644 index 00000000..1495121a --- /dev/null +++ b/infrastructure/terraform/modules/mock-webhook-alb-mtls/acm_certificate_mock_webhook_server.tf @@ -0,0 +1,6 @@ +resource "aws_acm_certificate" "mock_webhook_server" { + certificate_body = var.server_cert_pem + private_key = var.server_private_key_pem + certificate_chain = var.ca_cert_pem + tags = var.default_tags +} diff --git a/infrastructure/terraform/modules/mock-webhook-alb-mtls/lambda_permission_mock_webhook_mtls_alb.tf b/infrastructure/terraform/modules/mock-webhook-alb-mtls/lambda_permission_mock_webhook_mtls_alb.tf new file mode 100644 index 00000000..c7726bef --- /dev/null +++ b/infrastructure/terraform/modules/mock-webhook-alb-mtls/lambda_permission_mock_webhook_mtls_alb.tf @@ -0,0 +1,7 @@ +resource "aws_lambda_permission" "mock_webhook_mtls_alb" { + statement_id = "AllowMtlsAlb" + action = "lambda:InvokeFunction" + function_name = var.mock_webhook_lambda_function_name + principal = "elasticloadbalancing.amazonaws.com" + source_arn = aws_lb_target_group.mock_webhook_mtls.arn +} diff --git a/infrastructure/terraform/modules/mock-webhook-alb-mtls/lb_listener_mock_webhook_mtls.tf b/infrastructure/terraform/modules/mock-webhook-alb-mtls/lb_listener_mock_webhook_mtls.tf new file mode 100644 index 00000000..12ff3fc7 --- /dev/null +++ b/infrastructure/terraform/modules/mock-webhook-alb-mtls/lb_listener_mock_webhook_mtls.tf @@ -0,0 +1,18 @@ +resource "aws_lb_listener" "mock_webhook_mtls" { + load_balancer_arn = aws_lb.mock_webhook_mtls.arn + port = 443 + protocol = "HTTPS" + ssl_policy = "ELBSecurityPolicy-TLS13-1-2-2021-06" + certificate_arn = aws_acm_certificate.mock_webhook_server.arn + + mutual_authentication { + mode = "passthrough" + } + + default_action { + type = "forward" + target_group_arn = aws_lb_target_group.mock_webhook_mtls.arn + } + + tags = var.default_tags +} diff --git a/infrastructure/terraform/modules/mock-webhook-alb-mtls/lb_mock_webhook_mtls.tf b/infrastructure/terraform/modules/mock-webhook-alb-mtls/lb_mock_webhook_mtls.tf new file mode 100644 index 00000000..db2081c4 --- /dev/null +++ b/infrastructure/terraform/modules/mock-webhook-alb-mtls/lb_mock_webhook_mtls.tf @@ -0,0 +1,8 @@ +resource "aws_lb" "mock_webhook_mtls" { + name = substr("${var.csi}-mock-mtls", 0, 32) + internal = true + load_balancer_type = "application" + security_groups = [aws_security_group.mock_webhook_alb.id] + subnets = var.private_subnets + tags = var.default_tags +} diff --git a/infrastructure/terraform/modules/mock-webhook-alb-mtls/lb_target_group_attachment_mock_webhook_mtls.tf b/infrastructure/terraform/modules/mock-webhook-alb-mtls/lb_target_group_attachment_mock_webhook_mtls.tf new file mode 100644 index 00000000..6865b41e --- /dev/null +++ b/infrastructure/terraform/modules/mock-webhook-alb-mtls/lb_target_group_attachment_mock_webhook_mtls.tf @@ -0,0 +1,5 @@ +resource "aws_lb_target_group_attachment" "mock_webhook_mtls" { + target_group_arn = aws_lb_target_group.mock_webhook_mtls.arn + target_id = var.mock_webhook_lambda_function_arn + depends_on = [aws_lambda_permission.mock_webhook_mtls_alb] +} diff --git a/infrastructure/terraform/modules/mock-webhook-alb-mtls/lb_target_group_mock_webhook_mtls.tf b/infrastructure/terraform/modules/mock-webhook-alb-mtls/lb_target_group_mock_webhook_mtls.tf new file mode 100644 index 00000000..6e55a0c3 --- /dev/null +++ b/infrastructure/terraform/modules/mock-webhook-alb-mtls/lb_target_group_mock_webhook_mtls.tf @@ -0,0 +1,5 @@ +resource "aws_lb_target_group" "mock_webhook_mtls" { + name = substr("${var.csi}-mock-mtls", 0, 32) + target_type = "lambda" + tags = var.default_tags +} diff --git a/infrastructure/terraform/modules/mock-webhook-alb-mtls/outputs.tf b/infrastructure/terraform/modules/mock-webhook-alb-mtls/outputs.tf new file mode 100644 index 00000000..34cdad12 --- /dev/null +++ b/infrastructure/terraform/modules/mock-webhook-alb-mtls/outputs.tf @@ -0,0 +1,4 @@ +output "dns_name" { + description = "DNS name of the mock webhook mTLS ALB" + value = aws_lb.mock_webhook_mtls.dns_name +} diff --git a/infrastructure/terraform/modules/mock-webhook-alb-mtls/security_group_mock_webhook_alb.tf b/infrastructure/terraform/modules/mock-webhook-alb-mtls/security_group_mock_webhook_alb.tf new file mode 100644 index 00000000..e8848d67 --- /dev/null +++ b/infrastructure/terraform/modules/mock-webhook-alb-mtls/security_group_mock_webhook_alb.tf @@ -0,0 +1,12 @@ +resource "aws_security_group" "mock_webhook_alb" { + name = "${var.csi}-mock-webhook-alb" + description = "Security group for mock webhook ALB mTLS endpoint" + vpc_id = var.vpc_id + + tags = merge( + var.default_tags, + { + Name = "${var.csi}-mock-webhook-alb" + }, + ) +} diff --git a/infrastructure/terraform/modules/mock-webhook-alb-mtls/variables.tf b/infrastructure/terraform/modules/mock-webhook-alb-mtls/variables.tf new file mode 100644 index 00000000..7b0e620a --- /dev/null +++ b/infrastructure/terraform/modules/mock-webhook-alb-mtls/variables.tf @@ -0,0 +1,51 @@ +variable "csi" { + type = string + description = "Compound service identifier for resource naming" +} + +variable "vpc_id" { + type = string + description = "VPC ID for the security group" +} + +variable "private_subnets" { + type = list(string) + description = "Private subnet IDs for the ALB" +} + +variable "https_client_lambda_sg_id" { + type = string + description = "Security group ID of the HTTPS Client Lambda" +} + +variable "mock_webhook_lambda_function_name" { + type = string + description = "Function name of the mock webhook Lambda" +} + +variable "mock_webhook_lambda_function_arn" { + type = string + description = "Function ARN of the mock webhook Lambda" +} + +variable "server_cert_pem" { + type = string + description = "PEM-encoded server certificate for the ALB HTTPS listener" +} + +variable "server_private_key_pem" { + type = string + description = "PEM-encoded private key for the server certificate" + sensitive = true +} + +variable "ca_cert_pem" { + type = string + description = "PEM-encoded CA certificate chain for the ACM certificate" +} + +variable "default_tags" { + type = map(string) + description = "Default tags to apply to all resources" + default = {} +} diff --git a/infrastructure/terraform/modules/mock-webhook-alb-mtls/vpc_security_group_egress_rule_mock_webhook_alb.tf b/infrastructure/terraform/modules/mock-webhook-alb-mtls/vpc_security_group_egress_rule_mock_webhook_alb.tf new file mode 100644 index 00000000..7609f241 --- /dev/null +++ b/infrastructure/terraform/modules/mock-webhook-alb-mtls/vpc_security_group_egress_rule_mock_webhook_alb.tf @@ -0,0 +1,6 @@ +resource "aws_vpc_security_group_egress_rule" "mock_webhook_alb_egress" { + security_group_id = aws_security_group.mock_webhook_alb.id + ip_protocol = "-1" + cidr_ipv4 = "0.0.0.0/0" + tags = var.default_tags +} diff --git a/infrastructure/terraform/modules/mock-webhook-alb-mtls/vpc_security_group_ingress_rule_mock_webhook_alb_https.tf b/infrastructure/terraform/modules/mock-webhook-alb-mtls/vpc_security_group_ingress_rule_mock_webhook_alb_https.tf new file mode 100644 index 00000000..3aa119fd --- /dev/null +++ b/infrastructure/terraform/modules/mock-webhook-alb-mtls/vpc_security_group_ingress_rule_mock_webhook_alb_https.tf @@ -0,0 +1,9 @@ +resource "aws_vpc_security_group_ingress_rule" "mock_webhook_alb_https" { + security_group_id = aws_security_group.mock_webhook_alb.id + referenced_security_group_id = var.https_client_lambda_sg_id + from_port = 443 + to_port = 443 + ip_protocol = "tcp" + description = "Allow HTTPS Client Lambda to reach mock webhook (mTLS and non-mTLS)" + tags = var.default_tags +} diff --git a/knip.ts b/knip.ts index 6de5f65a..d88de1eb 100644 --- a/knip.ts +++ b/knip.ts @@ -21,7 +21,7 @@ const config: KnipConfig = { // ESLint peer deps – referenced indirectly through plugin configs "@stylistic/eslint-plugin", "@typescript-eslint/parser", - // Used in lambdas' lambda-build npm script via pnpm exec + // Used in lambdas' lambda-build script via pnpm exec "esbuild", // Used in scripts/tests/unit.sh (shell script, not scanned by knip) "lcov-result-merger", @@ -35,9 +35,18 @@ const config: KnipConfig = { // Resolved transitively through tsconfig.base.json → @tsconfig/node22 ignoreDependencies: ["@tsconfig/node22"], }, + "lambdas/https-client-lambda": { + ignoreDependencies: ["@tsconfig/node22"], + }, "lambdas/mock-webhook-lambda": { ignoreDependencies: ["@tsconfig/node22"], }, + "lambdas/perf-runner-lambda": { + ignoreDependencies: ["@tsconfig/node22", "@types/aws-lambda"], + }, + "src/config-subscription-cache": { + ignoreDependencies: ["@tsconfig/node22"], + }, "src/logger": { ignoreDependencies: ["@tsconfig/node22"], }, @@ -45,6 +54,7 @@ const config: KnipConfig = { ignoreDependencies: ["@tsconfig/node22"], }, "tests/integration": { + entry: ["helpers/**/*.ts"], ignoreDependencies: [ "@tsconfig/node22", // Used in helpers/sqs.ts and helpers/cloudwatch.ts; flagged because @@ -52,9 +62,6 @@ const config: KnipConfig = { "async-wait-until", ], }, - "tests/performance": { - ignoreDependencies: ["@tsconfig/node22"], - }, "tests/test-support": { ignoreDependencies: ["@tsconfig/node22"], }, diff --git a/lambdas/client-transform-filter-lambda/package.json b/lambdas/client-transform-filter-lambda/package.json index e3a2c7ef..0f8cc805 100644 --- a/lambdas/client-transform-filter-lambda/package.json +++ b/lambdas/client-transform-filter-lambda/package.json @@ -1,7 +1,7 @@ { "dependencies": { "@aws-sdk/client-s3": "catalog:aws", - "@aws-sdk/client-ssm": "catalog:aws", + "@nhs-notify-client-callbacks/config-subscription-cache": "workspace:*", "@nhs-notify-client-callbacks/logger": "workspace:*", "@nhs-notify-client-callbacks/models": "workspace:*", "aws-embedded-metrics": "catalog:app", diff --git a/lambdas/client-transform-filter-lambda/src/__tests__/index.component.test.ts b/lambdas/client-transform-filter-lambda/src/__tests__/index.component.test.ts index b46c49f8..8198cdfb 100644 --- a/lambdas/client-transform-filter-lambda/src/__tests__/index.component.test.ts +++ b/lambdas/client-transform-filter-lambda/src/__tests__/index.component.test.ts @@ -15,26 +15,12 @@ jest.mock("@aws-sdk/client-s3", () => { }; }); -const mockSsmSend = jest.fn(); -jest.mock("@aws-sdk/client-ssm", () => { - const actual = jest.requireActual("@aws-sdk/client-ssm"); - return { - ...actual, - SSMClient: jest.fn().mockImplementation(() => ({ - send: mockSsmSend, - })), - }; -}); - -// Set environment variables before importing the handler/module under test so that -// services constructed at module import time (e.g. applicationsMapService) see -// the correct configuration. +// Set environment variables before importing the handler/module under test. process.env.CLIENT_SUBSCRIPTION_CONFIG_BUCKET = "test-bucket"; process.env.CLIENT_SUBSCRIPTION_CONFIG_PREFIX = "client_subscriptions/"; process.env.CLIENT_SUBSCRIPTION_CACHE_TTL_SECONDS = "60"; process.env.METRICS_NAMESPACE = "test-namespace"; process.env.ENVIRONMENT = "test"; -process.env.APPLICATIONS_MAP_PARAMETER = "/test/applications-map"; jest.mock("aws-embedded-metrics", () => ({ createMetricsLogger: jest.fn(() => ({ @@ -47,15 +33,15 @@ jest.mock("aws-embedded-metrics", () => ({ Count: "Count", Milliseconds: "Milliseconds", }, + StorageResolution: { High: 1, Standard: 60 }, })); import { GetObjectCommand, NoSuchKey } from "@aws-sdk/client-s3"; -import { GetParameterCommand } from "@aws-sdk/client-ssm"; import type { SQSRecord } from "aws-lambda"; import { EventTypes } from "@nhs-notify-client-callbacks/models"; import { createMessageStatusConfig } from "__tests__/helpers/client-subscription-fixtures"; import { createS3Client } from "services/config-loader-service"; -import { applicationsMapService, configLoaderService, handler } from ".."; +import { configLoaderService, handler } from ".."; const makeSqsRecord = (body: object): SQSRecord => ({ messageId: "sqs-id", @@ -104,18 +90,8 @@ const validMessageStatusEvent = (clientId: string, messageStatus: string) => ({ }); describe("Lambda handler with S3 subscription filtering", () => { - const applicationsMap = JSON.stringify({ - "client-1": "app-id-1", - "client-a": "app-id-a", - "client-b": "app-id-b", - "client-no-config": "app-id-no-config", - }); - beforeEach(() => { mockSend.mockClear(); - mockSsmSend.mockClear(); - applicationsMapService.reset(); - mockSsmSend.mockResolvedValue({ Parameter: { Value: applicationsMap } }); // Reset loader and clear cache for clean state between tests configLoaderService.reset( createS3Client({ AWS_ENDPOINT_URL: "http://localhost:4566" }), @@ -129,7 +105,6 @@ describe("Lambda handler with S3 subscription filtering", () => { delete process.env.CLIENT_SUBSCRIPTION_CACHE_TTL_SECONDS; delete process.env.METRICS_NAMESPACE; delete process.env.ENVIRONMENT; - delete process.env.APPLICATIONS_MAP_PARAMETER; }); it("passes event through when client config matches subscription", async () => { @@ -148,12 +123,8 @@ describe("Lambda handler with S3 subscription filtering", () => { expect(result).toHaveLength(1); expect(mockSend).toHaveBeenCalledTimes(1); expect(mockSend.mock.calls[0][0]).toBeInstanceOf(GetObjectCommand); - expect(mockSsmSend).toHaveBeenCalledTimes(1); - expect(mockSsmSend.mock.calls[0][0]).toBeInstanceOf(GetParameterCommand); expect(result[0]).toHaveProperty("payload"); expect(result[0]).toHaveProperty("subscriptions"); - expect(result[0]).toHaveProperty("signatures"); - expect(Object.values(result[0].signatures)[0]).toMatch(/^[0-9a-f]+$/); }); it("filters out event when status is not in subscription", async () => { @@ -251,25 +222,4 @@ describe("Lambda handler with S3 subscription filtering", () => { // S3 fetched once per distinct client (client-a and client-b), not once per event expect(mockSend).toHaveBeenCalledTimes(2); }); - - it("filters out event when no applicationId found in SSM map", async () => { - mockSend.mockResolvedValue({ - Body: { - transformToString: jest - .fn() - .mockResolvedValue( - JSON.stringify(createValidConfig("client-unknown")), - ), - }, - }); - mockSsmSend.mockResolvedValue({ - Parameter: { Value: JSON.stringify({}) }, - }); - - const result = await handler([ - makeSqsRecord(validMessageStatusEvent("client-unknown", "DELIVERED")), - ]); - - expect(result).toHaveLength(0); - }); }); diff --git a/lambdas/client-transform-filter-lambda/src/__tests__/index.test.ts b/lambdas/client-transform-filter-lambda/src/__tests__/index.test.ts index 14b10096..d21cdf4e 100644 --- a/lambdas/client-transform-filter-lambda/src/__tests__/index.test.ts +++ b/lambdas/client-transform-filter-lambda/src/__tests__/index.test.ts @@ -7,10 +7,9 @@ import type { MessageStatusData, StatusPublishEvent, } from "@nhs-notify-client-callbacks/models"; -import type { Logger } from "services/logger"; +import type { Logger } from "@nhs-notify-client-callbacks/logger"; import type { CallbackMetrics } from "services/metrics"; import type { ConfigLoader } from "services/config-loader"; -import type { ApplicationsMapService } from "services/ssm-applications-map"; import { ObservabilityService } from "services/observability"; import { ConfigLoaderService } from "services/config-loader-service"; import { @@ -71,15 +70,6 @@ const makeStubConfigLoaderService = (): ConfigLoaderService => { return { getLoader: () => loader } as unknown as ConfigLoaderService; }; -const makeStubApplicationsMapService = (): ApplicationsMapService => - ({ - getApplicationId: jest - .fn() - .mockImplementation( - async (clientId: string) => `test-app-id-${clientId}`, - ), - }) as unknown as ApplicationsMapService; - describe("Lambda handler", () => { const mockLogger = { info: jest.fn(), @@ -109,7 +99,6 @@ describe("Lambda handler", () => { createObservabilityService: () => new ObservabilityService(mockLogger, mockMetrics, mockMetricsLogger), createConfigLoaderService: makeStubConfigLoaderService, - createApplicationsMapService: makeStubApplicationsMapService, }); beforeEach(() => { @@ -173,7 +162,6 @@ describe("Lambda handler", () => { expect(result).toHaveLength(1); expect(result[0]).toHaveProperty("payload"); expect(result[0]).toHaveProperty("subscriptions"); - expect(result[0]).toHaveProperty("signatures"); const dataItem = result[0].payload.data[0]; expect(dataItem.type).toBe("MessageStatus"); expect((dataItem.attributes as MessageStatusAttributes).messageStatus).toBe( @@ -203,7 +191,6 @@ describe("Lambda handler", () => { new ObservabilityService(mockLogger, mockMetrics, mockMetricsLogger), createConfigLoaderService: () => ({ getLoader: () => customConfigLoader }) as ConfigLoaderService, - createApplicationsMapService: makeStubApplicationsMapService, }); const sqsMessage: SQSRecord = { @@ -234,65 +221,6 @@ describe("Lambda handler", () => { ); }); - it("should throw when any target is missing an apiKey", async () => { - const customConfigLoader = { - loadClientConfig: jest.fn().mockResolvedValue( - createClientSubscriptionConfig("client-abc-123", { - subscriptions: [ - createMessageStatusSubscription(["DELIVERED"], { - targetIds: ["target-no-key", DEFAULT_TARGET_ID], - }), - ], - targets: [ - createTarget({ - targetId: "target-no-key", - apiKey: undefined as unknown as { - headerName: string; - headerValue: string; - }, - }), - createTarget({ - targetId: DEFAULT_TARGET_ID, - apiKey: { - headerName: "x-api-key", - headerValue: "valid-key", - }, - }), - ], - }), - ), - } as unknown as ConfigLoader; - - const handlerWithMixedTargets = createHandler({ - createObservabilityService: () => - new ObservabilityService(mockLogger, mockMetrics, mockMetricsLogger), - createConfigLoaderService: () => - ({ getLoader: () => customConfigLoader }) as ConfigLoaderService, - createApplicationsMapService: makeStubApplicationsMapService, - }); - - const sqsMessage: SQSRecord = { - messageId: "sqs-msg-id-mixed", - receiptHandle: "receipt-handle-mixed", - body: JSON.stringify(validMessageStatusEvent), - attributes: { - ApproximateReceiveCount: "1", - SentTimestamp: "1519211230", - SenderId: "ABCDEFGHIJ", - ApproximateFirstReceiveTimestamp: "1519211230", - }, - messageAttributes: {}, - md5OfBody: "mock-md5", - eventSource: "aws:sqs", - eventSourceARN: "arn:aws:sqs:eu-west-2:123456789:mock-queue", - awsRegion: "eu-west-2", - }; - - await expect(handlerWithMixedTargets([sqsMessage])).rejects.toThrow( - "Missing apiKey for target target-no-key", - ); - }); - it("should handle batch of SQS messages from EventBridge Pipes", async () => { const sqsMessages: SQSRecord[] = [ { @@ -414,7 +342,6 @@ describe("Lambda handler", () => { expect(result).toHaveLength(1); expect(result[0]).toHaveProperty("payload"); expect(result[0]).toHaveProperty("subscriptions"); - expect(result[0]).toHaveProperty("signatures"); const dataItem = result[0].payload.data[0]; expect(dataItem.type).toBe("ChannelStatus"); expect((dataItem.attributes as ChannelStatusAttributes).channelStatus).toBe( @@ -481,7 +408,6 @@ describe("Lambda handler", () => { const faultyHandler = createHandler({ createObservabilityService: () => faultyObservability, createConfigLoaderService: makeStubConfigLoaderService, - createApplicationsMapService: makeStubApplicationsMapService, }); const sqsMessage: SQSRecord = { @@ -621,7 +547,7 @@ describe("createHandler default wiring", () => { CallbackMetrics: state.CallbackMetrics, })); - jest.doMock("services/logger", () => ({ + jest.doMock("@nhs-notify-client-callbacks/logger", () => ({ Logger: state.LoggerCtor, })); @@ -662,12 +588,11 @@ describe("createHandler default wiring", () => { [], state.mockObservabilityInstance, expect.any(Object), - expect.any(Object), ); expect(result).toEqual(["ok"]); jest.unmock("services/metrics"); - jest.unmock("services/logger"); + jest.unmock("@nhs-notify-client-callbacks/logger"); jest.unmock("services/observability"); jest.unmock("handler"); }); diff --git a/lambdas/client-transform-filter-lambda/src/__tests__/services/callback-logger.test.ts b/lambdas/client-transform-filter-lambda/src/__tests__/services/callback-logger.test.ts index 4d18ce42..d3c89858 100644 --- a/lambdas/client-transform-filter-lambda/src/__tests__/services/callback-logger.test.ts +++ b/lambdas/client-transform-filter-lambda/src/__tests__/services/callback-logger.test.ts @@ -2,7 +2,7 @@ import { logCallbackGenerated, logCallbackSigned, } from "services/callback-logger"; -import type { Logger } from "services/logger"; +import type { Logger } from "@nhs-notify-client-callbacks/logger"; import { type ClientCallbackPayload, EventTypes, diff --git a/lambdas/client-transform-filter-lambda/src/__tests__/services/config-cache.test.ts b/lambdas/client-transform-filter-lambda/src/__tests__/services/config-cache.test.ts deleted file mode 100644 index 6199b92c..00000000 --- a/lambdas/client-transform-filter-lambda/src/__tests__/services/config-cache.test.ts +++ /dev/null @@ -1,66 +0,0 @@ -import type { ClientSubscriptionConfiguration } from "@nhs-notify-client-callbacks/models"; -import { - createClientSubscriptionConfig, - createMessageStatusSubscription, -} from "__tests__/helpers/client-subscription-fixtures"; -import { ConfigCache } from "services/config-cache"; - -const createConfig = (): ClientSubscriptionConfiguration => - createClientSubscriptionConfig("client-1", { - subscriptions: [ - createMessageStatusSubscription(["DELIVERED"], { targetIds: [] }), - ], - }); - -describe("ConfigCache", () => { - it("stores and retrieves configuration", () => { - const cache = new ConfigCache(60_000); - const config = createConfig(); - - cache.set("client-1", config); - - expect(cache.get("client-1")).toEqual(config); - }); - - it("returns undefined for non-existent key", () => { - const cache = new ConfigCache(60_000); - const result = cache.get("non-existent"); - - expect(result).toBeUndefined(); - }); - - it("returns undefined for expired entries", () => { - jest.useFakeTimers(); - jest.setSystemTime(new Date("2025-01-01T10:00:00Z")); - - const cache = new ConfigCache(1000); // 1 second TTL - const config = createConfig(); - - cache.set("client-1", config); - expect(cache.get("client-1")).toEqual(config); - - jest.advanceTimersByTime(1001); - - const result = cache.get("client-1"); - - expect(result).toBeUndefined(); - - jest.useRealTimers(); - }); - - it("clears all entries", () => { - const cache = new ConfigCache(60_000); - const config = createConfig(); - - cache.set("client-1", config); - cache.set("client-2", config); - - expect(cache.get("client-1")).toEqual(config); - expect(cache.get("client-2")).toEqual(config); - - cache.clear(); - - expect(cache.get("client-1")).toBeUndefined(); - expect(cache.get("client-2")).toBeUndefined(); - }); -}); diff --git a/lambdas/client-transform-filter-lambda/src/__tests__/services/config-loader-service.test.ts b/lambdas/client-transform-filter-lambda/src/__tests__/services/config-loader-service.test.ts index a5741d2b..c907bb3f 100644 --- a/lambdas/client-transform-filter-lambda/src/__tests__/services/config-loader-service.test.ts +++ b/lambdas/client-transform-filter-lambda/src/__tests__/services/config-loader-service.test.ts @@ -1,4 +1,5 @@ import { S3Client } from "@aws-sdk/client-s3"; +import { ConfigSubscriptionCache } from "@nhs-notify-client-callbacks/config-subscription-cache"; import { ConfigLoader } from "services/config-loader"; import { ConfigLoaderService, @@ -8,6 +9,7 @@ import { const mockS3Client = jest.mocked(S3Client); const mockConfigLoader = jest.mocked(ConfigLoader); +const mockConfigSubscriptionCache = jest.mocked(ConfigSubscriptionCache); jest.mock("@aws-sdk/client-s3", () => ({ S3Client: jest.fn(), @@ -17,12 +19,19 @@ jest.mock("services/config-loader", () => ({ ConfigLoader: jest.fn(), })); +jest.mock("@nhs-notify-client-callbacks/config-subscription-cache", () => ({ + ConfigSubscriptionCache: jest.fn().mockImplementation(() => ({ + reset: jest.fn(), + })), +})); + describe("ConfigLoaderService", () => { const originalBucket = process.env.CLIENT_SUBSCRIPTION_CONFIG_BUCKET; const originalPrefix = process.env.CLIENT_SUBSCRIPTION_CONFIG_PREFIX; beforeEach(() => { mockConfigLoader.mockClear(); + mockConfigSubscriptionCache.mockClear(); process.env.CLIENT_SUBSCRIPTION_CONFIG_BUCKET = "test-bucket"; }); @@ -60,7 +69,7 @@ describe("ConfigLoaderService", () => { delete process.env.CLIENT_SUBSCRIPTION_CONFIG_PREFIX; const service = new ConfigLoaderService(); service.getLoader(); - expect(mockConfigLoader).toHaveBeenCalledWith( + expect(mockConfigSubscriptionCache).toHaveBeenCalledWith( expect.objectContaining({ keyPrefix: "client_subscriptions/" }), ); }); @@ -69,7 +78,7 @@ describe("ConfigLoaderService", () => { process.env.CLIENT_SUBSCRIPTION_CONFIG_PREFIX = "custom_prefix/"; const service = new ConfigLoaderService(); service.getLoader(); - expect(mockConfigLoader).toHaveBeenCalledWith( + expect(mockConfigSubscriptionCache).toHaveBeenCalledWith( expect.objectContaining({ keyPrefix: "custom_prefix/" }), ); }); @@ -90,7 +99,6 @@ describe("ConfigLoaderService", () => { }); const service = new ConfigLoaderService(); service.reset(customClient); - // Should not throw and the loader should be available immediately expect(() => service.getLoader()).not.toThrow(); }); @@ -101,7 +109,7 @@ describe("ConfigLoaderService", () => { }); const service = new ConfigLoaderService(); service.reset(customClient); - expect(mockConfigLoader).toHaveBeenCalledWith( + expect(mockConfigSubscriptionCache).toHaveBeenCalledWith( expect.objectContaining({ keyPrefix: "custom_prefix/" }), ); }); diff --git a/lambdas/client-transform-filter-lambda/src/__tests__/services/config-loader.test.ts b/lambdas/client-transform-filter-lambda/src/__tests__/services/config-loader.test.ts index 495164fb..e15e7ee6 100644 --- a/lambdas/client-transform-filter-lambda/src/__tests__/services/config-loader.test.ts +++ b/lambdas/client-transform-filter-lambda/src/__tests__/services/config-loader.test.ts @@ -1,10 +1,10 @@ import { GetObjectCommand, NoSuchKey, S3Client } from "@aws-sdk/client-s3"; import { createMessageStatusConfig } from "__tests__/helpers/client-subscription-fixtures"; -import { ConfigCache } from "services/config-cache"; +import { ConfigSubscriptionCache } from "@nhs-notify-client-callbacks/config-subscription-cache"; import { ConfigLoader } from "services/config-loader"; import { ConfigValidationError } from "services/validators/config-validator"; -jest.mock("services/logger", () => ({ +jest.mock("@nhs-notify-client-callbacks/logger", () => ({ logger: { debug: jest.fn(), info: jest.fn(), @@ -20,13 +20,15 @@ const mockBody = (json: string) => ({ const createValidConfig = (clientId: string) => createMessageStatusConfig(["DELIVERED"], clientId); -const createLoader = (send: jest.Mock) => - new ConfigLoader({ +const createLoader = (send: jest.Mock) => { + const cache = new ConfigSubscriptionCache({ + s3Client: { send } as unknown as S3Client, bucketName: "bucket", keyPrefix: "client_subscriptions/", - s3Client: { send } as unknown as S3Client, - cache: new ConfigCache(60_000), + ttlMs: 60_000, }); + return new ConfigLoader(cache); +}; describe("ConfigLoader", () => { it("loads and validates client configuration from S3", async () => { diff --git a/lambdas/client-transform-filter-lambda/src/__tests__/services/config-update.component.test.ts b/lambdas/client-transform-filter-lambda/src/__tests__/services/config-update.component.test.ts index 81af7f04..487e6130 100644 --- a/lambdas/client-transform-filter-lambda/src/__tests__/services/config-update.component.test.ts +++ b/lambdas/client-transform-filter-lambda/src/__tests__/services/config-update.component.test.ts @@ -1,8 +1,17 @@ import { S3Client } from "@aws-sdk/client-s3"; import { createMessageStatusConfig } from "__tests__/helpers/client-subscription-fixtures"; -import { ConfigCache } from "services/config-cache"; +import { ConfigSubscriptionCache } from "@nhs-notify-client-callbacks/config-subscription-cache"; import { ConfigLoader } from "services/config-loader"; +jest.mock("@nhs-notify-client-callbacks/logger", () => ({ + logger: { + debug: jest.fn(), + info: jest.fn(), + warn: jest.fn(), + error: jest.fn(), + }, +})); + const makeConfig = (messageStatuses: string[]) => createMessageStatusConfig(messageStatuses as never); @@ -28,12 +37,13 @@ describe("config update component", () => { }, }); - const loader = new ConfigLoader({ + const cache = new ConfigSubscriptionCache({ + s3Client: { send } as unknown as S3Client, bucketName: "bucket", keyPrefix: "client_subscriptions/", - s3Client: { send } as unknown as S3Client, - cache: new ConfigCache(1000), + ttlMs: 1000, }); + const loader = new ConfigLoader(cache); const first = await loader.loadClientConfig("client-1"); const firstMessage = first?.subscriptions.find( diff --git a/lambdas/client-transform-filter-lambda/src/__tests__/services/filters/channel-status-filter.test.ts b/lambdas/client-transform-filter-lambda/src/__tests__/services/filters/channel-status-filter.test.ts index e04eaf85..259d06ff 100644 --- a/lambdas/client-transform-filter-lambda/src/__tests__/services/filters/channel-status-filter.test.ts +++ b/lambdas/client-transform-filter-lambda/src/__tests__/services/filters/channel-status-filter.test.ts @@ -6,7 +6,7 @@ import { EventTypes } from "@nhs-notify-client-callbacks/models"; import { createChannelStatusConfig } from "__tests__/helpers/client-subscription-fixtures"; import { matchesChannelStatusSubscription } from "services/filters/channel-status-filter"; -jest.mock("services/logger", () => ({ +jest.mock("@nhs-notify-client-callbacks/logger", () => ({ logger: { debug: jest.fn(), info: jest.fn(), diff --git a/lambdas/client-transform-filter-lambda/src/__tests__/services/filters/message-status-filter.test.ts b/lambdas/client-transform-filter-lambda/src/__tests__/services/filters/message-status-filter.test.ts index ca9bd416..418c790e 100644 --- a/lambdas/client-transform-filter-lambda/src/__tests__/services/filters/message-status-filter.test.ts +++ b/lambdas/client-transform-filter-lambda/src/__tests__/services/filters/message-status-filter.test.ts @@ -6,7 +6,7 @@ import { EventTypes } from "@nhs-notify-client-callbacks/models"; import { createMessageStatusConfig } from "__tests__/helpers/client-subscription-fixtures"; import { matchesMessageStatusSubscription } from "services/filters/message-status-filter"; -jest.mock("services/logger", () => ({ +jest.mock("@nhs-notify-client-callbacks/logger", () => ({ logger: { debug: jest.fn(), info: jest.fn(), diff --git a/lambdas/client-transform-filter-lambda/src/__tests__/services/metrics.test.ts b/lambdas/client-transform-filter-lambda/src/__tests__/services/metrics.test.ts index bdbcc3aa..ed774769 100644 --- a/lambdas/client-transform-filter-lambda/src/__tests__/services/metrics.test.ts +++ b/lambdas/client-transform-filter-lambda/src/__tests__/services/metrics.test.ts @@ -1,7 +1,15 @@ -import { Unit, createMetricsLogger } from "aws-embedded-metrics"; +import { + StorageResolution, + Unit, + createMetricsLogger, +} from "aws-embedded-metrics"; import { CallbackMetrics, createMetricLogger } from "services/metrics"; -jest.mock("aws-embedded-metrics"); +jest.mock("aws-embedded-metrics", () => ({ + Unit: { Count: "Count" }, + StorageResolution: { High: 1, Standard: 60 }, + createMetricsLogger: jest.fn(), +})); const mockPutMetric = jest.fn(); const mockSetDimensions = jest.fn(); @@ -36,7 +44,7 @@ describe("createMetricsLogger", () => { }); it("should throw if ENVIRONMENT is not set", () => { - process.env.METRICS_NAMESPACE = "nhs-notify-client-callbacks-metrics"; + process.env.METRICS_NAMESPACE = "nhs-notify-cb"; expect(() => createMetricLogger()).toThrow( "ENVIRONMENT environment variable is not set", @@ -53,7 +61,7 @@ describe("createMetricsLogger", () => { }); it("should use ENVIRONMENT environment variable", () => { - process.env.METRICS_NAMESPACE = "nhs-notify-client-callbacks-metrics"; + process.env.METRICS_NAMESPACE = "nhs-notify-cb"; process.env.ENVIRONMENT = "production"; createMetricLogger(); @@ -80,6 +88,7 @@ describe("CallbackMetrics", () => { "EventsReceived", 1, Unit.Count, + StorageResolution.High, ); }); }); @@ -92,6 +101,7 @@ describe("CallbackMetrics", () => { "TransformationsSuccessful", 1, Unit.Count, + StorageResolution.Standard, ); }); }); @@ -104,6 +114,7 @@ describe("CallbackMetrics", () => { "TransformationsFailed", 1, Unit.Count, + StorageResolution.Standard, ); }); }); @@ -116,6 +127,7 @@ describe("CallbackMetrics", () => { "CallbacksInitiated", 1, Unit.Count, + StorageResolution.Standard, ); }); }); @@ -128,6 +140,7 @@ describe("CallbackMetrics", () => { "ValidationErrors", 1, Unit.Count, + StorageResolution.Standard, ); }); }); @@ -140,6 +153,7 @@ describe("CallbackMetrics", () => { "FilteringStarted", 1, Unit.Count, + StorageResolution.Standard, ); }); }); @@ -152,6 +166,7 @@ describe("CallbackMetrics", () => { "FilteringMatched", 1, Unit.Count, + StorageResolution.Standard, ); }); }); diff --git a/lambdas/client-transform-filter-lambda/src/__tests__/services/payload-signer.test.ts b/lambdas/client-transform-filter-lambda/src/__tests__/services/payload-signer.test.ts deleted file mode 100644 index e1785d55..00000000 --- a/lambdas/client-transform-filter-lambda/src/__tests__/services/payload-signer.test.ts +++ /dev/null @@ -1,49 +0,0 @@ -import { createHmac } from "node:crypto"; -import type { ClientCallbackPayload } from "@nhs-notify-client-callbacks/models"; -import { signPayload } from "services/payload-signer"; - -const makePayload = (id = "msg-1") => - ({ data: [{ id }] }) as unknown as ClientCallbackPayload; - -describe("signPayload", () => { - it("produces the expected HMAC-SHA256 hex string", () => { - const payload = makePayload(); - const applicationId = "app-id-1"; - const apiKey = "api-key-1"; - - const expected = createHmac("sha256", `${applicationId}.${apiKey}`) - .update(JSON.stringify(payload)) - .digest("hex"); - - expect(signPayload(payload, applicationId, apiKey)).toBe(expected); - }); - - it("returns a non-empty hex string", () => { - const result = signPayload(makePayload(), "app-id", "api-key"); - expect(result).toMatch(/^[0-9a-f]+$/); - }); - - it("produces different signatures for different payloads", () => { - const apiKey = "key"; - const appId = "app"; - expect(signPayload(makePayload("msg-1"), appId, apiKey)).not.toBe( - signPayload(makePayload("msg-2"), appId, apiKey), - ); - }); - - it("produces different signatures for different applicationIds", () => { - const payload = makePayload(); - const apiKey = "key"; - expect(signPayload(payload, "app-1", apiKey)).not.toBe( - signPayload(payload, "app-2", apiKey), - ); - }); - - it("produces different signatures for different apiKeys", () => { - const payload = makePayload(); - const appId = "app"; - expect(signPayload(payload, appId, "key-1")).not.toBe( - signPayload(payload, appId, "key-2"), - ); - }); -}); diff --git a/lambdas/client-transform-filter-lambda/src/__tests__/services/ssm-applications-map.test.ts b/lambdas/client-transform-filter-lambda/src/__tests__/services/ssm-applications-map.test.ts deleted file mode 100644 index 7123009a..00000000 --- a/lambdas/client-transform-filter-lambda/src/__tests__/services/ssm-applications-map.test.ts +++ /dev/null @@ -1,156 +0,0 @@ -import { GetParameterCommand, SSMClient } from "@aws-sdk/client-ssm"; -import { - ApplicationsMapService, - createSsmClient, - resolveCacheTtlMs, -} from "services/ssm-applications-map"; - -jest.mock("services/logger", () => ({ - logger: { - debug: jest.fn(), - info: jest.fn(), - warn: jest.fn(), - error: jest.fn(), - }, -})); - -const makeSsmClient = (value: string | undefined) => - ({ - send: jest - .fn() - .mockResolvedValue( - value === undefined ? {} : { Parameter: { Value: value } }, - ), - }) as unknown as SSMClient; - -describe("ApplicationsMapService", () => { - beforeEach(() => { - jest.useFakeTimers(); - }); - - afterEach(() => { - jest.useRealTimers(); - }); - - it("returns the applicationId for a known clientId", async () => { - const ssmClient = makeSsmClient( - JSON.stringify({ "client-1": "app-id-1", "client-2": "app-id-2" }), - ); - const service = new ApplicationsMapService(ssmClient, "/test/param"); - - expect(await service.getApplicationId("client-1")).toBe("app-id-1"); - }); - - it("returns undefined for an unknown clientId", async () => { - const ssmClient = makeSsmClient(JSON.stringify({ "client-1": "app-id-1" })); - const service = new ApplicationsMapService(ssmClient, "/test/param"); - - expect(await service.getApplicationId("unknown")).toBeUndefined(); - }); - - it("loads from SSM and sends GetParameterCommand with WithDecryption", async () => { - const ssmClient = makeSsmClient(JSON.stringify({ "client-1": "app-id-1" })); - const service = new ApplicationsMapService(ssmClient, "/test/param"); - - await service.getApplicationId("client-1"); - - expect(ssmClient.send).toHaveBeenCalledTimes(1); - expect((ssmClient.send as jest.Mock).mock.calls[0][0]).toBeInstanceOf( - GetParameterCommand, - ); - }); - - it("caches the map and does not call SSM again within TTL", async () => { - const ssmClient = makeSsmClient(JSON.stringify({ "client-1": "app-id-1" })); - const service = new ApplicationsMapService(ssmClient, "/test/param", 5000); - - await service.getApplicationId("client-1"); - await service.getApplicationId("client-1"); - - expect(ssmClient.send).toHaveBeenCalledTimes(1); - }); - - it("reloads from SSM after TTL expires", async () => { - const ssmClient = makeSsmClient(JSON.stringify({ "client-1": "app-id-1" })); - const service = new ApplicationsMapService(ssmClient, "/test/param", 5000); - - await service.getApplicationId("client-1"); - jest.advanceTimersByTime(6000); - await service.getApplicationId("client-1"); - - expect(ssmClient.send).toHaveBeenCalledTimes(2); - }); - - it("throws when SSM parameter is missing", async () => { - const ssmClient = makeSsmClient(undefined); - const service = new ApplicationsMapService(ssmClient, "/test/param"); - - await expect(service.getApplicationId("client-1")).rejects.toThrow( - "SSM parameter '/test/param' not found or has no value", - ); - }); - - it("throws when APPLICATIONS_MAP_PARAMETER is not set", async () => { - const ssmClient = makeSsmClient(JSON.stringify({ "client-1": "app-id-1" })); - const service = new ApplicationsMapService(ssmClient, undefined); - - await expect(service.getApplicationId("client-1")).rejects.toThrow( - "APPLICATIONS_MAP_PARAMETER is required", - ); - }); - - it("throws when SSM parameter has empty value", async () => { - const ssmClient = { - send: jest.fn().mockResolvedValue({ Parameter: { Value: "" } }), - } as unknown as SSMClient; - const service = new ApplicationsMapService(ssmClient, "/test/param"); - - await expect(service.getApplicationId("client-1")).rejects.toThrow( - "SSM parameter '/test/param' not found or has no value", - ); - }); - - it("throws when SSM parameter contains invalid JSON", async () => { - const ssmClient = makeSsmClient("not valid json"); - const service = new ApplicationsMapService(ssmClient, "/test/param"); - - await expect(service.getApplicationId("client-1")).rejects.toThrow( - "SSM parameter '/test/param' contains invalid JSON", - ); - }); - - it("reset clears the cache and forces reload on next call", async () => { - const ssmClient = makeSsmClient(JSON.stringify({ "client-1": "app-id-1" })); - const service = new ApplicationsMapService(ssmClient, "/test/param", 5000); - - await service.getApplicationId("client-1"); - service.reset(); - await service.getApplicationId("client-1"); - - expect(ssmClient.send).toHaveBeenCalledTimes(2); - }); -}); - -describe("resolveCacheTtlMs", () => { - it("returns configured value in ms", () => { - expect( - resolveCacheTtlMs({ APPLICATIONS_MAP_CACHE_TTL_SECONDS: "30" }), - ).toBe(30_000); - }); - - it("returns default when env var is absent", () => { - expect(resolveCacheTtlMs({})).toBe(60_000); - }); - - it("returns default when env var is not a valid number", () => { - expect( - resolveCacheTtlMs({ APPLICATIONS_MAP_CACHE_TTL_SECONDS: "invalid" }), - ).toBe(60_000); - }); -}); - -describe("createSsmClient", () => { - it("returns an SSMClient instance", () => { - expect(createSsmClient({})).toBeInstanceOf(SSMClient); - }); -}); diff --git a/lambdas/client-transform-filter-lambda/src/__tests__/services/subscription-filter.test.ts b/lambdas/client-transform-filter-lambda/src/__tests__/services/subscription-filter.test.ts index 153ab934..b9ece544 100644 --- a/lambdas/client-transform-filter-lambda/src/__tests__/services/subscription-filter.test.ts +++ b/lambdas/client-transform-filter-lambda/src/__tests__/services/subscription-filter.test.ts @@ -18,7 +18,7 @@ import { import { TransformationError } from "services/error-handler"; import { evaluateSubscriptionFilters } from "services/subscription-filter"; -jest.mock("services/logger", () => ({ +jest.mock("@nhs-notify-client-callbacks/logger", () => ({ logger: { debug: jest.fn(), info: jest.fn(), diff --git a/lambdas/client-transform-filter-lambda/src/__tests__/transformers/message-status-transformer.test.ts b/lambdas/client-transform-filter-lambda/src/__tests__/transformers/message-status-transformer.test.ts index ce9b71ac..d3dd2f03 100644 --- a/lambdas/client-transform-filter-lambda/src/__tests__/transformers/message-status-transformer.test.ts +++ b/lambdas/client-transform-filter-lambda/src/__tests__/transformers/message-status-transformer.test.ts @@ -53,8 +53,6 @@ describe("message-status-transformer", () => { messageId: "msg-789-xyz", messageReference: "client-ref-12345", messageStatus: "delivered", - messageStatusDescription: "Message successfully delivered", - messageFailureReasonCode: undefined, channels: [ { type: "nhsapp", channelStatus: "delivered" }, { type: "sms", channelStatus: "skipped" }, @@ -65,6 +63,7 @@ describe("message-status-transformer", () => { version: "ztoe2qRAM8M8vS0bqajhyEBcvXacrGPp", createdDate: "2023-11-17T14:27:51.413Z", }, + messageStatusDescription: "Message successfully delivered", }; const expectedIdempotencyKey = createHash("sha256") .update(JSON.stringify(idempotencyBody)) diff --git a/lambdas/client-transform-filter-lambda/src/handler.ts b/lambdas/client-transform-filter-lambda/src/handler.ts index 0d1f20b6..f1f60c84 100644 --- a/lambdas/client-transform-filter-lambda/src/handler.ts +++ b/lambdas/client-transform-filter-lambda/src/handler.ts @@ -7,13 +7,11 @@ import type { } from "@nhs-notify-client-callbacks/models"; import { validateStatusPublishEvent } from "services/validators/event-validator"; import { transformEvent } from "services/transformers/event-transformer"; -import { extractCorrelationId, logger } from "services/logger"; +import { extractCorrelationId } from "@nhs-notify-client-callbacks/logger"; import { ValidationError, getEventError } from "services/error-handler"; import type { ObservabilityService } from "services/observability"; import type { ConfigLoader } from "services/config-loader"; import { evaluateSubscriptionFilters } from "services/subscription-filter"; -import type { ApplicationsMapService } from "services/ssm-applications-map"; -import { signPayload } from "services/payload-signer"; const BATCH_CONCURRENCY = Number(process.env.BATCH_CONCURRENCY) || 10; const MESSAGE_ROOT_URI = process.env.MESSAGE_ROOT_URI ?? ""; @@ -27,20 +25,9 @@ type FilteredEvent = UnsignedEvent & { targetIds: string[]; }; -type SignedEvent = { - transformedEvent: TransformedEvent; - deliveryContext: { - correlationId: string; - eventType: string; - clientId: string; - messageId: string; - }; -}; - export interface TransformedEvent { payload: ClientCallbackPayload; subscriptions: string[]; - signatures: Record; } class BatchStats { @@ -140,79 +127,6 @@ function processSingleEvent( type ClientConfigMap = Map; -async function signBatch( - filteredEvents: FilteredEvent[], - applicationsMapService: ApplicationsMapService, - configByClientId: ClientConfigMap, - stats: BatchStats, - observability: ObservabilityService, -): Promise { - const results = await pMap( - filteredEvents, - async (event): Promise => { - const { clientId } = event.data; - const correlationId = extractCorrelationId(event) ?? event.id; - - const applicationId = - await applicationsMapService.getApplicationId(clientId); - if (!applicationId) { - stats.recordFiltered(); - logger.warn( - "No applicationId found in SSM map - event will not be delivered", - { clientId, correlationId }, - ); - return undefined; - } - - const clientConfig = configByClientId.get(clientId); - const targetsById = new Map( - (clientConfig?.targets ?? []).map((t) => [t.targetId, t]), - ); - - const signaturesByTarget = new Map(); - - for (const targetId of event.targetIds) { - const target = targetsById.get(targetId); - const apiKey = target?.apiKey?.headerValue; - if (!apiKey) { - throw new ValidationError( - `Missing apiKey for target ${targetId}`, - correlationId, - ); - } - const signature = signPayload( - event.transformedPayload, - applicationId, - apiKey, - ); - signaturesByTarget.set(targetId.replaceAll("-", "_"), signature); - observability.recordCallbackSigned( - event.transformedPayload, - correlationId, - clientId, - signature, - ); - } - - return { - transformedEvent: { - payload: event.transformedPayload, - subscriptions: event.subscriptionIds, - signatures: Object.fromEntries(signaturesByTarget), - }, - deliveryContext: { - correlationId, - eventType: event.type, - clientId, - messageId: event.data.messageId, - }, - }; - }, - { concurrency: BATCH_CONCURRENCY }, - ); - return results.filter((e): e is SignedEvent => e !== undefined); -} - async function loadClientConfigs( events: UnsignedEvent[], configLoader: ConfigLoader, @@ -304,7 +218,6 @@ export async function processEvents( event: SQSRecord[], observability: ObservabilityService, configLoader: ConfigLoader, - applicationsMapService: ApplicationsMapService, ): Promise { const startTime = Date.now(); const stats = new BatchStats(); @@ -324,20 +237,21 @@ export async function processEvents( stats, ); - const signedEvents = await signBatch( - filteredEvents, - applicationsMapService, - configByClientId, - stats, - observability, - ); - - for (const signedEvent of signedEvents) { - observability.recordDeliveryInitiated(signedEvent.deliveryContext); - } + const deliverableEvents: TransformedEvent[] = filteredEvents.map( + (filteredEvent) => { + const correlationId = extractCorrelationId(filteredEvent); + observability.recordDeliveryInitiated({ + correlationId, + eventType: filteredEvent.type, + clientId: filteredEvent.data.clientId, + messageId: filteredEvent.data.messageId, + }); - const deliverableEvents = signedEvents.map( - (signedEvent) => signedEvent.transformedEvent, + return { + payload: filteredEvent.transformedPayload, + subscriptions: filteredEvent.subscriptionIds, + }; + }, ); const processingTime = Date.now() - startTime; diff --git a/lambdas/client-transform-filter-lambda/src/index.ts b/lambdas/client-transform-filter-lambda/src/index.ts index 9d631bfe..4b3cdc9f 100644 --- a/lambdas/client-transform-filter-lambda/src/index.ts +++ b/lambdas/client-transform-filter-lambda/src/index.ts @@ -1,19 +1,15 @@ import type { SQSRecord } from "aws-lambda"; -import { Logger } from "services/logger"; +import { Logger } from "@nhs-notify-client-callbacks/logger"; import { CallbackMetrics, createMetricLogger } from "services/metrics"; import { ObservabilityService } from "services/observability"; import { ConfigLoaderService } from "services/config-loader-service"; -import { ApplicationsMapService } from "services/ssm-applications-map"; import { type TransformedEvent, processEvents } from "handler"; export const configLoaderService = new ConfigLoaderService(); -export const applicationsMapService = new ApplicationsMapService(); - export interface HandlerDependencies { createObservabilityService?: () => ObservabilityService; createConfigLoaderService?: () => ConfigLoaderService; - createApplicationsMapService?: () => ApplicationsMapService; } function createDefaultObservabilityService(): ObservabilityService { @@ -28,10 +24,6 @@ function createDefaultConfigLoaderService(): ConfigLoaderService { return configLoaderService; } -function createDefaultApplicationsMapService(): ApplicationsMapService { - return applicationsMapService; -} - export function createHandler( dependencies: Partial = {}, ): (event: SQSRecord[]) => Promise { @@ -41,19 +33,10 @@ export function createHandler( const configLoader = ( dependencies.createConfigLoaderService ?? createDefaultConfigLoaderService )(); - const applicationsMap = ( - dependencies.createApplicationsMapService ?? - createDefaultApplicationsMapService - )(); return async (event: SQSRecord[]): Promise => { const observability = createObservabilityService(); - return processEvents( - event, - observability, - configLoader.getLoader(), - applicationsMap, - ); + return processEvents(event, observability, configLoader.getLoader()); }; } diff --git a/lambdas/client-transform-filter-lambda/src/services/callback-logger.ts b/lambdas/client-transform-filter-lambda/src/services/callback-logger.ts index a3ac6a25..b177fcbf 100644 --- a/lambdas/client-transform-filter-lambda/src/services/callback-logger.ts +++ b/lambdas/client-transform-filter-lambda/src/services/callback-logger.ts @@ -4,7 +4,7 @@ import { EventTypes, type MessageStatusAttributes, } from "@nhs-notify-client-callbacks/models"; -import type { Logger } from "services/logger"; +import type { Logger } from "@nhs-notify-client-callbacks/logger"; function isMessageStatusAttributes( attributes: MessageStatusAttributes | ChannelStatusAttributes, diff --git a/lambdas/client-transform-filter-lambda/src/services/config-cache.ts b/lambdas/client-transform-filter-lambda/src/services/config-cache.ts deleted file mode 100644 index 641cc60c..00000000 --- a/lambdas/client-transform-filter-lambda/src/services/config-cache.ts +++ /dev/null @@ -1,33 +0,0 @@ -import type { ClientSubscriptionConfiguration } from "@nhs-notify-client-callbacks/models"; - -type CacheEntry = { - value: ClientSubscriptionConfiguration; - expiresAt: number; -}; - -export class ConfigCache { - private readonly cache = new Map(); - - constructor(private readonly ttlMs: number) {} - - get(clientId: string): ClientSubscriptionConfiguration | undefined { - const entry = this.cache.get(clientId); - - if (entry && entry.expiresAt <= Date.now()) { - this.cache.delete(clientId); - } - - return this.cache.get(clientId)?.value; - } - - set(clientId: string, value: ClientSubscriptionConfiguration): void { - this.cache.set(clientId, { - value, - expiresAt: Date.now() + this.ttlMs, - }); - } - - clear(): void { - this.cache.clear(); - } -} diff --git a/lambdas/client-transform-filter-lambda/src/services/config-loader-service.ts b/lambdas/client-transform-filter-lambda/src/services/config-loader-service.ts index b0af71b0..b5542d01 100644 --- a/lambdas/client-transform-filter-lambda/src/services/config-loader-service.ts +++ b/lambdas/client-transform-filter-lambda/src/services/config-loader-service.ts @@ -1,5 +1,5 @@ import { S3Client } from "@aws-sdk/client-s3"; -import { ConfigCache } from "services/config-cache"; +import { ConfigSubscriptionCache } from "@nhs-notify-client-callbacks/config-subscription-cache"; import { ConfigLoader } from "services/config-loader"; const DEFAULT_CACHE_TTL_SECONDS = 60; @@ -26,52 +26,49 @@ export const createS3Client = ( }; export class ConfigLoaderService { - private readonly cache: ConfigCache; - private loader: ConfigLoader | undefined; + private cache: ConfigSubscriptionCache | undefined; + + private readonly ttlMs: number; + constructor(cacheTtlMs: number = resolveCacheTtlMs()) { - this.cache = new ConfigCache(cacheTtlMs); + this.ttlMs = cacheTtlMs; } getLoader(): ConfigLoader { - const bucketName = process.env.CLIENT_SUBSCRIPTION_CONFIG_BUCKET; - if (!bucketName) { - throw new Error("CLIENT_SUBSCRIPTION_CONFIG_BUCKET is required"); - } - if (this.loader) { return this.loader; } - this.loader = new ConfigLoader({ - bucketName, - keyPrefix: - process.env.CLIENT_SUBSCRIPTION_CONFIG_PREFIX ?? - "client_subscriptions/", - s3Client: createS3Client(), - cache: this.cache, - }); - + this.cache = this.createCache(createS3Client()); + this.loader = new ConfigLoader(this.cache); return this.loader; } reset(s3Client?: S3Client): void { + this.cache?.reset(); this.loader = undefined; - this.cache.clear(); + this.cache = undefined; if (s3Client) { - const bucketName = process.env.CLIENT_SUBSCRIPTION_CONFIG_BUCKET; - if (!bucketName) { - throw new Error("CLIENT_SUBSCRIPTION_CONFIG_BUCKET is required"); - } - this.loader = new ConfigLoader({ - bucketName, - keyPrefix: - process.env.CLIENT_SUBSCRIPTION_CONFIG_PREFIX ?? - "client_subscriptions/", - s3Client, - cache: this.cache, - }); + this.cache = this.createCache(s3Client); + this.loader = new ConfigLoader(this.cache); + } + } + + private createCache(s3Client: S3Client): ConfigSubscriptionCache { + const bucketName = process.env.CLIENT_SUBSCRIPTION_CONFIG_BUCKET; + if (!bucketName) { + throw new Error("CLIENT_SUBSCRIPTION_CONFIG_BUCKET is required"); } + + return new ConfigSubscriptionCache({ + s3Client, + bucketName, + keyPrefix: + process.env.CLIENT_SUBSCRIPTION_CONFIG_PREFIX ?? + "client_subscriptions/", + ttlMs: this.ttlMs, + }); } } diff --git a/lambdas/client-transform-filter-lambda/src/services/config-loader.ts b/lambdas/client-transform-filter-lambda/src/services/config-loader.ts index 2d5b388f..32a4e370 100644 --- a/lambdas/client-transform-filter-lambda/src/services/config-loader.ts +++ b/lambdas/client-transform-filter-lambda/src/services/config-loader.ts @@ -1,82 +1,21 @@ -import { GetObjectCommand, NoSuchKey, S3Client } from "@aws-sdk/client-s3"; +import type { ConfigSubscriptionCache } from "@nhs-notify-client-callbacks/config-subscription-cache"; import type { ClientSubscriptionConfiguration } from "@nhs-notify-client-callbacks/models"; -import { ConfigCache } from "services/config-cache"; -import { logger } from "services/logger"; +import { logger } from "@nhs-notify-client-callbacks/logger"; import { wrapUnknownError } from "services/error-handler"; -import { - ConfigValidationError, - validateClientConfig, -} from "services/validators/config-validator"; - -type ConfigLoaderOptions = { - bucketName: string; - keyPrefix: string; - s3Client: S3Client; - cache: ConfigCache; -}; - -function throwAsConfigError(error: unknown, clientId: string): never { - if (error instanceof ConfigValidationError) { - logger.error("Config validation failed with schema violations", { - clientId, - validationErrors: error.issues, - }); - throw error; - } - - const { message } = wrapUnknownError(error); - logger.error("Failed to load config from S3", { clientId }); - throw new ConfigValidationError([{ path: "config", message }]); -} +import { ConfigValidationError } from "services/validators/config-validator"; export class ConfigLoader { - constructor(private readonly options: ConfigLoaderOptions) {} + constructor(private readonly cache: ConfigSubscriptionCache) {} async loadClientConfig( clientId: string, ): Promise { - const cached = this.options.cache.get(clientId); - if (cached) { - logger.debug("Config loaded from cache", { clientId, cacheHit: true }); - return cached; - } - - logger.debug("Config not in cache, fetching from S3", { - clientId, - cacheHit: false, - }); - try { - const response = await this.options.s3Client.send( - new GetObjectCommand({ - Bucket: this.options.bucketName, - Key: `${this.options.keyPrefix}${clientId}.json`, - }), - ); - - if (!response.Body) { - throw new Error("S3 response body was empty"); - } - - const rawConfig = await response.Body.transformToString(); - const parsedConfig = JSON.parse(rawConfig) as unknown; - const validated = validateClientConfig(parsedConfig); - this.options.cache.set(clientId, validated); - logger.info("Config loaded successfully from S3", { - clientId, - subscriptionCount: validated.subscriptions.length, - }); - return validated; + return await this.cache.loadClientConfig(clientId); } catch (error) { - if (error instanceof NoSuchKey) { - logger.info( - "No config found in S3 for client - events will be filtered out", - { clientId }, - ); - return undefined; - } - throwAsConfigError(error, clientId); - return undefined; + const { message } = wrapUnknownError(error); + logger.error("Failed to load config", { clientId }); + throw new ConfigValidationError([{ path: "config", message }]); } } } diff --git a/lambdas/client-transform-filter-lambda/src/services/filters/channel-status-filter.ts b/lambdas/client-transform-filter-lambda/src/services/filters/channel-status-filter.ts index e43394bf..1a669281 100644 --- a/lambdas/client-transform-filter-lambda/src/services/filters/channel-status-filter.ts +++ b/lambdas/client-transform-filter-lambda/src/services/filters/channel-status-filter.ts @@ -4,7 +4,7 @@ import type { ClientSubscriptionConfiguration, StatusPublishEvent, } from "@nhs-notify-client-callbacks/models"; -import { logger } from "services/logger"; +import { logger } from "@nhs-notify-client-callbacks/logger"; const isChannelStatusSubscription = ( subscription: ClientSubscriptionConfiguration["subscriptions"][number], diff --git a/lambdas/client-transform-filter-lambda/src/services/filters/message-status-filter.ts b/lambdas/client-transform-filter-lambda/src/services/filters/message-status-filter.ts index 01bead4f..c51f8c71 100644 --- a/lambdas/client-transform-filter-lambda/src/services/filters/message-status-filter.ts +++ b/lambdas/client-transform-filter-lambda/src/services/filters/message-status-filter.ts @@ -4,7 +4,7 @@ import type { MessageStatusSubscriptionConfiguration, StatusPublishEvent, } from "@nhs-notify-client-callbacks/models"; -import { logger } from "services/logger"; +import { logger } from "@nhs-notify-client-callbacks/logger"; const isMessageStatusSubscription = ( subscription: ClientSubscriptionConfiguration["subscriptions"][number], diff --git a/lambdas/client-transform-filter-lambda/src/services/logger.ts b/lambdas/client-transform-filter-lambda/src/services/logger.ts deleted file mode 100644 index 5c373b25..00000000 --- a/lambdas/client-transform-filter-lambda/src/services/logger.ts +++ /dev/null @@ -1 +0,0 @@ -export * from "@nhs-notify-client-callbacks/logger"; diff --git a/lambdas/client-transform-filter-lambda/src/services/metrics.ts b/lambdas/client-transform-filter-lambda/src/services/metrics.ts index 398c5ecc..9984ed58 100644 --- a/lambdas/client-transform-filter-lambda/src/services/metrics.ts +++ b/lambdas/client-transform-filter-lambda/src/services/metrics.ts @@ -1,4 +1,8 @@ -import { Unit, createMetricsLogger } from "aws-embedded-metrics"; +import { + StorageResolution, + Unit, + createMetricsLogger, +} from "aws-embedded-metrics"; import type { MetricsLogger } from "aws-embedded-metrics"; export const createMetricLogger = (): MetricsLogger => { @@ -21,30 +25,65 @@ export class CallbackMetrics { constructor(private readonly metrics: MetricsLogger) {} emitEventReceived(): void { - this.metrics.putMetric("EventsReceived", 1, Unit.Count); + this.metrics.putMetric( + "EventsReceived", + 1, + Unit.Count, + StorageResolution.High, + ); } emitTransformationSuccess(): void { - this.metrics.putMetric("TransformationsSuccessful", 1, Unit.Count); + this.metrics.putMetric( + "TransformationsSuccessful", + 1, + Unit.Count, + StorageResolution.Standard, + ); } emitTransformationFailure(): void { - this.metrics.putMetric("TransformationsFailed", 1, Unit.Count); + this.metrics.putMetric( + "TransformationsFailed", + 1, + Unit.Count, + StorageResolution.Standard, + ); } emitDeliveryInitiated(): void { - this.metrics.putMetric("CallbacksInitiated", 1, Unit.Count); + this.metrics.putMetric( + "CallbacksInitiated", + 1, + Unit.Count, + StorageResolution.Standard, + ); } emitValidationError(): void { - this.metrics.putMetric("ValidationErrors", 1, Unit.Count); + this.metrics.putMetric( + "ValidationErrors", + 1, + Unit.Count, + StorageResolution.Standard, + ); } emitFilteringStarted(): void { - this.metrics.putMetric("FilteringStarted", 1, Unit.Count); + this.metrics.putMetric( + "FilteringStarted", + 1, + Unit.Count, + StorageResolution.Standard, + ); } emitFilteringMatched(): void { - this.metrics.putMetric("FilteringMatched", 1, Unit.Count); + this.metrics.putMetric( + "FilteringMatched", + 1, + Unit.Count, + StorageResolution.Standard, + ); } } diff --git a/lambdas/client-transform-filter-lambda/src/services/observability.ts b/lambdas/client-transform-filter-lambda/src/services/observability.ts index 4cfbf469..e921db26 100644 --- a/lambdas/client-transform-filter-lambda/src/services/observability.ts +++ b/lambdas/client-transform-filter-lambda/src/services/observability.ts @@ -1,11 +1,8 @@ import type { MetricsLogger } from "aws-embedded-metrics"; import type { ClientCallbackPayload } from "@nhs-notify-client-callbacks/models"; -import { - logCallbackGenerated, - logCallbackSigned, -} from "services/callback-logger"; -import type { Logger } from "services/logger"; -import { logLifecycleEvent } from "services/logger"; +import { logCallbackGenerated } from "services/callback-logger"; +import type { Logger } from "@nhs-notify-client-callbacks/logger"; +import { logLifecycleEvent } from "@nhs-notify-client-callbacks/logger"; import type { CallbackMetrics } from "services/metrics"; export class ObservabilityService { @@ -95,15 +92,6 @@ export class ObservabilityService { this.metrics.emitTransformationSuccess(); } - recordCallbackSigned( - payload: ClientCallbackPayload, - correlationId: string | undefined, - clientId: string, - signature: string, - ): void { - logCallbackSigned(this.logger, payload, correlationId, clientId, signature); - } - createChild(context: { correlationId?: string; eventType: string; diff --git a/lambdas/client-transform-filter-lambda/src/services/ssm-applications-map.ts b/lambdas/client-transform-filter-lambda/src/services/ssm-applications-map.ts deleted file mode 100644 index 87cead24..00000000 --- a/lambdas/client-transform-filter-lambda/src/services/ssm-applications-map.ts +++ /dev/null @@ -1,85 +0,0 @@ -import { GetParameterCommand, SSMClient } from "@aws-sdk/client-ssm"; -import { logger } from "services/logger"; - -const DEFAULT_CACHE_TTL_SECONDS = 60; - -export const createSsmClient = ( - env: NodeJS.ProcessEnv = process.env, -): SSMClient => { - const endpoint = env.AWS_ENDPOINT_URL; - return new SSMClient({ endpoint }); -}; - -export const resolveCacheTtlMs = ( - env: NodeJS.ProcessEnv = process.env, -): number => { - const ttlSeconds = Number.parseInt( - env.APPLICATIONS_MAP_CACHE_TTL_SECONDS ?? `${DEFAULT_CACHE_TTL_SECONDS}`, - 10, - ); - return ( - (Number.isFinite(ttlSeconds) ? ttlSeconds : DEFAULT_CACHE_TTL_SECONDS) * - 1000 - ); -}; - -export class ApplicationsMapService { - private cachedMap: Map | undefined; - - private cacheExpiresAt = 0; - - constructor( - private readonly ssmClient: SSMClient = createSsmClient(), - private readonly parameterName: string | undefined = process.env - .APPLICATIONS_MAP_PARAMETER, - private readonly cacheTtlMs: number = resolveCacheTtlMs(), - ) {} - - async getApplicationId(clientId: string): Promise { - const map = await this.getMap(); - return map.get(clientId); - } - - private async getMap(): Promise> { - if (!this.parameterName) { - throw new Error("APPLICATIONS_MAP_PARAMETER is required"); - } - const { parameterName } = this; - - if (this.cachedMap && Date.now() < this.cacheExpiresAt) { - logger.debug("Applications map loaded from cache"); - return this.cachedMap; - } - - const response = await this.ssmClient.send( - new GetParameterCommand({ - Name: parameterName, - WithDecryption: true, - }), - ); - - if (!response.Parameter?.Value) { - throw new Error( - `SSM parameter '${parameterName}' not found or has no value`, - ); - } - - let parsed: Record; - try { - parsed = JSON.parse(response.Parameter.Value) as Record; - } catch { - throw new Error(`SSM parameter '${parameterName}' contains invalid JSON`); - } - this.cachedMap = new Map(Object.entries(parsed)); - this.cacheExpiresAt = Date.now() + this.cacheTtlMs; - logger.info("Applications map loaded from SSM", { - parameterName, - }); - return this.cachedMap; - } - - reset(): void { - this.cachedMap = undefined; - this.cacheExpiresAt = 0; - } -} diff --git a/lambdas/client-transform-filter-lambda/src/services/subscription-filter.ts b/lambdas/client-transform-filter-lambda/src/services/subscription-filter.ts index 2a51627f..dbca66ff 100644 --- a/lambdas/client-transform-filter-lambda/src/services/subscription-filter.ts +++ b/lambdas/client-transform-filter-lambda/src/services/subscription-filter.ts @@ -8,7 +8,7 @@ import { EventTypes } from "@nhs-notify-client-callbacks/models"; import { matchesChannelStatusSubscription } from "services/filters/channel-status-filter"; import { matchesMessageStatusSubscription } from "services/filters/message-status-filter"; import { TransformationError } from "services/error-handler"; -import { logger } from "services/logger"; +import { logger } from "@nhs-notify-client-callbacks/logger"; type FilterResult = { matched: boolean; diff --git a/lambdas/client-transform-filter-lambda/src/services/transformers/message-status-transformer.ts b/lambdas/client-transform-filter-lambda/src/services/transformers/message-status-transformer.ts index a2803568..d4b493db 100644 --- a/lambdas/client-transform-filter-lambda/src/services/transformers/message-status-transformer.ts +++ b/lambdas/client-transform-filter-lambda/src/services/transformers/message-status-transformer.ts @@ -24,20 +24,6 @@ export function transformMessageStatus( }), ); - const idempotencyBody = { - messageId, - messageReference: data.messageReference, - messageStatus, - messageStatusDescription: data.messageStatusDescription, - messageFailureReasonCode: data.messageFailureReasonCode, - channels, - routingPlan: data.routingPlan, - }; - - const idempotencyKey = createHash("sha256") - .update(JSON.stringify(idempotencyBody)) - .digest("hex"); - const attributes: MessageStatusAttributes = { messageId, messageReference: data.messageReference, @@ -55,6 +41,13 @@ export function transformMessageStatus( attributes.messageFailureReasonCode = data.messageFailureReasonCode; } + const idempotencyBody = Object.fromEntries( + Object.entries(attributes).filter(([key]) => key !== "timestamp"), + ); + const idempotencyKey = createHash("sha256") + .update(JSON.stringify(idempotencyBody)) + .digest("hex"); + const payload: ClientCallbackPayload = { data: [ { diff --git a/lambdas/client-transform-filter-lambda/src/services/validators/event-validator.ts b/lambdas/client-transform-filter-lambda/src/services/validators/event-validator.ts index 03e37807..a726eac4 100644 --- a/lambdas/client-transform-filter-lambda/src/services/validators/event-validator.ts +++ b/lambdas/client-transform-filter-lambda/src/services/validators/event-validator.ts @@ -11,7 +11,7 @@ import { ValidationError, formatValidationIssuePath, } from "services/error-handler"; -import { extractCorrelationId } from "services/logger"; +import { extractCorrelationId } from "@nhs-notify-client-callbacks/logger"; const NHSNotifyExtensionsSchema = z.object({ traceparent: z.string().min(1), diff --git a/lambdas/https-client-lambda/jest.config.ts b/lambdas/https-client-lambda/jest.config.ts new file mode 100644 index 00000000..cd0ed08e --- /dev/null +++ b/lambdas/https-client-lambda/jest.config.ts @@ -0,0 +1,9 @@ +import { nodeJestConfig } from "../../jest.config.base.ts"; + +export default { + ...nodeJestConfig, + transform: { + ...nodeJestConfig.transform, + "\\.lua$": "/lua-transform.js", + }, +}; diff --git a/lambdas/https-client-lambda/lua-transform.js b/lambdas/https-client-lambda/lua-transform.js new file mode 100644 index 00000000..e6e0a1c9 --- /dev/null +++ b/lambdas/https-client-lambda/lua-transform.js @@ -0,0 +1,7 @@ +module.exports = { + process(sourceText) { + return { + code: `module.exports = ${JSON.stringify(sourceText)};`, + }; + }, +}; diff --git a/lambdas/https-client-lambda/package.json b/lambdas/https-client-lambda/package.json new file mode 100644 index 00000000..8082b859 --- /dev/null +++ b/lambdas/https-client-lambda/package.json @@ -0,0 +1,42 @@ +{ + "dependencies": { + "@aws-crypto/sha256-js": "catalog:aws", + "@aws-sdk/client-s3": "catalog:aws", + "@aws-sdk/client-sqs": "catalog:aws", + "@aws-sdk/credential-providers": "catalog:aws", + "@smithy/signature-v4": "catalog:aws", + "@nhs-notify-client-callbacks/config-subscription-cache": "workspace:*", + "@nhs-notify-client-callbacks/logger": "workspace:*", + "@nhs-notify-client-callbacks/models": "workspace:*", + "@redis/client": "catalog:app", + "aws-embedded-metrics": "catalog:app", + "esbuild": "catalog:tools", + "node-forge": "catalog:app", + "p-map": "catalog:app" + }, + "devDependencies": { + "@smithy/types": "catalog:aws", + "@tsconfig/node22": "catalog:tools", + "@types/aws-lambda": "catalog:tools", + "@types/jest": "catalog:test", + "@types/node": "catalog:tools", + "@types/node-forge": "catalog:tools", + "eslint": "catalog:lint", + "fengari": "^0.1.5", + "jest": "catalog:test", + "typescript": "catalog:tools" + }, + "engines": { + "node": ">=24.14.1" + }, + "name": "@nhs-notify-client-callbacks/https-client-lambda", + "private": true, + "scripts": { + "lambda-build": "rm -rf dist && pnpm exec esbuild --bundle --minify --sourcemap --target=es2020 --platform=node --loader:.node=file --loader:.lua=text --entry-names=[name] --outdir=dist src/index.ts", + "lint": "eslint .", + "lint:fix": "eslint . --fix", + "test:unit": "jest", + "typecheck": "tsc --noEmit" + }, + "version": "0.0.1" +} diff --git a/lambdas/https-client-lambda/src/__tests__/admit-lua.test.ts b/lambdas/https-client-lambda/src/__tests__/admit-lua.test.ts new file mode 100644 index 00000000..930198c6 --- /dev/null +++ b/lambdas/https-client-lambda/src/__tests__/admit-lua.test.ts @@ -0,0 +1,810 @@ +import admitLuaSrc from "services/admit.lua"; +import { createRedisStore, evalLua } from "__tests__/helpers/lua-redis-mock"; + +// ARGV: [now, capacity, targetRateLimit, cooldownMs, recoveryPeriodMs, probeRateLimit, targetBatchSize, cbEnabled] +// KEYS: [epKey] +// Returns: [consumedTokens, reason, retryAfterMs, effectiveRate] + +type AdmitArgs = { + now: number; + capacity: number; + targetRateLimit: number; + cooldownMs: number; + recoveryPeriodMs: number; + probeRateLimit: number; + targetBatchSize: number; + cbEnabled: boolean; +}; + +const defaultArgs: AdmitArgs = { + now: 1_000_000, + capacity: 2250, + targetRateLimit: 10, + cooldownMs: 120_000, + recoveryPeriodMs: 600_000, + probeRateLimit: 1 / 60, + targetBatchSize: 1, + cbEnabled: true, +}; + +type AdmitResult = { + consumedTokens: number; + reason: string; + retryAfterMs: number; + effectiveRate: number; +}; + +function runAdmit( + store: ReturnType, + args: Partial = {}, + targetId = "t1", +): AdmitResult { + const merged = { ...defaultArgs, ...args }; + const raw = evalLua( + admitLuaSrc, + [`ep:${targetId}`], + [ + merged.now.toString(), + merged.capacity.toString(), + merged.targetRateLimit.toString(), + merged.cooldownMs.toString(), + merged.recoveryPeriodMs.toString(), + merged.probeRateLimit.toString(), + merged.targetBatchSize.toString(), + merged.cbEnabled ? "1" : "0", + ], + store, + ) as [number, string, number, number]; + return { + consumedTokens: raw[0], + reason: raw[1], + retryAfterMs: raw[2], + effectiveRate: raw[3], + }; +} + +describe("admit.lua", () => { + describe("first contact (no prior state)", () => { + it("allows one initial probe token on a fresh endpoint", () => { + const store = createRedisStore(); + const now = 1_000_000; + + const { consumedTokens, effectiveRate, reason } = runAdmit(store, { + now, + targetRateLimit: 10, + }); + + expect(consumedTokens).toBe(1); + expect(reason).toBe("some_allowed"); + expect(effectiveRate).toBeCloseTo(1 / 60, 5); + }); + + it("generates an additional probe token on a subsequent call after enough elapsed time", () => { + const store = createRedisStore(); + + runAdmit(store, { now: 1_000_000, targetRateLimit: 10 }); + + const { consumedTokens, effectiveRate, reason } = runAdmit(store, { + now: 1_060_001, + targetRateLimit: 10, + }); + + expect(effectiveRate).toBeCloseTo(1 / 60, 5); + expect(consumedTokens).toBe(1); + expect(reason).toBe("some_allowed"); + }); + }); + + describe("token bucket", () => { + it("consumes up to targetBatchSize tokens", () => { + const store = createRedisStore(); + const now = 1_000_000; + store.set( + "ep:t1", + new Map([ + ["is_open", "0"], + ["bucket_tokens", "5"], + ["bucket_refilled_at", now.toString()], + ["switched_at", "0"], + ]), + ); + + const { consumedTokens } = runAdmit(store, { + now, + targetBatchSize: 3, + }); + expect(consumedTokens).toBe(3); + }); + + it("consumes all available when batch exceeds available tokens", () => { + const store = createRedisStore(); + const now = 1_000_000; + store.set( + "ep:t1", + new Map([ + ["is_open", "0"], + ["bucket_tokens", "2"], + ["bucket_refilled_at", now.toString()], + ["switched_at", "0"], + ]), + ); + + const { consumedTokens } = runAdmit(store, { + now, + targetBatchSize: 5, + }); + expect(consumedTokens).toBe(2); + }); + + it("returns rate_limited when no tokens available", () => { + const store = createRedisStore(); + const now = 1_000_000; + store.set( + "ep:t1", + new Map([ + ["is_open", "0"], + ["bucket_tokens", "0"], + ["bucket_refilled_at", now.toString()], + ["switched_at", "0"], + ]), + ); + + const { consumedTokens, reason, retryAfterMs } = runAdmit(store, { now }); + expect(consumedTokens).toBe(0); + expect(reason).toBe("rate_limited"); + expect(retryAfterMs).toBe(1000); + }); + + it("refills tokens over time", () => { + const store = createRedisStore(); + const now = 1_000_000; + store.set( + "ep:t1", + new Map([ + ["is_open", "0"], + ["bucket_tokens", "0"], + ["bucket_refilled_at", now.toString()], + ["switched_at", "0"], + ]), + ); + + const { consumedTokens } = runAdmit(store, { + now: now + 1000, + targetRateLimit: 10, + }); + expect(consumedTokens).toBe(1); + }); + + it("caps tokens at capacity", () => { + const store = createRedisStore(); + const now = 1_000_000; + store.set( + "ep:t1", + new Map([ + ["is_open", "0"], + ["bucket_tokens", "0"], + ["bucket_refilled_at", now.toString()], + ["switched_at", "0"], + ]), + ); + + const { consumedTokens } = runAdmit(store, { + now: now + 1000, + capacity: 5, + targetRateLimit: 100, + targetBatchSize: 10, + }); + expect(consumedTokens).toBe(5); + }); + + it("handles zero refill rate", () => { + const store = createRedisStore(); + const now = 1_000_000; + store.set( + "ep:t1", + new Map([ + ["is_open", "0"], + ["bucket_tokens", "0"], + ["bucket_refilled_at", now.toString()], + ["switched_at", "0"], + ]), + ); + + const { consumedTokens, reason } = runAdmit(store, { + now: now + 10_000, + targetRateLimit: 0, + }); + expect(consumedTokens).toBe(0); + expect(reason).toBe("rate_limited"); + }); + + it("advances bucket_refilled_at by token cost to preserve the sub-token remainder for the next call", () => { + const store = createRedisStore(); + const ratePerSecond = 10; + const msPerToken = 1000 / ratePerSecond; // 100ms + const elapsedMs = 150; // enough elapsed for 1 token (100ms), with 50ms remainder + const now = 1_000_000; + store.set( + "ep:t1", + new Map([ + ["is_open", "0"], + ["bucket_tokens", "0"], + ["bucket_refilled_at", (now - elapsedMs).toString()], + ["switched_at", "0"], + ]), + ); + + runAdmit(store, { now, targetRateLimit: ratePerSecond }); + + const epHash = store.get("ep:t1")!; + const refilledAt = Number(epHash.get("bucket_refilled_at")); + expect(refilledAt).not.toBe(now); // must not advance to now (elapsed time) + expect(refilledAt).toBe(now - elapsedMs + msPerToken); // must advance by token cost only + }); + + it("does not update bucket_refilled_at when no tokens are generated", () => { + const store = createRedisStore(); + const now = 1_000_000; + store.set( + "ep:t1", + new Map([ + ["is_open", "0"], + ["bucket_tokens", "5"], + ["bucket_refilled_at", now.toString()], + ["switched_at", "0"], + ]), + ); + + runAdmit(store, { now, targetRateLimit: 10 }); + + const epHash = store.get("ep:t1")!; + expect(Number(epHash.get("bucket_refilled_at"))).toBe(now); + }); + + it("returns retryAfterMs=0 when some tokens are consumed", () => { + const store = createRedisStore(); + const now = 1_000_000; + store.set( + "ep:t1", + new Map([ + ["is_open", "0"], + ["bucket_tokens", "5"], + ["bucket_refilled_at", now.toString()], + ["switched_at", "0"], + ]), + ); + + const { retryAfterMs } = runAdmit(store, { now, targetBatchSize: 3 }); + + expect(retryAfterMs).toBe(0); + }); + + it("returns rate_limited when targetBatchSize is 0", () => { + const store = createRedisStore(); + const now = 1_000_000; + store.set( + "ep:t1", + new Map([ + ["is_open", "0"], + ["bucket_tokens", "5"], + ["bucket_refilled_at", now.toString()], + ["switched_at", "0"], + ]), + ); + + const { consumedTokens, reason, retryAfterMs } = runAdmit(store, { + now, + targetBatchSize: 0, + }); + + expect(consumedTokens).toBe(0); + expect(reason).toBe("rate_limited"); + expect(retryAfterMs).toBe(1000); + }); + + it("accumulates tokens accurately across multiple sequential calls", () => { + const store = createRedisStore(); + const ratePerSecond = 10; + const msPerToken = 1000 / ratePerSecond; // 100ms + const callCount = 10; + const start = 1_000_000; + + store.set( + "ep:t1", + new Map([ + ["is_open", "0"], + ["switched_at", "0"], + ["bucket_tokens", "0"], + ["bucket_refilled_at", start.toString()], + ]), + ); + + let totalConsumed = 0; + for (let i = 1; i <= callCount; i++) { + const { consumedTokens } = runAdmit(store, { + now: start + i * msPerToken, + targetRateLimit: ratePerSecond, + targetBatchSize: 1, + }); + totalConsumed += consumedTokens; + } + + expect(totalConsumed).toBe(callCount); + }); + }); + + describe("circuit breaker — open (during cooldown)", () => { + it("blocks completely when circuit is open during cooldown", () => { + const store = createRedisStore(); + const now = 1_000_000; + const switchedAt = now - 10_000; + + store.set( + "ep:t1", + new Map([ + ["is_open", "1"], + ["switched_at", switchedAt.toString()], + ["bucket_tokens", "100"], + ]), + ); + + const { consumedTokens, reason } = runAdmit(store, { + now, + cooldownMs: 120_000, + }); + expect(consumedTokens).toBe(0); + expect(reason).toBe("circuit_open"); + }); + + it("does not consume bucket tokens when fully open", () => { + const store = createRedisStore(); + const now = 1_000_000; + const switchedAt = now - 10_000; + + store.set( + "ep:t1", + new Map([ + ["is_open", "1"], + ["switched_at", switchedAt.toString()], + ["bucket_tokens", "100"], + ["bucket_refilled_at", now.toString()], + ]), + ); + + runAdmit(store, { now, cooldownMs: 120_000 }); + + const epHash = store.get("ep:t1")!; + expect(Number(epHash.get("bucket_tokens"))).toBe(100); + }); + + it("returns retryAfterMs for open circuit", () => { + const store = createRedisStore(); + const now = 1_000_000; + const switchedAt = now - 10_000; + + store.set( + "ep:t1", + new Map([ + ["is_open", "1"], + ["switched_at", switchedAt.toString()], + ]), + ); + + const { retryAfterMs } = runAdmit(store, { now, cooldownMs: 120_000 }); + expect(retryAfterMs).toBe(110_000); + }); + }); + + describe("circuit breaker — half-open (after cooldown)", () => { + it("uses probeRateLimit when half-open", () => { + const store = createRedisStore(); + const now = 1_000_000; + const switchedAt = now - 130_000; + + store.set( + "ep:t1", + new Map([ + ["is_open", "1"], + ["switched_at", switchedAt.toString()], + ["bucket_tokens", "0"], + ["bucket_refilled_at", (now - 60_000).toString()], + ]), + ); + + const { effectiveRate } = runAdmit(store, { + now, + cooldownMs: 120_000, + probeRateLimit: 1 / 60, + }); + expect(effectiveRate).toBeCloseTo(1 / 60, 5); + }); + + it("zeroes residual bucket tokens when circuit is half-open", () => { + const store = createRedisStore(); + const now = 1_000_000; + const switchedAt = now - 130_000; + + store.set( + "ep:t1", + new Map([ + ["is_open", "1"], + ["switched_at", switchedAt.toString()], + ["bucket_tokens", "100"], + ["bucket_refilled_at", (now - 60_000).toString()], + ]), + ); + + const { consumedTokens } = runAdmit(store, { + now, + cooldownMs: 120_000, + probeRateLimit: 1 / 60, + }); + + expect(consumedTokens).toBe(1); + const epHash = store.get("ep:t1")!; + expect(Number(epHash.get("bucket_tokens"))).toBe(0); + }); + + it("gives 1 probe token when half-open with no prior bucket state", () => { + const store = createRedisStore(); + const now = 1_000_000; + store.set( + "ep:t1", + new Map([ + ["is_open", "1"], + ["switched_at", (now - 130_000).toString()], + ]), + ); + + const { consumedTokens, effectiveRate, reason } = runAdmit(store, { + now, + cooldownMs: 120_000, + probeRateLimit: 1 / 60, + }); + + expect(consumedTokens).toBe(1); + expect(reason).toBe("some_allowed"); + expect(effectiveRate).toBeCloseTo(1 / 60, 5); + }); + }); + + describe("circuit breaker — recovery ramp", () => { + const targetRateLimit = 10; + const probeRateLimit = 1 / 60; + const recoveryPeriodMs = 600_000; + const switchedAt = 1_000_000; + + it("uses probeRate at recovery start (progress=0)", () => { + const store = createRedisStore(); + const progress = 0; + const now = switchedAt + recoveryPeriodMs * progress; + + store.set( + "ep:t1", + new Map([ + ["is_open", "0"], + ["switched_at", switchedAt.toString()], + ["bucket_tokens", "0"], + ["bucket_refilled_at", "0"], + ]), + ); + + const { effectiveRate } = runAdmit(store, { + now, + targetRateLimit, + recoveryPeriodMs, + probeRateLimit, + }); + + expect(effectiveRate).toBeCloseTo(probeRateLimit, 5); + }); + + it("uses recovery ramp at midpoint (progress=0.5)", () => { + const store = createRedisStore(); + const progress = 0.5; + const now = switchedAt + recoveryPeriodMs * progress; + + store.set( + "ep:t1", + new Map([ + ["is_open", "0"], + ["switched_at", switchedAt.toString()], + ["bucket_tokens", "0"], + ["bucket_refilled_at", "0"], + ]), + ); + + const { effectiveRate } = runAdmit(store, { + now, + targetRateLimit, + recoveryPeriodMs, + probeRateLimit, + }); + const expectedRate = + probeRateLimit + progress * (targetRateLimit - probeRateLimit); + expect(effectiveRate).toBeCloseTo(expectedRate, 5); + }); + + it("uses full targetRate at recovery boundary (progress=1)", () => { + const store = createRedisStore(); + const progress = 1; + const now = switchedAt + recoveryPeriodMs * progress; + + store.set( + "ep:t1", + new Map([ + ["is_open", "0"], + ["switched_at", switchedAt.toString()], + ["bucket_tokens", "0"], + ["bucket_refilled_at", "0"], + ]), + ); + + const { effectiveRate } = runAdmit(store, { + now, + targetRateLimit, + recoveryPeriodMs, + probeRateLimit, + }); + + expect(effectiveRate).toBe(targetRateLimit); + }); + + it("uses full rate when closed and past recovery period", () => { + const store = createRedisStore(); + const now = switchedAt + recoveryPeriodMs + 1; + + store.set( + "ep:t1", + new Map([ + ["is_open", "0"], + ["switched_at", switchedAt.toString()], + ["bucket_tokens", "0"], + ["bucket_refilled_at", "0"], + ]), + ); + + const { effectiveRate } = runAdmit(store, { + now, + targetRateLimit, + recoveryPeriodMs, + }); + expect(effectiveRate).toBe(targetRateLimit); + }); + }); + + describe("circuit breaker disabled", () => { + it("uses full targetRateLimit on a fresh endpoint (no cautious probe startup without CB)", () => { + const store = createRedisStore(); + const now = 1_000_000; + + const { effectiveRate } = runAdmit(store, { + now, + targetRateLimit: 10, + cbEnabled: false, + }); + + expect(effectiveRate).toBe(10); + }); + + it("applies initial values on fresh endpoint so first call has no tokens", () => { + const store = createRedisStore(); + const now = 1_000_000; + + const { consumedTokens, effectiveRate, reason } = runAdmit(store, { + now, + targetRateLimit: 10, + cbEnabled: false, + }); + + expect(effectiveRate).toBe(10); + expect(consumedTokens).toBe(0); + expect(reason).toBe("rate_limited"); + }); + + it("ignores is_open state", () => { + const store = createRedisStore(); + const now = 1_000_000; + store.set( + "ep:t1", + new Map([ + ["is_open", "1"], + ["switched_at", now.toString()], + ["bucket_tokens", "5"], + ["bucket_refilled_at", now.toString()], + ]), + ); + + const { consumedTokens, effectiveRate, reason } = runAdmit(store, { + now, + targetRateLimit: 10, + cbEnabled: false, + }); + + expect(effectiveRate).toBe(10); + expect(consumedTokens).toBe(1); + expect(reason).toBe("some_allowed"); + }); + + it("does not zero bucket tokens when is_open", () => { + const store = createRedisStore(); + const now = 1_000_000; + store.set( + "ep:t1", + new Map([ + ["is_open", "1"], + ["switched_at", now.toString()], + ["bucket_tokens", "5"], + ["bucket_refilled_at", now.toString()], + ]), + ); + + const { consumedTokens } = runAdmit(store, { + now, + targetRateLimit: 10, + cbEnabled: false, + targetBatchSize: 3, + }); + + expect(consumedTokens).toBe(3); + }); + + it("never returns circuit_open", () => { + const store = createRedisStore(); + const now = 1_000_000; + store.set( + "ep:t1", + new Map([ + ["is_open", "1"], + ["switched_at", (now - 10_000).toString()], + ["bucket_tokens", "0"], + ["bucket_refilled_at", now.toString()], + ]), + ); + + const { reason } = runAdmit(store, { + now, + cooldownMs: 120_000, + cbEnabled: false, + }); + + expect(reason).not.toBe("circuit_open"); + }); + + it("uses full rate during recovery period", () => { + const store = createRedisStore(); + const now = 1_000_000; + store.set( + "ep:t1", + new Map([ + ["is_open", "0"], + ["switched_at", (now - 1000).toString()], + ["bucket_tokens", "0"], + ["bucket_refilled_at", "0"], + ]), + ); + + const { effectiveRate } = runAdmit(store, { + now, + targetRateLimit: 10, + recoveryPeriodMs: 600_000, + cbEnabled: false, + }); + + expect(effectiveRate).toBe(10); + }); + + it("returns rate_limited on fresh endpoint unlike cbEnabled=true which grants a probe token", () => { + const store = createRedisStore(); + const now = 1_000_000; + + const withCbDisabled = runAdmit(store, { + now, + targetRateLimit: 10, + cbEnabled: false, + }); + + const freshStore = createRedisStore(); + const withCbEnabled = runAdmit(freshStore, { + now, + targetRateLimit: 10, + cbEnabled: true, + }); + + expect(withCbDisabled.consumedTokens).toBe(0); + expect(withCbDisabled.reason).toBe("rate_limited"); + expect(withCbEnabled.consumedTokens).toBe(1); + expect(withCbEnabled.reason).toBe("some_allowed"); + }); + + it("generates tokens at full rate after initial contact", () => { + const store = createRedisStore(); + + runAdmit(store, { + now: 1_000_000, + targetRateLimit: 10, + cbEnabled: false, + }); + const { consumedTokens, reason } = runAdmit(store, { + now: 1_000_100, + targetRateLimit: 10, + cbEnabled: false, + }); + + expect(consumedTokens).toBe(1); + expect(reason).toBe("some_allowed"); + }); + }); + + describe("state persistence", () => { + it("persists bucket_tokens and bucket_refilled_at", () => { + const store = createRedisStore(); + const now = 1_000_000; + store.set( + "ep:t1", + new Map([ + ["is_open", "0"], + ["bucket_tokens", "5"], + ["bucket_refilled_at", now.toString()], + ["switched_at", "0"], + ]), + ); + + runAdmit(store, { now, targetBatchSize: 2 }); + + const epHash = store.get("ep:t1")!; + expect(Number(epHash.get("bucket_tokens"))).toBe(3); + }); + + it("does not write any fields when circuit_open early return", () => { + const store = createRedisStore(); + runAdmit(store, { + now: 10_000, + }); + + expect(store.has("ep:t1")).toBe(false); + }); + + it("does not write sampling or circuit fields on half-open path", () => { + const store = createRedisStore(); + runAdmit(store, { + now: 200_000, + }); + + const epHash = store.get("ep:t1")!; + expect(epHash.has("bucket_tokens")).toBe(true); + expect(epHash.has("bucket_refilled_at")).toBe(true); + expect(epHash.has("cur_attempts")).toBe(false); + expect(epHash.has("cur_failures")).toBe(false); + expect(epHash.has("sample_till")).toBe(false); + expect(epHash.has("is_open")).toBe(false); + expect(epHash.has("switched_at")).toBe(false); + }); + + it("isolates state between targets", () => { + const store = createRedisStore(); + store.set( + "ep:target-a", + new Map([ + ["is_open", "0"], + ["bucket_tokens", "5"], + ["bucket_refilled_at", "10000"], + ]), + ); + store.set( + "ep:target-b", + new Map([ + ["is_open", "0"], + ["bucket_tokens", "3"], + ["bucket_refilled_at", "10000"], + ]), + ); + + runAdmit(store, { now: 10_000 }, "target-a"); + runAdmit(store, { now: 10_000 }, "target-b"); + + expect(store.has("ep:target-a")).toBe(true); + expect(store.has("ep:target-b")).toBe(true); + }); + }); +}); diff --git a/lambdas/https-client-lambda/src/__tests__/applications-map.test.ts b/lambdas/https-client-lambda/src/__tests__/applications-map.test.ts new file mode 100644 index 00000000..23193ad9 --- /dev/null +++ b/lambdas/https-client-lambda/src/__tests__/applications-map.test.ts @@ -0,0 +1,117 @@ +import { GetObjectCommand } from "@aws-sdk/client-s3"; +import type { SdkStream } from "@smithy/types"; + +import { getApplicationId, resetCache } from "services/applications-map"; + +const mockSend = jest.fn(); +jest.mock("@aws-sdk/client-s3", () => { + const actual = jest.requireActual("@aws-sdk/client-s3"); + return { + ...actual, + S3Client: jest.fn().mockImplementation(() => ({ + send: (...args: unknown[]) => mockSend(...args), + })), + }; +}); + +jest.mock("@nhs-notify-client-callbacks/logger", () => ({ + logger: { + info: jest.fn(), + warn: jest.fn(), + error: jest.fn(), + debug: jest.fn(), + }, +})); + +const mockBody = (content: string) => + ({ + transformToString: jest.fn().mockResolvedValue(content), + }) as unknown as SdkStream; + +describe("getApplicationId", () => { + beforeEach(() => { + mockSend.mockReset(); + resetCache(); + process.env.APPLICATIONS_MAP_S3_BUCKET = "test-bucket"; + process.env.APPLICATIONS_MAP_S3_KEY = "test/applications-map.json"; + }); + + afterEach(() => { + delete process.env.APPLICATIONS_MAP_S3_BUCKET; + delete process.env.APPLICATIONS_MAP_S3_KEY; + }); + + it("returns correct applicationId for a known clientId", async () => { + mockSend.mockResolvedValue({ + Body: mockBody( + JSON.stringify({ "client-1": "app-id-1", "client-2": "app-id-2" }), + ), + }); + + const result = await getApplicationId("client-1"); + + expect(result).toBe("app-id-1"); + expect(mockSend).toHaveBeenCalledTimes(1); + expect(mockSend.mock.calls[0][0]).toBeInstanceOf(GetObjectCommand); + }); + + it("throws for unknown clientId", async () => { + mockSend.mockResolvedValue({ + Body: mockBody(JSON.stringify({ "client-1": "app-id-1" })), + }); + + await expect(getApplicationId("unknown")).rejects.toThrow( + "No applicationId found for clientId 'unknown' in applications map", + ); + }); + + it("surfaces S3 SDK errors", async () => { + mockSend.mockRejectedValue(new Error("S3 unavailable")); + + await expect(getApplicationId("client-1")).rejects.toThrow( + "S3 unavailable", + ); + }); + + it("throws when env vars are not set", async () => { + let getFn: typeof getApplicationId; + delete process.env.APPLICATIONS_MAP_S3_BUCKET; + delete process.env.APPLICATIONS_MAP_S3_KEY; + + jest.isolateModules(() => { + // eslint-disable-next-line @typescript-eslint/no-require-imports -- jest.isolateModules requires synchronous require + getFn = require("services/applications-map").getApplicationId; + }); + + await expect(getFn!("client-1")).rejects.toThrow( + "APPLICATIONS_MAP_S3_BUCKET and APPLICATIONS_MAP_S3_KEY are required", + ); + }); + + it("throws when S3 object body is empty", async () => { + mockSend.mockResolvedValue({ Body: undefined }); + + await expect(getApplicationId("client-1")).rejects.toThrow("is empty"); + }); + + it("throws when S3 object contains invalid JSON", async () => { + mockSend.mockResolvedValue({ + Body: mockBody("not-json"), + }); + + await expect(getApplicationId("client-1")).rejects.toThrow( + "contains invalid JSON", + ); + }); + + it("caches the applications map between calls", async () => { + mockSend.mockResolvedValue({ + Body: mockBody(JSON.stringify({ "client-1": "app-id-1" })), + }); + + await getApplicationId("client-1"); + await getApplicationId("client-1"); + + expect(mockSend).toHaveBeenCalledTimes(1); + }); +}); diff --git a/lambdas/https-client-lambda/src/__tests__/config-loader.test.ts b/lambdas/https-client-lambda/src/__tests__/config-loader.test.ts new file mode 100644 index 00000000..f635aaa0 --- /dev/null +++ b/lambdas/https-client-lambda/src/__tests__/config-loader.test.ts @@ -0,0 +1,209 @@ +import { GetObjectCommand } from "@aws-sdk/client-s3"; +import { ConfigSubscriptionCache } from "@nhs-notify-client-callbacks/config-subscription-cache"; + +import { loadTargetConfig, resetCache } from "services/config-loader"; + +const mockS3Send = jest.fn(); +jest.mock("@aws-sdk/client-s3", () => { + const actual = jest.requireActual("@aws-sdk/client-s3"); + return { + ...actual, + S3Client: jest.fn().mockImplementation(() => ({ + send: (...args: unknown[]) => mockS3Send(...args), + })), + }; +}); + +jest.mock("@nhs-notify-client-callbacks/logger", () => ({ + logger: { + info: jest.fn(), + warn: jest.fn(), + error: jest.fn(), + debug: jest.fn(), + }, +})); + +const VALID_TARGET = { + targetId: "target-1", + type: "API" as const, + invocationEndpoint: "https://webhook.example.invalid", + invocationMethod: "POST" as const, + invocationRateLimit: 10, + apiKey: { headerName: "x-api-key", headerValue: "secret" }, +}; + +const VALID_CONFIG = { + clientId: "client-1", + subscriptions: [], + targets: [VALID_TARGET], +}; + +const makeS3Response = (body: unknown) => ({ + Body: { + transformToString: jest.fn().mockResolvedValue(JSON.stringify(body)), + }, +}); + +describe("loadTargetConfig", () => { + beforeEach(() => { + mockS3Send.mockReset(); + resetCache(); + process.env.CLIENT_SUBSCRIPTION_CONFIG_BUCKET = "test-bucket"; + process.env.CLIENT_SUBSCRIPTION_CONFIG_PREFIX = "client_subscriptions/"; + process.env.CLIENT_SUBSCRIPTION_CACHE_TTL_SECONDS = "1"; + }); + + afterEach(() => { + delete process.env.CLIENT_SUBSCRIPTION_CONFIG_BUCKET; + delete process.env.CLIENT_SUBSCRIPTION_CONFIG_PREFIX; + delete process.env.CLIENT_SUBSCRIPTION_CACHE_TTL_SECONDS; + }); + + it("parses valid S3 config and returns the matching target", async () => { + mockS3Send.mockResolvedValue(makeS3Response(VALID_CONFIG)); + + const result = await loadTargetConfig("client-1", "target-1"); + + expect(result).toEqual(VALID_TARGET); + expect(mockS3Send).toHaveBeenCalledTimes(1); + expect(mockS3Send.mock.calls[0][0]).toBeInstanceOf(GetObjectCommand); + }); + + it("uses CLIENT_SUBSCRIPTION_CONFIG_PREFIX for the S3 key", async () => { + mockS3Send.mockResolvedValue(makeS3Response(VALID_CONFIG)); + + await loadTargetConfig("client-1", "target-1"); + + const command: GetObjectCommand = mockS3Send.mock.calls[0][0]; + expect(command.input.Key).toBe("client_subscriptions/client-1.json"); + }); + + it("rejects config missing required field", async () => { + const invalidConfig = { + ...VALID_CONFIG, + targets: [ + { + type: VALID_TARGET.type, + invocationEndpoint: VALID_TARGET.invocationEndpoint, + invocationMethod: VALID_TARGET.invocationMethod, + invocationRateLimit: VALID_TARGET.invocationRateLimit, + apiKey: VALID_TARGET.apiKey, + }, + ], + }; + mockS3Send.mockResolvedValue(makeS3Response(invalidConfig)); + + await expect(loadTargetConfig("client-1", "target-1")).rejects.toThrow( + "Invalid client config for 'client-1'", + ); + }); + + it("returns cached value without S3 call on subsequent requests", async () => { + mockS3Send.mockResolvedValue(makeS3Response(VALID_CONFIG)); + + await loadTargetConfig("client-1", "target-1"); + await loadTargetConfig("client-1", "target-1"); + + expect(mockS3Send).toHaveBeenCalledTimes(1); + }); + + it("re-fetches from S3 after TTL expiry", async () => { + jest.useFakeTimers(); + jest.setSystemTime(new Date("2026-01-01T10:00:00Z")); + + mockS3Send.mockResolvedValue(makeS3Response(VALID_CONFIG)); + + await loadTargetConfig("client-1", "target-1"); + + jest.advanceTimersByTime(1001); + + await loadTargetConfig("client-1", "target-1"); + + expect(mockS3Send).toHaveBeenCalledTimes(2); + + jest.useRealTimers(); + }); + + it("throws when CLIENT_SUBSCRIPTION_CONFIG_BUCKET is not set", async () => { + let loadFn: typeof loadTargetConfig; + delete process.env.CLIENT_SUBSCRIPTION_CONFIG_BUCKET; + + jest.isolateModules(() => { + // eslint-disable-next-line @typescript-eslint/no-require-imports -- jest.isolateModules requires synchronous require + loadFn = require("services/config-loader").loadTargetConfig; + }); + + await expect(loadFn!("client-1", "target-1")).rejects.toThrow( + "CLIENT_SUBSCRIPTION_CONFIG_BUCKET is required", + ); + }); + + it("throws when S3 response body is empty", async () => { + mockS3Send.mockResolvedValue({ Body: undefined }); + + await expect(loadTargetConfig("client-1", "target-1")).rejects.toThrow( + "S3 response body was empty for client 'client-1'", + ); + }); + + it("throws when client config is not found", async () => { + mockS3Send.mockResolvedValue(makeS3Response(null)); + + await expect( + loadTargetConfig("unknown-client", "target-1"), + ).rejects.toThrow("Invalid client config for 'unknown-client'"); + }); + + it("throws when target not found in config", async () => { + mockS3Send.mockResolvedValue(makeS3Response(VALID_CONFIG)); + + await expect(loadTargetConfig("client-1", "nonexistent")).rejects.toThrow( + "Target 'nonexistent' not found in config for client 'client-1'", + ); + }); + + it("uses default prefix when CLIENT_SUBSCRIPTION_CONFIG_PREFIX is not set", async () => { + delete process.env.CLIENT_SUBSCRIPTION_CONFIG_PREFIX; + resetCache(); + mockS3Send.mockResolvedValue(makeS3Response(VALID_CONFIG)); + + await loadTargetConfig("client-1", "target-1"); + + const command: GetObjectCommand = mockS3Send.mock.calls[0][0]; + expect(command.input.Key).toBe("client_subscriptions/client-1.json"); + }); + + it("uses default TTL when CLIENT_SUBSCRIPTION_CACHE_TTL_SECONDS is not set", async () => { + delete process.env.CLIENT_SUBSCRIPTION_CACHE_TTL_SECONDS; + resetCache(); + mockS3Send.mockResolvedValue(makeS3Response(VALID_CONFIG)); + + const result = await loadTargetConfig("client-1", "target-1"); + + expect(result).toEqual(VALID_TARGET); + }); + + it("throws when loadClientConfig resolves to undefined", async () => { + const spy = jest + .spyOn(ConfigSubscriptionCache.prototype, "loadClientConfig") + .mockResolvedValueOnce(undefined); + + await expect(loadTargetConfig("client-1", "target-1")).rejects.toThrow( + "No configuration found for client 'client-1'", + ); + + spy.mockRestore(); + }); + + it("propagates S3 NoSuchKey error from loadClientConfig", async () => { + const noSuchKeyError = Object.assign(new Error("NoSuchKey"), { + name: "NoSuchKey", + }); + mockS3Send.mockRejectedValue(noSuchKeyError); + resetCache(); + + await expect( + loadTargetConfig("unknown-client", "target-1"), + ).rejects.toThrow("NoSuchKey"); + }); +}); diff --git a/lambdas/https-client-lambda/src/__tests__/delivery-metrics.test.ts b/lambdas/https-client-lambda/src/__tests__/delivery-metrics.test.ts new file mode 100644 index 00000000..e5619199 --- /dev/null +++ b/lambdas/https-client-lambda/src/__tests__/delivery-metrics.test.ts @@ -0,0 +1,239 @@ +const mockCreateMetricsLogger = jest.fn(); +jest.mock("aws-embedded-metrics", () => ({ + Unit: { Count: "Count", Milliseconds: "Milliseconds" }, + StorageResolution: { High: 1, Standard: 60 }, + createMetricsLogger: () => mockCreateMetricsLogger(), +})); + +describe("delivery-metrics", () => { + const mockMetrics = { + setNamespace: jest.fn(), + setDimensions: jest.fn(), + setProperty: jest.fn(), + putMetric: jest.fn(), + flush: jest.fn().mockResolvedValue(undefined), + }; + + let mod: typeof import("services/delivery-metrics"); + + beforeEach(async () => { + jest.resetModules(); + jest.clearAllMocks(); + mockCreateMetricsLogger.mockReturnValue(mockMetrics); + process.env.METRICS_NAMESPACE = "TestNamespace"; + process.env.ENVIRONMENT = "test"; + process.env.CLIENT_ID = "client-1"; + // @ts-expect-error -- modulePaths resolves at runtime + mod = await import("services/delivery-metrics"); + }); + + afterEach(() => { + delete process.env.METRICS_NAMESPACE; + delete process.env.ENVIRONMENT; + delete process.env.CLIENT_ID; + }); + + it("throws when METRICS_NAMESPACE is not set", async () => { + delete process.env.METRICS_NAMESPACE; + jest.resetModules(); + // @ts-expect-error -- modulePaths resolves at runtime + const { emitDeliveryAttempt } = await import("services/delivery-metrics"); + + expect(() => emitDeliveryAttempt("t-1")).toThrow( + "METRICS_NAMESPACE environment variable is not set", + ); + }); + + it("throws when ENVIRONMENT is not set", async () => { + delete process.env.ENVIRONMENT; + jest.resetModules(); + // @ts-expect-error -- modulePaths resolves at runtime + const { emitDeliveryAttempt } = await import("services/delivery-metrics"); + + expect(() => emitDeliveryAttempt("t-1")).toThrow( + "ENVIRONMENT environment variable is not set", + ); + }); + + it("throws when CLIENT_ID is not set", async () => { + delete process.env.CLIENT_ID; + jest.resetModules(); + // @ts-expect-error -- modulePaths resolves at runtime + const { emitDeliveryAttempt } = await import("services/delivery-metrics"); + + expect(() => emitDeliveryAttempt("t-1")).toThrow( + "CLIENT_ID environment variable is not set", + ); + }); + + it("creates metrics logger with correct namespace and dimensions", () => { + mod.emitDeliveryAttempt("t-1"); + + expect(mockMetrics.setNamespace).toHaveBeenCalledWith("TestNamespace"); + expect(mockMetrics.setDimensions).toHaveBeenCalledWith({ + Environment: "test", + ClientId: "client-1", + }); + }); + + it("caches the metrics logger on subsequent calls", () => { + mod.emitDeliveryAttempt("t-1"); + mod.emitDeliverySuccess("t-1"); + + expect(mockCreateMetricsLogger).toHaveBeenCalledTimes(1); + }); + + it("emitDeliveryAttempt emits correct metric", () => { + mod.emitDeliveryAttempt("target-42"); + + expect(mockMetrics.setProperty).toHaveBeenCalledWith( + "targetId", + "target-42", + ); + expect(mockMetrics.putMetric).toHaveBeenCalledWith( + "DeliveryAttempt", + 1, + "Count", + 1, + ); + }); + + it("emitDeliverySuccess emits correct metric", () => { + mod.emitDeliverySuccess("target-42"); + + expect(mockMetrics.putMetric).toHaveBeenCalledWith( + "DeliverySuccess", + 1, + "Count", + 1, + ); + }); + + it("emitDeliveryFailure emits correct metric", () => { + mod.emitDeliveryFailure("target-42"); + + expect(mockMetrics.putMetric).toHaveBeenCalledWith( + "DeliveryFailure", + 1, + "Count", + 1, + ); + }); + + it("emitDeliveryPermanentFailure emits correct metric", () => { + mod.emitDeliveryPermanentFailure("target-42"); + + expect(mockMetrics.putMetric).toHaveBeenCalledWith( + "DeliveryPermanentFailure", + 1, + "Count", + 60, + ); + }); + + it("emitCircuitBreakerOpen emits correct metric", () => { + mod.emitCircuitBreakerOpen("target-42"); + + expect(mockMetrics.putMetric).toHaveBeenCalledWith( + "CircuitBreakerOpen", + 1, + "Count", + 60, + ); + }); + + it("emitServerRateLimited emits correct metric", () => { + mod.emitServerRateLimited("target-42"); + + expect(mockMetrics.putMetric).toHaveBeenCalledWith( + "DeliveryServerRateLimited", + 1, + "Count", + 1, + ); + }); + + it("emitCircuitBreakerClosed emits correct metric", () => { + mod.emitCircuitBreakerClosed("target-42"); + + expect(mockMetrics.putMetric).toHaveBeenCalledWith( + "CircuitBreakerClosed", + 1, + "Count", + 60, + ); + }); + + it("emitRetryWindowExhausted emits correct metric", () => { + mod.emitRetryWindowExhausted("target-42"); + + expect(mockMetrics.putMetric).toHaveBeenCalledWith( + "DeliveryRetryWindowExhausted", + 1, + "Count", + 60, + ); + }); + + it("emitClientRateLimited emits correct metric", () => { + mod.emitClientRateLimited("target-42", 3); + + expect(mockMetrics.setProperty).toHaveBeenCalledWith( + "targetId", + "target-42", + ); + expect(mockMetrics.putMetric).toHaveBeenCalledWith( + "DeliveryRateLimited", + 3, + "Count", + 1, + ); + }); + + it("emitCircuitBlocked emits correct metric", () => { + mod.emitCircuitBlocked("target-42", 2); + + expect(mockMetrics.setProperty).toHaveBeenCalledWith( + "targetId", + "target-42", + ); + expect(mockMetrics.putMetric).toHaveBeenCalledWith( + "DeliveryCircuitBlocked", + 2, + "Count", + 1, + ); + }); + + it("emitDeliveryDuration emits correct metric", () => { + mod.emitDeliveryDuration("target-42", 250); + + expect(mockMetrics.putMetric).toHaveBeenCalledWith( + "DeliveryDurationMs", + 250, + "Milliseconds", + 1, + ); + }); + + it("flushMetrics calls flush on the instance", async () => { + mod.emitDeliveryAttempt("t-1"); + await mod.flushMetrics(); + + expect(mockMetrics.flush).toHaveBeenCalled(); + }); + + it("flushMetrics does nothing when no metrics instance exists", async () => { + await mod.flushMetrics(); + + expect(mockMetrics.flush).not.toHaveBeenCalled(); + }); + + it("resetMetrics clears the cached instance", () => { + mod.emitDeliveryAttempt("t-1"); + mod.resetMetrics(); + mod.emitDeliveryAttempt("t-2"); + + expect(mockCreateMetricsLogger).toHaveBeenCalledTimes(2); + }); +}); diff --git a/lambdas/https-client-lambda/src/__tests__/delivery-observability.test.ts b/lambdas/https-client-lambda/src/__tests__/delivery-observability.test.ts new file mode 100644 index 00000000..651298c6 --- /dev/null +++ b/lambdas/https-client-lambda/src/__tests__/delivery-observability.test.ts @@ -0,0 +1,277 @@ +import { + recordAdmissionDenied, + recordCircuitBreakerClosed, + recordCircuitBreakerOpen, + recordDeliveryAttempt, + recordDeliveryDuration, + recordDeliveryFailure, + recordDeliveryPermanentFailure, + recordDeliveryRateLimited, + recordDeliverySuccess, + recordRetryWindowExhausted, +} from "services/delivery-observability"; + +jest.mock("@nhs-notify-client-callbacks/logger", () => ({ + logger: { + info: jest.fn(), + warn: jest.fn(), + error: jest.fn(), + }, +})); + +jest.mock("services/delivery-metrics", () => ({ + emitCircuitBlocked: jest.fn(), + emitCircuitBreakerClosed: jest.fn(), + emitCircuitBreakerOpen: jest.fn(), + emitClientRateLimited: jest.fn(), + emitDeliveryAttempt: jest.fn(), + emitDeliveryDuration: jest.fn(), + emitDeliveryFailure: jest.fn(), + emitDeliveryPermanentFailure: jest.fn(), + emitDeliverySuccess: jest.fn(), + emitServerRateLimited: jest.fn(), + emitRetryWindowExhausted: jest.fn(), +})); + +describe("delivery-observability", () => { + it("recordDeliveryAttempt emits metric and logs", () => { + const { emitDeliveryAttempt } = jest.requireMock( + "services/delivery-metrics", + ); + const { logger } = jest.requireMock("@nhs-notify-client-callbacks/logger"); + + recordDeliveryAttempt("client-1", "target-1", "msg-123", "sqs-msg-1", 3); + + expect(emitDeliveryAttempt).toHaveBeenCalledWith("target-1"); + expect(logger.info).toHaveBeenCalledWith( + "Attempting delivery", + expect.objectContaining({ + clientId: "client-1", + targetId: "target-1", + correlationId: "msg-123", + sqsMessageId: "sqs-msg-1", + receiveCount: 3, + }), + ); + }); + + it("recordDeliverySuccess emits metric and logs", () => { + const { emitDeliverySuccess } = jest.requireMock( + "services/delivery-metrics", + ); + const { logger } = jest.requireMock("@nhs-notify-client-callbacks/logger"); + + recordDeliverySuccess("client-1", "target-1", "msg-123"); + + expect(emitDeliverySuccess).toHaveBeenCalledWith("target-1"); + expect(logger.info).toHaveBeenCalledWith( + "Delivery succeeded", + expect.objectContaining({ + clientId: "client-1", + targetId: "target-1", + correlationId: "msg-123", + }), + ); + }); + + it("recordDeliveryPermanentFailure emits metric and logs warning", () => { + const { emitDeliveryPermanentFailure } = jest.requireMock( + "services/delivery-metrics", + ); + const { logger } = jest.requireMock("@nhs-notify-client-callbacks/logger"); + + recordDeliveryPermanentFailure( + "client-1", + "target-1", + undefined, + undefined, + "msg-123", + ); + + expect(emitDeliveryPermanentFailure).toHaveBeenCalledWith("target-1"); + expect(logger.warn).toHaveBeenCalledWith( + "Permanent delivery failure \u2014 sending to DLQ", + expect.objectContaining({ + clientId: "client-1", + targetId: "target-1", + correlationId: "msg-123", + }), + ); + }); + + it("recordDeliveryPermanentFailure includes statusCode and errorCode when provided", () => { + const { emitDeliveryPermanentFailure } = jest.requireMock( + "services/delivery-metrics", + ); + const { logger } = jest.requireMock("@nhs-notify-client-callbacks/logger"); + + recordDeliveryPermanentFailure( + "client-1", + "target-1", + 400, + "INVALID_PAYLOAD", + "msg-456", + ); + + expect(emitDeliveryPermanentFailure).toHaveBeenCalledWith("target-1"); + expect(logger.warn).toHaveBeenCalledWith( + "Permanent delivery failure \u2014 sending to DLQ", + expect.objectContaining({ + clientId: "client-1", + targetId: "target-1", + correlationId: "msg-456", + statusCode: 400, + errorCode: "INVALID_PAYLOAD", + }), + ); + }); + + it("recordDeliveryRateLimited emits metric and logs", () => { + const { emitServerRateLimited } = jest.requireMock( + "services/delivery-metrics", + ); + const { logger } = jest.requireMock("@nhs-notify-client-callbacks/logger"); + + recordDeliveryRateLimited("client-1", "target-1", "msg-123"); + + expect(emitServerRateLimited).toHaveBeenCalledWith("target-1"); + expect(logger.info).toHaveBeenCalledWith( + "Server rate limited (429)", + expect.objectContaining({ + clientId: "client-1", + targetId: "target-1", + correlationId: "msg-123", + }), + ); + }); + + it("recordDeliveryFailure emits metric and logs warning with context", () => { + const { emitDeliveryFailure } = jest.requireMock( + "services/delivery-metrics", + ); + const { logger } = jest.requireMock("@nhs-notify-client-callbacks/logger"); + + recordDeliveryFailure("client-1", "target-1", 503, 30, 3, "msg-123"); + + expect(emitDeliveryFailure).toHaveBeenCalledWith("target-1"); + expect(logger.warn).toHaveBeenCalledWith( + "Transient delivery failure \u2014 requeuing", + expect.objectContaining({ + clientId: "client-1", + targetId: "target-1", + correlationId: "msg-123", + statusCode: 503, + backoffSec: 30, + receiveCount: 3, + }), + ); + }); + + it("recordCircuitBreakerOpen emits metric and logs", () => { + const { emitCircuitBreakerOpen } = jest.requireMock( + "services/delivery-metrics", + ); + const { logger } = jest.requireMock("@nhs-notify-client-callbacks/logger"); + + recordCircuitBreakerOpen("target-1", "msg-123"); + + expect(emitCircuitBreakerOpen).toHaveBeenCalledWith("target-1"); + expect(logger.warn).toHaveBeenCalledWith( + "Circuit breaker opened", + expect.objectContaining({ + targetId: "target-1", + correlationId: "msg-123", + }), + ); + }); + + it("recordCircuitBreakerClosed emits metric and logs", () => { + const { emitCircuitBreakerClosed } = jest.requireMock( + "services/delivery-metrics", + ); + const { logger } = jest.requireMock("@nhs-notify-client-callbacks/logger"); + + recordCircuitBreakerClosed("target-1", "msg-123"); + + expect(emitCircuitBreakerClosed).toHaveBeenCalledWith("target-1"); + expect(logger.info).toHaveBeenCalledWith( + "Circuit breaker closed", + expect.objectContaining({ + targetId: "target-1", + correlationId: "msg-123", + }), + ); + }); + + it("recordRetryWindowExhausted emits metric and logs", () => { + const { emitRetryWindowExhausted } = jest.requireMock( + "services/delivery-metrics", + ); + const { logger } = jest.requireMock("@nhs-notify-client-callbacks/logger"); + + recordRetryWindowExhausted("client-1", "target-1", "msg-123"); + + expect(emitRetryWindowExhausted).toHaveBeenCalledWith("target-1"); + expect(logger.warn).toHaveBeenCalledWith( + "Retry window exhausted \u2014 sending to DLQ", + expect.objectContaining({ + clientId: "client-1", + targetId: "target-1", + correlationId: "msg-123", + }), + ); + }); + + it("recordAdmissionDenied emits rate limited metric for rate_limited reason", () => { + const { emitClientRateLimited } = jest.requireMock( + "services/delivery-metrics", + ); + const { logger } = jest.requireMock("@nhs-notify-client-callbacks/logger"); + + recordAdmissionDenied("client-1", "target-1", "rate_limited", [ + "msg-a", + "msg-b", + ]); + + expect(emitClientRateLimited).toHaveBeenCalledWith("target-1", 2); + expect(logger.warn).toHaveBeenCalledWith( + "Client rate limited", + expect.objectContaining({ + clientId: "client-1", + targetId: "target-1", + deniedCount: 2, + correlationIds: ["msg-a", "msg-b"], + }), + ); + }); + + it("recordAdmissionDenied emits circuit blocked metric for circuit_open reason", () => { + const { emitCircuitBlocked } = jest.requireMock( + "services/delivery-metrics", + ); + const { logger } = jest.requireMock("@nhs-notify-client-callbacks/logger"); + + recordAdmissionDenied("client-1", "target-1", "circuit_open", ["msg-a"]); + + expect(emitCircuitBlocked).toHaveBeenCalledWith("target-1", 1); + expect(logger.warn).toHaveBeenCalledWith( + "Circuit blocked", + expect.objectContaining({ + clientId: "client-1", + targetId: "target-1", + deniedCount: 1, + correlationIds: ["msg-a"], + }), + ); + }); + + it("recordDeliveryDuration emits metric", () => { + const { emitDeliveryDuration } = jest.requireMock( + "services/delivery-metrics", + ); + + recordDeliveryDuration("target-1", 250); + + expect(emitDeliveryDuration).toHaveBeenCalledWith("target-1", 250); + }); +}); diff --git a/lambdas/https-client-lambda/src/__tests__/dlq-sender.test.ts b/lambdas/https-client-lambda/src/__tests__/dlq-sender.test.ts new file mode 100644 index 00000000..cd2286e5 --- /dev/null +++ b/lambdas/https-client-lambda/src/__tests__/dlq-sender.test.ts @@ -0,0 +1,134 @@ +import { SendMessageCommand } from "@aws-sdk/client-sqs"; + +import { sendToDlq } from "services/dlq-sender"; + +const mockSend = jest.fn(); +jest.mock("@aws-sdk/client-sqs", () => { + const actual = jest.requireActual("@aws-sdk/client-sqs"); + return { + ...actual, + SQSClient: jest.fn().mockImplementation(() => ({ + send: (...args: unknown[]) => mockSend(...args), + })), + }; +}); + +describe("sendToDlq", () => { + beforeEach(() => { + mockSend.mockReset(); + process.env.DLQ_URL = "https://sqs.eu-west-2.invalid/123456789/test-dlq"; + }); + + afterEach(() => { + delete process.env.DLQ_URL; + }); + + it("sends SendMessageCommand with correct QueueUrl and MessageBody", async () => { + mockSend.mockResolvedValue({}); + + await sendToDlq('{"test":"message"}'); + + expect(mockSend).toHaveBeenCalledTimes(1); + const command = mockSend.mock.calls[0][0]; + expect(command).toBeInstanceOf(SendMessageCommand); + expect(command.input).toEqual({ + QueueUrl: "https://sqs.eu-west-2.invalid/123456789/test-dlq", + MessageBody: '{"test":"message"}', + }); + }); + + it("surfaces SDK errors", async () => { + mockSend.mockRejectedValue(new Error("SQS send failed")); + + await expect(sendToDlq("body")).rejects.toThrow("SQS send failed"); + }); + + it("throws when DLQ_URL is not set", async () => { + let sendFn: typeof sendToDlq; + delete process.env.DLQ_URL; + + jest.isolateModules(() => { + // eslint-disable-next-line @typescript-eslint/no-require-imports -- jest.isolateModules requires synchronous require + sendFn = require("services/dlq-sender").sendToDlq; + }); + + await expect(sendFn!("body")).rejects.toThrow("DLQ_URL is required"); + }); + + it("includes ERROR_CODE and ERROR_MESSAGE for HTTP error with JSON body", async () => { + mockSend.mockResolvedValue({}); + + await sendToDlq('{"test":"message"}', { + statusCode: 400, + responseBody: JSON.stringify({ message: "Bad request" }), + }); + + const command = mockSend.mock.calls[0][0]; + expect(command).toBeInstanceOf(SendMessageCommand); + expect(command.input.MessageAttributes).toEqual({ + ERROR_CODE: { DataType: "String", StringValue: "HTTP_CLIENT_ERROR" }, + ERROR_MESSAGE: { DataType: "String", StringValue: "Bad request" }, + }); + }); + + it("uses raw response body as ERROR_MESSAGE when not valid JSON", async () => { + mockSend.mockResolvedValue({}); + + await sendToDlq('{"test":"message"}', { + statusCode: 400, + responseBody: "Bad request", + }); + + const command = mockSend.mock.calls[0][0]; + expect(command.input.MessageAttributes).toEqual({ + ERROR_CODE: { DataType: "String", StringValue: "HTTP_CLIENT_ERROR" }, + ERROR_MESSAGE: { DataType: "String", StringValue: "Bad request" }, + }); + }); + + it("uses errorCode as ERROR_CODE when provided", async () => { + mockSend.mockResolvedValue({}); + + await sendToDlq('{"test":"message"}', { + errorCode: "CERT_HAS_EXPIRED", + }); + + const command = mockSend.mock.calls[0][0]; + expect(command.input.MessageAttributes).toEqual({ + ERROR_CODE: { DataType: "String", StringValue: "CERT_HAS_EXPIRED" }, + }); + }); + + it("sends empty MessageAttributes when errorInfo has no relevant fields", async () => { + mockSend.mockResolvedValue({}); + + await sendToDlq('{"test":"message"}', {}); + + const command = mockSend.mock.calls[0][0]; + expect(command.input.MessageAttributes).toEqual({}); + }); + + it("sends no MessageAttributes when errorInfo is omitted", async () => { + mockSend.mockResolvedValue({}); + + await sendToDlq('{"test":"message"}'); + + const command = mockSend.mock.calls[0][0]; + expect(command.input.MessageAttributes).toBeUndefined(); + }); + + it("uses JSON body message field when present in responseBody", async () => { + mockSend.mockResolvedValue({}); + + await sendToDlq('{"test":"message"}', { + statusCode: 422, + responseBody: JSON.stringify({ message: "Validation failed", code: 42 }), + }); + + const command = mockSend.mock.calls[0][0]; + expect(command.input.MessageAttributes?.ERROR_MESSAGE).toEqual({ + DataType: "String", + StringValue: "Validation failed", + }); + }); +}); diff --git a/lambdas/https-client-lambda/src/__tests__/endpoint-gate.test.ts b/lambdas/https-client-lambda/src/__tests__/endpoint-gate.test.ts new file mode 100644 index 00000000..901307dc --- /dev/null +++ b/lambdas/https-client-lambda/src/__tests__/endpoint-gate.test.ts @@ -0,0 +1,308 @@ +import { + type EndpointGateConfig, + admit, + recordResult, + resetAdmitSha, +} from "services/endpoint-gate"; + +const mockSendCommand = jest.fn(); +const mockConnect = jest.fn().mockResolvedValue(undefined); +const mockDisconnect = jest.fn().mockResolvedValue(undefined); +const mockOn = jest.fn(); + +const defaultConfig: EndpointGateConfig = { + burstCapacity: 2250, + probeRateLimit: 1 / 60, + recoveryPeriodMs: 600_000, + samplePeriodMs: 300_000, + failureThreshold: 0.3, + minAttempts: 5, + cooldownPeriodMs: 120_000, +}; + +const mockRedis = { + sendCommand: mockSendCommand, + connect: mockConnect, + disconnect: mockDisconnect, + on: mockOn, + isOpen: true, +} as never; + +beforeEach(() => { + jest.clearAllMocks(); + resetAdmitSha(); +}); + +describe("admit", () => { + it("returns allowed with consumedTokens when tokens available", async () => { + mockSendCommand.mockResolvedValueOnce([5, "some_allowed", 0, 10]); + + const result = await admit( + mockRedis, + "target-1", + 10, + true, + 5, + defaultConfig, + ); + + expect(result).toEqual({ + allowed: true, + consumedTokens: 5, + effectiveRate: 10, + }); + expect(mockSendCommand).toHaveBeenCalledWith( + expect.arrayContaining(["EVALSHA"]), + ); + }); + + it("returns rate_limited when tokens exhausted", async () => { + mockSendCommand.mockResolvedValueOnce([0, "rate_limited", 1000, 10]); + + const result = await admit( + mockRedis, + "target-1", + 10, + false, + 5, + defaultConfig, + ); + + expect(result).toEqual({ + allowed: false, + reason: "rate_limited", + retryAfterMs: 1000, + effectiveRate: 10, + }); + }); + + it("returns circuit_open when circuit is fully open", async () => { + mockSendCommand.mockResolvedValueOnce([0, "circuit_open", 30_000, 0]); + + const result = await admit( + mockRedis, + "target-1", + 10, + true, + 5, + defaultConfig, + ); + + expect(result).toEqual({ + allowed: false, + reason: "circuit_open", + retryAfterMs: 30_000, + effectiveRate: 0, + }); + }); + + it("falls back to EVAL on NOSCRIPT error", async () => { + mockSendCommand + .mockRejectedValueOnce(new Error("NOSCRIPT No matching script")) + .mockResolvedValueOnce([1, "some_allowed", 0, 10]); + + const result = await admit( + mockRedis, + "target-1", + 10, + true, + 1, + defaultConfig, + ); + + expect(result).toEqual({ + allowed: true, + consumedTokens: 1, + effectiveRate: 10, + }); + expect(mockSendCommand).toHaveBeenCalledTimes(2); + expect(mockSendCommand).toHaveBeenNthCalledWith( + 1, + expect.arrayContaining(["EVALSHA"]), + ); + expect(mockSendCommand).toHaveBeenNthCalledWith( + 2, + expect.arrayContaining(["EVAL"]), + ); + }); + + it("passes cbEnabled=0 when circuit breaker is disabled", async () => { + mockSendCommand.mockResolvedValueOnce([1, "some_allowed", 0, 10]); + + await admit(mockRedis, "target-1", 10, false, 1, defaultConfig); + + const args = mockSendCommand.mock.calls[0]![0] as string[]; + const cbEnabledArg = args[11]; + expect(cbEnabledArg).toBe("0"); + }); + + it("passes single epKey", async () => { + mockSendCommand.mockResolvedValueOnce([1, "some_allowed", 0, 5]); + + await admit(mockRedis, "my-target", 5, true, 1, defaultConfig); + + const args = mockSendCommand.mock.calls[0]![0] as string[]; + expect(args[3]).toBe("ep:{my-target}"); + }); + + it("passes targetBatchSize as ARGV", async () => { + mockSendCommand.mockResolvedValueOnce([3, "some_allowed", 0, 10]); + + await admit(mockRedis, "target-1", 10, true, 7, defaultConfig); + + const args = mockSendCommand.mock.calls[0]![0] as string[]; + const batchSizeArg = args[10]; + expect(batchSizeArg).toBe("7"); + }); +}); + +describe("evalScript", () => { + it("throws a wrapped error including the original message when EVALSHA fails with a non-NOSCRIPT Error", async () => { + const redisError = new Error("WRONGTYPE Operation against a key"); + mockSendCommand.mockRejectedValueOnce(redisError); + + const thrown = await admit( + mockRedis, + "target-1", + 10, + true, + 1, + defaultConfig, + ).catch((error: unknown) => error); + + expect(thrown).toBeInstanceOf(Error); + expect((thrown as Error).message).toContain("Redis error in script"); + expect((thrown as Error).message).toContain( + "WRONGTYPE Operation against a key", + ); + expect((thrown as Error & { cause: unknown }).cause).toBe(redisError); + }); + + it("throws a wrapped error using String() when EVALSHA rejects with a non-Error value", async () => { + mockSendCommand.mockRejectedValueOnce("connection refused"); + + const thrown = await admit( + mockRedis, + "target-1", + 10, + true, + 1, + defaultConfig, + ).catch((error: unknown) => error); + + expect(thrown).toBeInstanceOf(Error); + expect((thrown as Error).message).toContain("Redis error in script"); + expect((thrown as Error).message).toContain("connection refused"); + }); +}); + +describe("recordResult", () => { + it("returns closed state when circuit is steady-state", async () => { + mockSendCommand.mockResolvedValueOnce(["closed", 0]); + + const result = await recordResult( + mockRedis, + "target-1", + 5, + 0, + defaultConfig, + ); + + expect(result).toEqual({ circuitState: "closed", circuitSwitched: false }); + expect(mockSendCommand).toHaveBeenCalledWith( + expect.arrayContaining(["EVALSHA"]), + ); + }); + + it("returns open with circuitSwitched when failure crosses threshold", async () => { + mockSendCommand.mockResolvedValueOnce(["open", 1]); + + const result = await recordResult( + mockRedis, + "target-1", + 5, + 5, + defaultConfig, + ); + + expect(result).toEqual({ circuitState: "open", circuitSwitched: true }); + }); + + it("returns closed_recovery with circuitSwitched when circuit closes", async () => { + mockSendCommand.mockResolvedValueOnce(["closed_recovery", 1]); + + const result = await recordResult( + mockRedis, + "target-1", + 5, + 0, + defaultConfig, + ); + + expect(result).toEqual({ + circuitState: "closed_recovery", + circuitSwitched: true, + }); + }); + + it("returns open_half without circuitSwitched when probing", async () => { + mockSendCommand.mockResolvedValueOnce(["open_half", 0]); + + const result = await recordResult( + mockRedis, + "target-1", + 5, + 1, + defaultConfig, + ); + + expect(result).toEqual({ + circuitState: "open_half", + circuitSwitched: false, + }); + }); + + it("falls back to EVAL on NOSCRIPT error", async () => { + mockSendCommand + .mockRejectedValueOnce(new Error("NOSCRIPT No matching script")) + .mockResolvedValueOnce(["closed", 0]); + + const result = await recordResult( + mockRedis, + "target-1", + 1, + 0, + defaultConfig, + ); + + expect(result).toEqual({ circuitState: "closed", circuitSwitched: false }); + expect(mockSendCommand).toHaveBeenCalledTimes(2); + expect(mockSendCommand).toHaveBeenNthCalledWith( + 1, + expect.arrayContaining(["EVALSHA"]), + ); + expect(mockSendCommand).toHaveBeenNthCalledWith( + 2, + expect.arrayContaining(["EVAL"]), + ); + }); + + it("passes correct ep key for target", async () => { + mockSendCommand.mockResolvedValueOnce(["closed", 0]); + + await recordResult(mockRedis, "my-target", 1, 0, defaultConfig); + + const args = mockSendCommand.mock.calls[0]![0] as string[]; + expect(args[3]).toBe("ep:{my-target}"); + }); + + it("passes consumedTokens and processingFailures as ARGV", async () => { + mockSendCommand.mockResolvedValueOnce(["closed", 0]); + + await recordResult(mockRedis, "target-1", 8, 3, defaultConfig); + + const args = mockSendCommand.mock.calls[0]![0] as string[]; + expect(args[5]).toBe("8"); + expect(args[6]).toBe("3"); + }); +}); diff --git a/lambdas/https-client-lambda/src/__tests__/fixtures/handler-fixtures.ts b/lambdas/https-client-lambda/src/__tests__/fixtures/handler-fixtures.ts new file mode 100644 index 00000000..731d478a --- /dev/null +++ b/lambdas/https-client-lambda/src/__tests__/fixtures/handler-fixtures.ts @@ -0,0 +1,45 @@ +import type { SQSRecord } from "aws-lambda"; + +export const DEFAULT_TARGET = { + targetId: "target-1", + type: "API" as const, + invocationEndpoint: "https://webhook.example.invalid", + invocationMethod: "POST" as const, + invocationRateLimit: 10, + apiKey: { headerName: "x-api-key", headerValue: "secret-key" }, + delivery: { + mtls: { enabled: true }, + }, +}; + +export const makeRecord = (overrides: Partial = {}): SQSRecord => ({ + messageId: "msg-1", + receiptHandle: "receipt-1", + body: JSON.stringify({ + payload: { + data: [ + { + type: "MessageStatus", + attributes: { + messageId: "test-message-id", + messageStatus: "delivered", + }, + }, + ], + }, + subscriptionId: "sub-1", + targetId: "target-1", + }), + attributes: { + ApproximateReceiveCount: "1", + SentTimestamp: "0", + SenderId: "sender", + ApproximateFirstReceiveTimestamp: "0", + }, + messageAttributes: {}, + md5OfBody: "abc", + eventSource: "aws:sqs", + eventSourceARN: "arn:aws:sqs:eu-west-2:123:queue", + awsRegion: "eu-west-2", + ...overrides, +}); diff --git a/lambdas/https-client-lambda/src/__tests__/handler.test.ts b/lambdas/https-client-lambda/src/__tests__/handler.test.ts new file mode 100644 index 00000000..853d6e63 --- /dev/null +++ b/lambdas/https-client-lambda/src/__tests__/handler.test.ts @@ -0,0 +1,977 @@ +import { processRecords } from "handler"; +import { + DEFAULT_TARGET, + makeRecord, +} from "__tests__/fixtures/handler-fixtures"; + +jest.mock("@nhs-notify-client-callbacks/logger", () => ({ + logger: { + info: jest.fn(), + warn: jest.fn(), + error: jest.fn(), + debug: jest.fn(), + }, +})); + +const mockLoadTargetConfig = jest.fn(); +jest.mock("services/config-loader", () => ({ + loadTargetConfig: (...args: unknown[]) => mockLoadTargetConfig(...args), +})); + +const mockGetApplicationId = jest.fn(); +jest.mock("services/applications-map", () => ({ + getApplicationId: (...args: unknown[]) => mockGetApplicationId(...args), +})); + +const mockSignPayload = jest.fn(); +jest.mock("services/payload-signer", () => ({ + signPayload: (...args: unknown[]) => mockSignPayload(...args), +})); + +const mockBuildAgent = jest.fn(); +jest.mock("services/delivery/tls-agent-factory", () => ({ + buildAgent: (...args: unknown[]) => mockBuildAgent(...args), +})); + +const mockDeliverPayload = jest.fn(); +jest.mock("services/delivery/https-client", () => ({ + deliverPayload: (...args: unknown[]) => mockDeliverPayload(...args), + OUTCOME_SUCCESS: "success", + OUTCOME_PERMANENT_FAILURE: "permanent_failure", + OUTCOME_RATE_LIMITED: "rate_limited", + OUTCOME_TRANSIENT_FAILURE: "transient_failure", +})); + +const mockSendToDlq = jest.fn(); +jest.mock("services/dlq-sender", () => ({ + sendToDlq: (...args: unknown[]) => mockSendToDlq(...args), +})); + +const mockChangeVisibility = jest.fn(); +jest.mock("services/sqs-visibility", () => ({ + changeVisibility: (...args: unknown[]) => mockChangeVisibility(...args), +})); + +const mockJitteredBackoff = jest.fn(); +const mockIsWindowExhausted = jest.fn(); +const mockHandleRateLimitedRecord = jest.fn(); +jest.mock("services/delivery/retry-policy", () => ({ + jitteredBackoffSeconds: (...args: unknown[]) => mockJitteredBackoff(...args), + isWindowExhausted: (...args: unknown[]) => mockIsWindowExhausted(...args), + handleRateLimitedRecord: (...args: unknown[]) => + mockHandleRateLimitedRecord(...args), +})); + +const mockAdmit = jest.fn(); +const mockGetRedisClient = jest.fn(); +const mockRecordResult = jest.fn(); +jest.mock("services/endpoint-gate", () => ({ + admit: (...args: unknown[]) => mockAdmit(...args), + recordResult: (...args: unknown[]) => mockRecordResult(...args), +})); +jest.mock("services/redis-client", () => ({ + getRedisClient: (...args: unknown[]) => mockGetRedisClient(...args), +})); + +jest.mock("services/delivery-observability", () => ({ + recordAdmissionDenied: jest.fn(), + recordCircuitBreakerClosed: jest.fn(), + recordCircuitBreakerOpen: jest.fn(), + recordDeliveryAttempt: jest.fn(), + recordDeliveryDuration: jest.fn(), + recordDeliveryFailure: jest.fn(), + recordDeliveryPermanentFailure: jest.fn(), + recordDeliveryRateLimited: jest.fn(), + recordDeliverySuccess: jest.fn(), + recordRetryWindowExhausted: jest.fn(), +})); + +jest.mock("services/delivery-metrics", () => ({ + flushMetrics: jest.fn().mockResolvedValue(undefined), + resetMetrics: jest.fn(), +})); + +describe("processRecords", () => { + const mockAgent = {}; + + beforeEach(() => { + jest.clearAllMocks(); + process.env.CLIENT_ID = "client-1"; + mockLoadTargetConfig.mockResolvedValue(DEFAULT_TARGET); + mockGetApplicationId.mockResolvedValue("app-id-1"); + mockSignPayload.mockReturnValue("signature-abc"); + mockBuildAgent.mockResolvedValue(mockAgent); + mockDeliverPayload.mockResolvedValue({ outcome: "success" }); + mockSendToDlq.mockResolvedValue(undefined); + mockChangeVisibility.mockResolvedValue(undefined); + mockJitteredBackoff.mockReturnValue(5); + mockIsWindowExhausted.mockReturnValue(false); + mockHandleRateLimitedRecord.mockResolvedValue("retry"); + mockGetRedisClient.mockResolvedValue({}); + mockAdmit.mockResolvedValue({ + allowed: true, + consumedTokens: 100, + effectiveRate: 10, + }); + mockRecordResult.mockResolvedValue({ ok: true, state: "ok" }); + }); + + afterEach(() => { + delete process.env.CLIENT_ID; + }); + + describe("delivery outcomes", () => { + it("returns no failures on successful delivery", async () => { + const failures = await processRecords([makeRecord()]); + + expect(failures).toEqual([]); + expect(mockLoadTargetConfig).toHaveBeenCalledWith("client-1", "target-1"); + expect(mockGetApplicationId).toHaveBeenCalledWith("client-1"); + expect(mockSignPayload).toHaveBeenCalledWith( + "app-id-1", + "secret-key", + expect.objectContaining({ data: expect.any(Array) }), + ); + expect(mockBuildAgent).toHaveBeenCalledWith(DEFAULT_TARGET); + expect(mockDeliverPayload).toHaveBeenCalledWith( + DEFAULT_TARGET, + expect.any(String), + "signature-abc", + mockAgent, + ); + }); + + it("sends permanent failure to DLQ and returns no failure", async () => { + mockDeliverPayload.mockResolvedValue({ outcome: "permanent_failure" }); + + const failures = await processRecords([makeRecord()]); + + expect(failures).toEqual([]); + expect(mockSendToDlq).toHaveBeenCalledWith(makeRecord().body, { + outcome: "permanent_failure", + }); + }); + + it("returns failure for transient 5xx errors", async () => { + mockDeliverPayload.mockResolvedValue({ + outcome: "transient_failure", + statusCode: 503, + }); + + const failures = await processRecords([makeRecord()]); + + expect(failures).toEqual([{ itemIdentifier: "msg-1" }]); + }); + + it("returns failure for 429 when handleRateLimitedRecord rejects", async () => { + mockDeliverPayload.mockResolvedValue({ + outcome: "rate_limited", + retryAfterHeader: "60", + }); + + const failures = await processRecords([makeRecord()]); + + expect(failures).toEqual([{ itemIdentifier: "msg-1" }]); + expect(mockHandleRateLimitedRecord).toHaveBeenCalledWith( + makeRecord(), + "client-1", + "target-1", + "60", + 1, + ); + }); + + it("returns no failure when handleRateLimitedRecord resolves (e.g. DLQ path)", async () => { + mockDeliverPayload.mockResolvedValue({ + outcome: "rate_limited", + retryAfterHeader: "99999", + }); + mockHandleRateLimitedRecord.mockResolvedValue("dlq"); + + const failures = await processRecords([makeRecord()]); + + expect(failures).toEqual([]); + }); + + it("sends to DLQ when retry window is exhausted", async () => { + mockIsWindowExhausted.mockReturnValue(true); + + const failures = await processRecords([makeRecord()]); + + expect(failures).toEqual([]); + expect(mockSendToDlq).toHaveBeenCalledWith(makeRecord().body); + expect(mockDeliverPayload).not.toHaveBeenCalled(); + }); + }); + + describe("batch processing", () => { + it("processes multiple records in a single target batch", async () => { + const record1 = makeRecord({ messageId: "msg-1" }); + const record2 = makeRecord({ messageId: "msg-2" }); + + mockDeliverPayload + .mockResolvedValueOnce({ outcome: "success" }) + .mockResolvedValueOnce({ + outcome: "transient_failure", + statusCode: 500, + }); + + const failures = await processRecords([record1, record2]); + + expect(failures).toEqual([{ itemIdentifier: "msg-2" }]); + expect(mockAdmit).toHaveBeenCalledTimes(1); + }); + + it("delivers only admitted records when consumedTokens is less than batch size", async () => { + const record1 = makeRecord({ + messageId: "msg-1", + receiptHandle: "receipt-1", + }); + const record2 = makeRecord({ + messageId: "msg-2", + receiptHandle: "receipt-2", + }); + const record3 = makeRecord({ + messageId: "msg-3", + receiptHandle: "receipt-3", + }); + + mockAdmit.mockResolvedValue({ + allowed: true, + consumedTokens: 1, + effectiveRate: 10, + }); + + const { recordAdmissionDenied } = jest.requireMock( + "services/delivery-observability", + ); + + const failures = await processRecords([record1, record2, record3]); + + expect(mockDeliverPayload).toHaveBeenCalledTimes(1); + expect(failures).toEqual([ + { itemIdentifier: "msg-2" }, + { itemIdentifier: "msg-3" }, + ]); + expect(recordAdmissionDenied).toHaveBeenCalledWith( + "client-1", + "target-1", + "rate_limited", + ["test-message-id", "test-message-id"], + ); + + expect(mockChangeVisibility).toHaveBeenCalledWith("receipt-2", 1); + expect(mockChangeVisibility).toHaveBeenCalledWith("receipt-3", 1); + }); + + it("groups records by target and processes each batch separately", async () => { + const record1 = makeRecord({ messageId: "msg-1" }); + const record2 = makeRecord({ + messageId: "msg-2", + body: JSON.stringify({ + payload: { + data: [ + { + type: "MessageStatus", + attributes: { messageStatus: "delivered" }, + }, + ], + }, + subscriptionId: "sub-2", + targetId: "target-2", + }), + }); + + const failures = await processRecords([record1, record2]); + + expect(failures).toEqual([]); + expect(mockAdmit).toHaveBeenCalledTimes(2); + expect(mockLoadTargetConfig).toHaveBeenCalledWith("client-1", "target-1"); + expect(mockLoadTargetConfig).toHaveBeenCalledWith("client-1", "target-2"); + }); + }); + + describe("endpoint gate", () => { + it("requeues all records when rate limited by endpoint gate", async () => { + mockAdmit.mockResolvedValue({ + allowed: false, + reason: "rate_limited", + retryAfterMs: 2000, + effectiveRate: 10, + }); + + const failures = await processRecords([ + makeRecord({ messageId: "msg-1", receiptHandle: "receipt-1" }), + makeRecord({ messageId: "msg-2", receiptHandle: "receipt-2" }), + ]); + + expect(failures).toEqual([ + { itemIdentifier: "msg-1" }, + { itemIdentifier: "msg-2" }, + ]); + expect(mockChangeVisibility).toHaveBeenCalledWith("receipt-1", 2); + expect(mockChangeVisibility).toHaveBeenCalledWith("receipt-2", 2); + expect(mockSendToDlq).not.toHaveBeenCalled(); + expect(mockDeliverPayload).not.toHaveBeenCalled(); + }); + + it("requeues all records when circuit is open", async () => { + mockAdmit.mockResolvedValue({ + allowed: false, + reason: "circuit_open", + retryAfterMs: 30_000, + effectiveRate: 0, + }); + + const failures = await processRecords([ + makeRecord({ messageId: "msg-1", receiptHandle: "receipt-1" }), + makeRecord({ messageId: "msg-2", receiptHandle: "receipt-2" }), + ]); + + expect(failures).toEqual([ + { itemIdentifier: "msg-1" }, + { itemIdentifier: "msg-2" }, + ]); + expect(mockChangeVisibility).toHaveBeenCalledWith("receipt-1", 30); + expect(mockChangeVisibility).toHaveBeenCalledWith("receipt-2", 30); + expect(mockSendToDlq).not.toHaveBeenCalled(); + expect(mockDeliverPayload).not.toHaveBeenCalled(); + }); + + it("calls recordAdmissionDenied with correlationIds when batch denied", async () => { + const record1 = makeRecord({ messageId: "msg-1" }); + const record2 = makeRecord({ messageId: "msg-2" }); + + mockAdmit.mockResolvedValue({ + allowed: false, + reason: "circuit_open", + retryAfterMs: 30_000, + effectiveRate: 0, + }); + + const { recordAdmissionDenied } = jest.requireMock( + "services/delivery-observability", + ); + + await processRecords([record1, record2]); + + expect(recordAdmissionDenied).toHaveBeenCalledWith( + "client-1", + "target-1", + "circuit_open", + ["test-message-id", "test-message-id"], + ); + }); + + it("does not call recordResult on gate admission-denied path", async () => { + mockAdmit.mockResolvedValue({ + allowed: false, + reason: "rate_limited", + retryAfterMs: 2000, + effectiveRate: 10, + }); + + await processRecords([makeRecord()]); + + expect(mockRecordResult).not.toHaveBeenCalled(); + }); + }); + + describe("circuit breaker", () => { + it("proceeds to delivery when circuit breaker is disabled", async () => { + const targetNoCb = { + ...DEFAULT_TARGET, + delivery: { circuitBreaker: { enabled: false } }, + }; + mockLoadTargetConfig.mockResolvedValue(targetNoCb); + + const failures = await processRecords([makeRecord()]); + + expect(failures).toEqual([]); + expect(mockAdmit).toHaveBeenCalledWith( + expect.anything(), + "target-1", + 10, + false, + 1, + expect.objectContaining({ burstCapacity: 50 }), + ); + expect(mockDeliverPayload).toHaveBeenCalled(); + }); + + it("defaults cbEnabled to true when delivery exists but circuitBreaker is absent", async () => { + const targetDeliveryNoCb = { + ...DEFAULT_TARGET, + delivery: { mtls: { enabled: true } }, + }; + mockLoadTargetConfig.mockResolvedValue(targetDeliveryNoCb); + + await processRecords([makeRecord()]); + + expect(mockAdmit).toHaveBeenCalledWith( + expect.anything(), + "target-1", + 10, + true, + 1, + expect.objectContaining({ burstCapacity: 50 }), + ); + }); + + it("defaults cbEnabled to false when delivery is absent", async () => { + const targetNoDelivery = { ...DEFAULT_TARGET, delivery: undefined }; + mockLoadTargetConfig.mockResolvedValue(targetNoDelivery); + + await processRecords([makeRecord()]); + + expect(mockAdmit).toHaveBeenCalledWith( + expect.anything(), + "target-1", + 10, + false, + 1, + expect.objectContaining({ burstCapacity: 50 }), + ); + }); + + it("computes burst capacity as invocationRateLimit * 5", async () => { + const targetHighRate = { + ...DEFAULT_TARGET, + invocationRateLimit: 100, + }; + mockLoadTargetConfig.mockResolvedValue(targetHighRate); + + await processRecords([makeRecord()]); + + expect(mockAdmit).toHaveBeenCalledWith( + expect.anything(), + "target-1", + 100, + true, + 1, + expect.objectContaining({ burstCapacity: 500 }), + ); + }); + + it("caps burst capacity at TOKEN_BUCKET_BURST_CAPACITY", async () => { + const targetVeryHighRate = { + ...DEFAULT_TARGET, + invocationRateLimit: 1000, + }; + mockLoadTargetConfig.mockResolvedValue(targetVeryHighRate); + + await processRecords([makeRecord()]); + + expect(mockAdmit).toHaveBeenCalledWith( + expect.anything(), + "target-1", + 1000, + true, + 1, + expect.objectContaining({ burstCapacity: 2250 }), + ); + }); + + it("calls recordResult with batch counts on successful delivery when CB enabled", async () => { + const targetCb = { + ...DEFAULT_TARGET, + delivery: { circuitBreaker: { enabled: true } }, + }; + mockLoadTargetConfig.mockResolvedValue(targetCb); + mockAdmit.mockResolvedValue({ + allowed: true, + consumedTokens: 1, + effectiveRate: 10, + }); + + const failures = await processRecords([makeRecord()]); + + expect(failures).toEqual([]); + expect(mockRecordResult).toHaveBeenCalledWith( + expect.anything(), + "target-1", + 1, + 0, + expect.any(Object), + ); + }); + + it("calls recordResult with failure count on 5xx when CB enabled", async () => { + const targetCb = { + ...DEFAULT_TARGET, + delivery: { circuitBreaker: { enabled: true } }, + }; + mockLoadTargetConfig.mockResolvedValue(targetCb); + mockAdmit.mockResolvedValue({ + allowed: true, + consumedTokens: 1, + effectiveRate: 10, + }); + mockDeliverPayload.mockResolvedValue({ + outcome: "transient_failure", + statusCode: 503, + }); + + const failures = await processRecords([makeRecord()]); + + expect(failures).toEqual([{ itemIdentifier: "msg-1" }]); + expect(mockRecordResult).toHaveBeenCalledWith( + expect.anything(), + "target-1", + 1, + 1, + expect.any(Object), + ); + expect(mockChangeVisibility).toHaveBeenCalled(); + }); + + it("does not call recordResult when CB is disabled on transient failure", async () => { + const targetNoCb = { + ...DEFAULT_TARGET, + delivery: { circuitBreaker: { enabled: false } }, + }; + mockLoadTargetConfig.mockResolvedValue(targetNoCb); + mockDeliverPayload.mockResolvedValue({ + outcome: "transient_failure", + statusCode: 503, + }); + + await processRecords([makeRecord()]); + + expect(mockRecordResult).not.toHaveBeenCalled(); + expect(mockChangeVisibility).toHaveBeenCalled(); + }); + + it("does not call recordResult when CB is disabled on success", async () => { + const targetNoCb = { + ...DEFAULT_TARGET, + delivery: { circuitBreaker: { enabled: false } }, + }; + mockLoadTargetConfig.mockResolvedValue(targetNoCb); + + await processRecords([makeRecord()]); + + expect(mockRecordResult).not.toHaveBeenCalled(); + }); + + it("records CircuitBreakerOpen when recordResult indicates circuit opened", async () => { + const targetCb = { + ...DEFAULT_TARGET, + delivery: { circuitBreaker: { enabled: true } }, + }; + mockLoadTargetConfig.mockResolvedValue(targetCb); + mockDeliverPayload.mockResolvedValue({ + outcome: "transient_failure", + statusCode: 503, + }); + mockRecordResult.mockResolvedValue({ + circuitState: "open", + circuitSwitched: true, + }); + + const { recordCircuitBreakerOpen } = jest.requireMock( + "services/delivery-observability", + ); + + await processRecords([makeRecord()]); + + expect(recordCircuitBreakerOpen).toHaveBeenCalledWith("target-1"); + }); + + it("does not record CircuitBreakerOpen when recordResult has no state change", async () => { + const targetCb = { + ...DEFAULT_TARGET, + delivery: { circuitBreaker: { enabled: true } }, + }; + mockLoadTargetConfig.mockResolvedValue(targetCb); + mockDeliverPayload.mockResolvedValue({ + outcome: "transient_failure", + statusCode: 503, + }); + mockRecordResult.mockResolvedValue({ + circuitState: "open", + circuitSwitched: false, + }); + + const { recordCircuitBreakerOpen } = jest.requireMock( + "services/delivery-observability", + ); + + await processRecords([makeRecord()]); + + expect(recordCircuitBreakerOpen).not.toHaveBeenCalled(); + }); + + it("does not record CircuitBreakerOpen when circuit is closed", async () => { + const targetCb = { + ...DEFAULT_TARGET, + delivery: { circuitBreaker: { enabled: true } }, + }; + mockLoadTargetConfig.mockResolvedValue(targetCb); + mockDeliverPayload.mockResolvedValue({ + outcome: "transient_failure", + statusCode: 503, + }); + mockRecordResult.mockResolvedValue({ + circuitState: "closed", + circuitSwitched: false, + }); + + const { recordCircuitBreakerOpen } = jest.requireMock( + "services/delivery-observability", + ); + + await processRecords([makeRecord()]); + + expect(recordCircuitBreakerOpen).not.toHaveBeenCalled(); + }); + + it("records CircuitBreakerClosed when recordResult indicates circuit closed", async () => { + const targetCb = { + ...DEFAULT_TARGET, + delivery: { circuitBreaker: { enabled: true } }, + }; + mockLoadTargetConfig.mockResolvedValue(targetCb); + mockDeliverPayload.mockResolvedValue({ + outcome: "success", + statusCode: 200, + }); + mockRecordResult.mockResolvedValue({ + circuitState: "closed_recovery", + circuitSwitched: true, + }); + + const { recordCircuitBreakerClosed } = jest.requireMock( + "services/delivery-observability", + ); + + await processRecords([makeRecord()]); + + expect(recordCircuitBreakerClosed).toHaveBeenCalledWith("target-1"); + }); + }); + + describe("retry and visibility", () => { + it("calls changeVisibility with backoff on 5xx", async () => { + mockDeliverPayload.mockResolvedValue({ + outcome: "transient_failure", + statusCode: 503, + }); + + const failures = await processRecords([makeRecord()]); + + expect(failures).toEqual([{ itemIdentifier: "msg-1" }]); + expect(mockChangeVisibility).toHaveBeenCalledWith("receipt-1", 5); + }); + + it("changes visibility once for transient failure", async () => { + mockDeliverPayload.mockResolvedValue({ + outcome: "transient_failure", + statusCode: 503, + }); + + await processRecords([makeRecord()]); + + expect(mockChangeVisibility).toHaveBeenCalledTimes(1); + }); + + it("does not override visibility on rate-limited requeue", async () => { + mockDeliverPayload.mockResolvedValue({ + outcome: "rate_limited", + retryAfterHeader: "120", + statusCode: 429, + }); + mockHandleRateLimitedRecord.mockImplementation(async () => { + await mockChangeVisibility("receipt-1", 120); + return "retry"; + }); + + await processRecords([makeRecord()]); + + expect(mockChangeVisibility).toHaveBeenCalledTimes(1); + expect(mockChangeVisibility).toHaveBeenCalledWith("receipt-1", 120); + }); + + it("changes visibility once per record for admission-denied batch", async () => { + mockAdmit.mockResolvedValue({ + allowed: false, + reason: "rate_limited", + retryAfterMs: 2000, + effectiveRate: 10, + }); + + await processRecords([ + makeRecord({ messageId: "msg-1", receiptHandle: "receipt-1" }), + makeRecord({ messageId: "msg-2", receiptHandle: "receipt-2" }), + ]); + + expect(mockChangeVisibility).toHaveBeenCalledTimes(2); + expect(mockChangeVisibility).toHaveBeenCalledWith("receipt-1", 2); + expect(mockChangeVisibility).toHaveBeenCalledWith("receipt-2", 2); + }); + + it("caps visibility delay at SQS maximum (12 hours) for admission-denied batch", async () => { + mockAdmit.mockResolvedValue({ + allowed: false, + reason: "rate_limited", + retryAfterMs: 60_000, + effectiveRate: 10, + }); + + const record = makeRecord({ + attributes: { + ApproximateReceiveCount: "1000", + SentTimestamp: "0", + SenderId: "sender", + ApproximateFirstReceiveTimestamp: "0", + }, + }); + + await processRecords([record]); + + expect(mockChangeVisibility).toHaveBeenCalledWith("receipt-1", 43_200); + }); + + it("delegates 429 handling to handleRateLimitedRecord", async () => { + mockDeliverPayload.mockResolvedValue({ + outcome: "rate_limited", + retryAfterHeader: "120", + }); + + await processRecords([makeRecord()]); + + expect(mockHandleRateLimitedRecord).toHaveBeenCalledWith( + makeRecord(), + "client-1", + "target-1", + "120", + 1, + ); + }); + + it("uses configured maxRetryDurationSeconds when set on target", async () => { + const targetWithRetry = { + ...DEFAULT_TARGET, + delivery: { + ...DEFAULT_TARGET.delivery, + maxRetryDurationSeconds: 3600, + }, + }; + mockLoadTargetConfig.mockResolvedValue(targetWithRetry); + mockIsWindowExhausted.mockReturnValue(false); + + const failures = await processRecords([makeRecord()]); + + expect(failures).toEqual([]); + expect(mockIsWindowExhausted).toHaveBeenCalledWith( + expect.any(Number), + 3_600_000, + ); + }); + + it("uses default maxRetryDurationMs when target has no maxRetryDurationSeconds", async () => { + const failures = await processRecords([makeRecord()]); + + expect(failures).toEqual([]); + expect(mockIsWindowExhausted).toHaveBeenCalledWith( + expect.any(Number), + 7_200_000, + ); + }); + }); + + describe("error handling", () => { + it("sends all records to DLQ when CLIENT_ID is not set", async () => { + delete process.env.CLIENT_ID; + + const record1 = makeRecord({ messageId: "msg-1" }); + const record2 = makeRecord({ messageId: "msg-2" }); + + const failures = await processRecords([record1, record2]); + + expect(failures).toEqual([]); + expect(mockSendToDlq).toHaveBeenCalledWith(record1.body); + expect(mockSendToDlq).toHaveBeenCalledWith(record2.body); + expect(mockSendToDlq).toHaveBeenCalledTimes(2); + expect(mockDeliverPayload).not.toHaveBeenCalled(); + }); + + it("an unexpected delivery error does not prevent other records in the batch", async () => { + const record1 = makeRecord({ messageId: "msg-1" }); + const record2 = makeRecord({ messageId: "msg-2" }); + + mockDeliverPayload + .mockRejectedValueOnce(new Error("Connection reset")) + .mockResolvedValueOnce({ outcome: "success" }); + + const failures = await processRecords([record1, record2]); + + expect(failures).toEqual([]); + expect(mockSendToDlq).toHaveBeenCalledWith(record1.body); + }); + + it("sends unhandled errors to DLQ", async () => { + mockDeliverPayload.mockRejectedValue(new Error("Infrastructure error")); + + const failures = await processRecords([makeRecord()]); + + expect(failures).toEqual([]); + expect(mockSendToDlq).toHaveBeenCalledWith(makeRecord().body); + expect(mockChangeVisibility).not.toHaveBeenCalled(); + }); + + it("sends unparseable records to DLQ and logs error", async () => { + const badRecord = makeRecord({ + messageId: "bad-msg", + body: "not-valid-json{{{", + }); + const goodRecord = makeRecord({ messageId: "good-msg" }); + + const { logger } = jest.requireMock( + "@nhs-notify-client-callbacks/logger", + ); + + const result = await processRecords([badRecord, goodRecord]); + + expect(mockSendToDlq).toHaveBeenCalledWith(badRecord.body); + expect(logger.error).toHaveBeenCalledWith( + "Unparseable message body \u2014 sending to DLQ", + expect.objectContaining({ messageId: "bad-msg" }), + ); + expect(result).toHaveLength(0); + expect(logger.info).toHaveBeenCalledWith( + "Batch complete", + expect.objectContaining({ dlqCount: 1, deliveredCount: 1 }), + ); + }); + + it("returns record for retry when DLQ send fails after delivery error", async () => { + mockDeliverPayload.mockRejectedValue(new Error("Connection reset")); + mockSendToDlq.mockRejectedValue(new Error("DLQ unavailable")); + + const { logger } = jest.requireMock( + "@nhs-notify-client-callbacks/logger", + ); + + const result = await processRecords([makeRecord()]); + + expect(logger.error).toHaveBeenCalledWith( + "DLQ send also failed \u2014 returning for retry", + expect.objectContaining({ messageId: "msg-1" }), + ); + expect(result).toHaveLength(1); + expect(result[0].itemIdentifier).toBe("msg-1"); + }); + + it("swallows reportCircuitBreaker errors without affecting delivery", async () => { + mockRecordResult.mockRejectedValue(new Error("Redis timeout")); + + const { logger } = jest.requireMock( + "@nhs-notify-client-callbacks/logger", + ); + + const result = await processRecords([makeRecord()]); + + expect(logger.error).toHaveBeenCalledWith( + "Failed to report circuit breaker result", + expect.objectContaining({ targetId: "target-1" }), + ); + expect(result).toHaveLength(0); + expect(logger.info).toHaveBeenCalledWith( + "Batch complete", + expect.objectContaining({ deliveredCount: 1 }), + ); + }); + + it("sends all batch records to DLQ when processTargetBatch throws", async () => { + mockLoadTargetConfig.mockRejectedValue( + new Error("Config service unavailable"), + ); + + const { logger } = jest.requireMock( + "@nhs-notify-client-callbacks/logger", + ); + + const records = [ + makeRecord({ messageId: "msg-1" }), + makeRecord({ messageId: "msg-2" }), + ]; + const result = await processRecords(records); + + expect(logger.error).toHaveBeenCalledWith( + "Target batch failed \u2014 sending all records to DLQ", + expect.objectContaining({ targetId: "target-1" }), + ); + expect(mockSendToDlq).toHaveBeenCalledTimes(2); + expect(result).toHaveLength(0); + expect(logger.info).toHaveBeenCalledWith( + "Batch complete", + expect.objectContaining({ dlqCount: 2 }), + ); + }); + }); + + describe("observability", () => { + it("records RateLimited on 429 response", async () => { + mockDeliverPayload.mockResolvedValue({ + outcome: "rate_limited", + retryAfterHeader: "60", + }); + + const { recordDeliveryRateLimited } = jest.requireMock( + "services/delivery-observability", + ); + + await processRecords([makeRecord()]); + + expect(recordDeliveryRateLimited).toHaveBeenCalledWith( + "client-1", + "target-1", + "test-message-id", + ); + }); + + it("includes correlationId in error log on unexpected delivery failure", async () => { + mockDeliverPayload.mockRejectedValue(new Error("Connection reset")); + + const { logger } = jest.requireMock( + "@nhs-notify-client-callbacks/logger", + ); + + await processRecords([makeRecord()]); + + expect(logger.error).toHaveBeenCalledWith( + "Failed to process record", + expect.objectContaining({ + messageId: "msg-1", + correlationId: "test-message-id", + }), + ); + }); + + it("logs deliveredCount and dlqCount in batch complete", async () => { + const record1 = makeRecord({ messageId: "msg-1" }); + const record2 = makeRecord({ messageId: "msg-2" }); + + mockDeliverPayload + .mockResolvedValueOnce({ outcome: "success" }) + .mockResolvedValueOnce({ outcome: "permanent_failure" }); + + const { logger } = jest.requireMock( + "@nhs-notify-client-callbacks/logger", + ); + + await processRecords([record1, record2]); + + expect(logger.info).toHaveBeenCalledWith( + "Batch complete", + expect.objectContaining({ + batchSize: 2, + deliveredCount: 1, + dlqCount: 1, + retryCount: 0, + }), + ); + }); + }); +}); diff --git a/lambdas/https-client-lambda/src/__tests__/helpers/fengari.d.ts b/lambdas/https-client-lambda/src/__tests__/helpers/fengari.d.ts new file mode 100644 index 00000000..e40c1e59 --- /dev/null +++ b/lambdas/https-client-lambda/src/__tests__/helpers/fengari.d.ts @@ -0,0 +1,32 @@ +declare module "fengari" { + type LuaState = object; + + const lua: { + LUA_OK: number; + lua_close(L: LuaState): void; + lua_createtable(L: LuaState, narr: number, nrec: number): void; + lua_getglobal(L: LuaState, name: Uint8Array): number; + lua_gettop(L: LuaState): number; + lua_pushboolean(L: LuaState, b: number): void; + lua_pushcfunction(L: LuaState, fn: (L: LuaState) => number): void; + lua_pushinteger(L: LuaState, n: number): void; + lua_pushstring(L: LuaState, s: Uint8Array): void; + lua_rawseti(L: LuaState, idx: number, n: number): void; + lua_setglobal(L: LuaState, name: Uint8Array): void; + lua_tostring(L: LuaState, idx: number): Uint8Array; + }; + + const lauxlib: { + luaL_dostring(L: LuaState, s: Uint8Array): number; + luaL_newstate(): LuaState; + }; + + const lualib: { + luaL_openlibs(L: LuaState): void; + }; + + // eslint-disable-next-line @typescript-eslint/naming-convention -- fengari uses snake_case names + function to_jsstring(s: Uint8Array): string; + // eslint-disable-next-line @typescript-eslint/naming-convention -- fengari uses snake_case names + function to_luastring(s: string): Uint8Array; +} diff --git a/lambdas/https-client-lambda/src/__tests__/helpers/lua-redis-mock.ts b/lambdas/https-client-lambda/src/__tests__/helpers/lua-redis-mock.ts new file mode 100644 index 00000000..f6d11d50 --- /dev/null +++ b/lambdas/https-client-lambda/src/__tests__/helpers/lua-redis-mock.ts @@ -0,0 +1,164 @@ +import { + lauxlib, + lua, + lualib, + to_jsstring as toJsstring, + to_luastring as toLuastring, +} from "fengari"; + +type LuaState = ReturnType; +type RedisStore = Map>; + +export function createRedisStore(): RedisStore { + return new Map(); +} + +function hset(store: RedisStore, key: string, pairs: string[]): number { + const hash = store.get(key) ?? new Map(); + store.set(key, hash); + let fieldsSet = 0; + for (let i = 0; i < pairs.length; i += 2) { + // eslint-disable-next-line security/detect-object-injection -- pairs is a controlled array from redis HSET parsing + hash.set(pairs[i], pairs[i + 1]); + fieldsSet += 1; + } + return fieldsSet; +} + +function redisCallHandler(L: LuaState, store: RedisStore): number { + const cmd = toJsstring(lua.lua_tostring(L, 1)).toUpperCase(); + + if (cmd === "HMGET") { + const key = toJsstring(lua.lua_tostring(L, 2)); + const nArgs = lua.lua_gettop(L); + const hash = store.get(key); + lua.lua_createtable(L, nArgs - 2, 0); + for (let i = 3; i <= nArgs; i++) { + const field = toJsstring(lua.lua_tostring(L, i)); + const val = hash?.get(field); + if (val === undefined) { + lua.lua_pushboolean(L, 0); + } else { + lua.lua_pushstring(L, toLuastring(val)); + } + lua.lua_rawseti(L, -2, i - 2); + } + return 1; + } + + if (cmd === "HSET") { + const key = toJsstring(lua.lua_tostring(L, 2)); + const nArgs = lua.lua_gettop(L); + const pairs: string[] = []; + for (let i = 3; i <= nArgs; i++) { + pairs.push(toJsstring(lua.lua_tostring(L, i))); + } + const count = hset(store, key, pairs); + lua.lua_pushinteger(L, count); + return 1; + } + + if (cmd === "EXPIRE") { + lua.lua_pushinteger(L, 1); + return 1; + } + + throw new Error(`Unsupported Redis command in mock: ${cmd}`); +} + +const CJSON_AND_REDIS_PREAMBLE = ` + cjson = {} + function cjson.encode(t) + if t == nil then return "null" end + if type(t) ~= "table" then + if type(t) == "string" then return '"' .. t .. '"' end + if type(t) == "boolean" then return t and "true" or "false" end + if type(t) == "number" then + if t == math.floor(t) and t < 1e15 and t > -1e15 then + return string.format("%d", t) + end + return tostring(t) + end + return tostring(t) + end + local n = #t + local isArray = n > 0 + if isArray then + for k in pairs(t) do + if type(k) ~= "number" or k ~= math.floor(k) or k < 1 or k > n then + isArray = false + break + end + end + end + if isArray then + local parts = {} + for i = 1, n do + parts[#parts + 1] = cjson.encode(t[i]) + end + return "[" .. table.concat(parts, ",") .. "]" + end + local parts = {} + for k, v in pairs(t) do + parts[#parts + 1] = '"' .. tostring(k) .. '":' .. cjson.encode(v) + end + return "{" .. table.concat(parts, ",") .. "}" + end + + redis = {} + function redis.call(cmd, ...) + return __redis_call(cmd, ...) + end +`; + +function registerRedisCallGlobal(L: LuaState, store: RedisStore): void { + lua.lua_pushcfunction(L, (ls: LuaState) => redisCallHandler(ls, store)); + lua.lua_setglobal(L, toLuastring("__redis_call")); +} + +function installCjsonAndRedisShims(L: LuaState): void { + lauxlib.luaL_dostring(L, toLuastring(CJSON_AND_REDIS_PREAMBLE)); +} + +function setStringArrayGlobal( + L: LuaState, + name: string, + values: string[], +): void { + lua.lua_createtable(L, values.length, 0); + for (const [i, value] of values.entries()) { + lua.lua_pushstring(L, toLuastring(value)); + lua.lua_rawseti(L, -2, i + 1); + } + lua.lua_setglobal(L, toLuastring(name)); +} + +function runScript(L: LuaState, script: string): string { + const wrapped = `local __r = (function()\n${script}\nend)()\nreturn cjson.encode(__r)`; + const status = lauxlib.luaL_dostring(L, toLuastring(wrapped)); + if (status !== lua.LUA_OK) { + const errMsg = toJsstring(lua.lua_tostring(L, -1)); + throw new Error(`Lua error: ${errMsg}`); + } + return toJsstring(lua.lua_tostring(L, -1)); +} + +export function evalLua( + script: string, + keys: string[], + argv: string[], + store: RedisStore, +): unknown { + const L: LuaState = lauxlib.luaL_newstate(); + lualib.luaL_openlibs(L); + + try { + registerRedisCallGlobal(L, store); + installCjsonAndRedisShims(L); + setStringArrayGlobal(L, "KEYS", keys); + setStringArrayGlobal(L, "ARGV", argv); + return JSON.parse(runScript(L, script)); + } finally { + lua.lua_close(L); + } +} diff --git a/lambdas/https-client-lambda/src/__tests__/https-client.test.ts b/lambdas/https-client-lambda/src/__tests__/https-client.test.ts new file mode 100644 index 00000000..917f13c7 --- /dev/null +++ b/lambdas/https-client-lambda/src/__tests__/https-client.test.ts @@ -0,0 +1,328 @@ +/* eslint-disable unicorn/prefer-event-target -- Node.js http module mock requires EventEmitter API */ +import { EventEmitter } from "node:events"; +import https, { Agent } from "node:https"; +import type { CallbackTarget } from "@nhs-notify-client-callbacks/models"; + +import { deliverPayload } from "services/delivery/https-client"; + +jest.mock("services/delivery/tls-agent-factory", () => ({ + PERMANENT_TLS_ERROR_CODES: new Set([ + "CERT_HAS_EXPIRED", + "DEPTH_ZERO_SELF_SIGNED_CERT", + "ERR_CERT_PINNING_FAILED", + "ERR_TLS_CERT_ALTNAME_INVALID", + "SELF_SIGNED_CERT_IN_CHAIN", + "UNABLE_TO_VERIFY_LEAF_SIGNATURE", + ]), +})); + +const createTarget = (): CallbackTarget => ({ + targetId: "target-1", + type: "API", + invocationEndpoint: "https://webhook.example.invalid:8443/callback", + invocationMethod: "POST", + invocationRateLimit: 10, + apiKey: { headerName: "x-api-key", headerValue: "secret" }, +}); + +const createMockAgent = () => ({}) as Agent; + +type MockResponse = EventEmitter & { + statusCode: number; + headers: Record; + resume: jest.Mock; +}; + +function mockHttpsRequest( + statusCode: number, + headers: Record = {}, + body = "", +) { + const mockReq = new EventEmitter() as EventEmitter & { + end: jest.Mock; + destroy: jest.Mock; + }; + mockReq.end = jest.fn(); + mockReq.destroy = jest.fn(); + + jest.spyOn(https, "request").mockImplementation((...args: unknown[]) => { + const callback = args.find((a) => typeof a === "function") as + | ((res: MockResponse) => void) + | undefined; + + const res: MockResponse = Object.assign(new EventEmitter(), { + statusCode, + headers, + resume: jest.fn(), + }); + + if (callback) { + process.nextTick(() => { + callback(res); + process.nextTick(() => { + if (body) res.emit("data", Buffer.from(body)); + res.emit("end"); + }); + }); + } + + return mockReq as unknown as ReturnType; + }); + + return mockReq; +} + +function mockHttpsRequestError(errorCode: string) { + const mockReq = new EventEmitter() as EventEmitter & { + end: jest.Mock; + destroy: jest.Mock; + }; + mockReq.end = jest.fn(); + mockReq.destroy = jest.fn(); + + jest.spyOn(https, "request").mockImplementation(() => { + process.nextTick(() => { + const error = new Error("TLS error") as NodeJS.ErrnoException; + error.code = errorCode; + mockReq.emit("error", error); + }); + + return mockReq as unknown as ReturnType; + }); + + return mockReq; +} + +function mockHttpsRequestTimeout() { + const mockReq = new EventEmitter() as EventEmitter & { + end: jest.Mock; + destroy: jest.Mock; + }; + mockReq.end = jest.fn(); + mockReq.destroy = jest.fn((error?: Error) => { + if (error) { + process.nextTick(() => mockReq.emit("error", error)); + } + }); + + jest.spyOn(https, "request").mockImplementation(() => { + process.nextTick(() => mockReq.emit("timeout")); + return mockReq as unknown as ReturnType; + }); + + return mockReq; +} + +describe("deliverPayload", () => { + afterEach(() => { + jest.restoreAllMocks(); + }); + + it("returns success on 2xx", async () => { + mockHttpsRequest(200); + + const result = await deliverPayload( + createTarget(), + '{"test":true}', + "sig-abc", + createMockAgent(), + ); + + expect(result).toEqual({ outcome: "success" }); + }); + + it("returns permanent_failure on 4xx non-429 non-retryable", async () => { + mockHttpsRequest(400, {}, JSON.stringify({ message: "Bad request" })); + + const result = await deliverPayload( + createTarget(), + '{"test":true}', + "sig-abc", + createMockAgent(), + ); + + expect(result).toEqual({ + outcome: "permanent_failure", + statusCode: 400, + responseBody: JSON.stringify({ message: "Bad request" }), + }); + }); + + it.each([401, 407, 409])( + "returns transient_failure on retryable 4xx status %s", + async (statusCode) => { + mockHttpsRequest(statusCode); + + const result = await deliverPayload( + createTarget(), + '{"test":true}', + "sig-abc", + createMockAgent(), + ); + + expect(result).toEqual({ outcome: "transient_failure", statusCode }); + }, + ); + + it("returns permanent_failure on TLS error CERT_HAS_EXPIRED", async () => { + mockHttpsRequestError("CERT_HAS_EXPIRED"); + + const result = await deliverPayload( + createTarget(), + '{"test":true}', + "sig-abc", + createMockAgent(), + ); + + expect(result).toEqual({ + outcome: "permanent_failure", + errorCode: "CERT_HAS_EXPIRED", + }); + }); + + it("returns permanent_failure on TLS pinning error", async () => { + mockHttpsRequestError("ERR_CERT_PINNING_FAILED"); + + const result = await deliverPayload( + createTarget(), + '{"test":true}', + "sig-abc", + createMockAgent(), + ); + + expect(result).toEqual({ + outcome: "permanent_failure", + errorCode: "ERR_CERT_PINNING_FAILED", + }); + }); + + it("returns transient_failure on 5xx", async () => { + mockHttpsRequest(503); + + const result = await deliverPayload( + createTarget(), + '{"test":true}', + "sig-abc", + createMockAgent(), + ); + + expect(result).toEqual({ outcome: "transient_failure", statusCode: 503 }); + }); + + it("returns rate_limited with Retry-After header value", async () => { + mockHttpsRequest(429, { "retry-after": "60" }); + + const result = await deliverPayload( + createTarget(), + '{"test":true}', + "sig-abc", + createMockAgent(), + ); + + expect(result).toEqual({ + outcome: "rate_limited", + retryAfterHeader: "60", + statusCode: 429, + }); + }); + + it("returns rate_limited with undefined retryAfterHeader when header is absent", async () => { + mockHttpsRequest(429); + + const result = await deliverPayload( + createTarget(), + '{"test":true}', + "sig-abc", + createMockAgent(), + ); + + expect(result).toEqual({ + outcome: "rate_limited", + retryAfterHeader: undefined, + statusCode: 429, + }); + }); + + it("returns transient_failure on TCP error", async () => { + mockHttpsRequestError("ECONNREFUSED"); + + const result = await deliverPayload( + createTarget(), + '{"test":true}', + "sig-abc", + createMockAgent(), + ); + + expect(result).toEqual({ outcome: "transient_failure", statusCode: 0 }); + }); + + it("uses port 443 when URL has no explicit port", async () => { + mockHttpsRequest(200); + const target = createTarget(); + target.invocationEndpoint = "https://webhook.example.invalid/callback"; + + const result = await deliverPayload( + target, + '{"test":true}', + "sig-abc", + createMockAgent(), + ); + + expect(result).toEqual({ outcome: "success" }); + const callUrl = (https.request as jest.Mock).mock.calls[0][0] as URL; + expect(callUrl).toBeInstanceOf(URL); + expect(callUrl.port).toBe(""); + }); + + it("returns transient failure on request timeout", async () => { + mockHttpsRequestTimeout(); + + const result = await deliverPayload( + createTarget(), + '{"test":true}', + "sig-abc", + createMockAgent(), + ); + + expect(result).toEqual({ outcome: "transient_failure", statusCode: 0 }); + }); + + it("treats undefined statusCode as 0", async () => { + const mockReq = new EventEmitter() as EventEmitter & { + end: jest.Mock; + destroy: jest.Mock; + }; + mockReq.end = jest.fn(); + mockReq.destroy = jest.fn(); + + jest.spyOn(https, "request").mockImplementation((...args: unknown[]) => { + const callback = args.find((a) => typeof a === "function") as + | ((res: MockResponse) => void) + | undefined; + + const res = Object.assign(new EventEmitter(), { + statusCode: undefined as unknown as number, + headers: {}, + resume: jest.fn(), + }) as MockResponse; + + if (callback) { + process.nextTick(() => { + callback(res); + process.nextTick(() => (res as EventEmitter).emit("end")); + }); + } + + return mockReq as unknown as ReturnType; + }); + + const result = await deliverPayload( + createTarget(), + '{"test":true}', + "sig-abc", + createMockAgent(), + ); + + expect(result).toEqual({ outcome: "transient_failure", statusCode: 0 }); + }); +}); diff --git a/lambdas/https-client-lambda/src/__tests__/index.test.ts b/lambdas/https-client-lambda/src/__tests__/index.test.ts new file mode 100644 index 00000000..53394149 --- /dev/null +++ b/lambdas/https-client-lambda/src/__tests__/index.test.ts @@ -0,0 +1,36 @@ +import { handler } from "index"; +import { processRecords } from "handler"; + +jest.mock("handler", () => ({ + processRecords: jest.fn().mockResolvedValue([]), +})); + +describe("handler", () => { + it("returns batchItemFailures from processRecords", async () => { + const event = { + Records: [ + { + messageId: "msg-1", + receiptHandle: "r-1", + body: "{}", + attributes: { + ApproximateReceiveCount: "1", + SentTimestamp: "0", + SenderId: "sender", + ApproximateFirstReceiveTimestamp: "0", + }, + messageAttributes: {}, + md5OfBody: "abc", + eventSource: "aws:sqs", + eventSourceARN: "arn:aws:sqs:eu-west-2:123:queue", + awsRegion: "eu-west-2", + }, + ], + }; + + const result = await handler(event); + + expect(result).toEqual({ batchItemFailures: [] }); + expect(processRecords).toHaveBeenCalledWith(event.Records); + }); +}); diff --git a/lambdas/https-client-lambda/src/__tests__/payload-signer.test.ts b/lambdas/https-client-lambda/src/__tests__/payload-signer.test.ts new file mode 100644 index 00000000..191f85dc --- /dev/null +++ b/lambdas/https-client-lambda/src/__tests__/payload-signer.test.ts @@ -0,0 +1,52 @@ +import { createHmac } from "node:crypto"; +import { signPayload } from "services/payload-signer"; + +const makePayload = () => + ({ + data: [ + { type: "MessageStatus", attributes: { messageStatus: "delivered" } }, + ], + }) as Parameters[2]; + +describe("signPayload", () => { + it("produces correct HMAC-SHA256 output for a known input", () => { + const payload = makePayload(); + // eslint-disable-next-line sonarjs/hardcoded-secret-signatures -- test fixture, not a real secret + const expected = createHmac("sha256", "app-1.key-1") + .update(JSON.stringify(payload)) + .digest("hex"); + + expect(signPayload("app-1", "key-1", payload)).toBe(expected); + }); + + it("produces different signatures for different appId/apiKey combinations", () => { + const payload = makePayload(); + + const sig1 = signPayload("app-1", "key-1", payload); + const sig2 = signPayload("app-2", "key-2", payload); + + expect(sig1).not.toBe(sig2); + }); + + it("produces the same signature for the same inputs", () => { + const payload = makePayload(); + + const sig1 = signPayload("app-1", "key-1", payload); + const sig2 = signPayload("app-1", "key-1", payload); + + expect(sig1).toBe(sig2); + }); + + it("produces a deterministic non-empty signature for an empty payload object", () => { + const emptyPayload = {} as Parameters[2]; + + const sig = signPayload("app-1", "key-1", emptyPayload); + + expect(sig).toBeTruthy(); + expect(typeof sig).toBe("string"); + expect(sig.length).toBeGreaterThan(0); + + const sig2 = signPayload("app-1", "key-1", emptyPayload); + expect(sig).toBe(sig2); + }); +}); diff --git a/lambdas/https-client-lambda/src/__tests__/record-result-lua.test.ts b/lambdas/https-client-lambda/src/__tests__/record-result-lua.test.ts new file mode 100644 index 00000000..6d2e13f3 --- /dev/null +++ b/lambdas/https-client-lambda/src/__tests__/record-result-lua.test.ts @@ -0,0 +1,694 @@ +import recordResultLuaSrc from "services/record-result.lua"; +import { createRedisStore, evalLua } from "__tests__/helpers/lua-redis-mock"; + +// ARGV: [now, consumedTokens, processingFailures, cooldownPeriodMs, recoveryPeriodMs, failureThreshold, minAttempts, samplePeriodMs] +// KEYS: [epKey] +// Returns: [circuitState, circuitSwitched] +// circuitState: "open" | "open_half" | "closed_recovery" | "closed" +// circuitSwitched: 0 | 1 + +type RecordResultArgs = { + now: number; + consumedTokens: number; + processingFailures: number; + cooldownPeriodMs: number; + recoveryPeriodMs: number; + failureThreshold: number; + minAttempts: number; + samplePeriodMs: number; +}; + +const defaultArgs: RecordResultArgs = { + now: 1_000_000, + consumedTokens: 1, + processingFailures: 0, + cooldownPeriodMs: 120_000, + recoveryPeriodMs: 600_000, + failureThreshold: 0.3, + minAttempts: 5, + samplePeriodMs: 300_000, +}; + +type RecordResultResult = [string, number]; + +function runRecordResult( + store: ReturnType, + args: Partial = {}, + targetId = "t1", +): RecordResultResult { + const merged = { ...defaultArgs, ...args }; + return evalLua( + recordResultLuaSrc, + [`ep:${targetId}`], + [ + merged.now.toString(), + merged.consumedTokens.toString(), + merged.processingFailures.toString(), + merged.cooldownPeriodMs.toString(), + merged.recoveryPeriodMs.toString(), + merged.failureThreshold.toString(), + merged.minAttempts.toString(), + merged.samplePeriodMs.toString(), + ], + store, + ) as RecordResultResult; +} + +describe("record-result.lua", () => { + describe("recording attempts and failures", () => { + it("returns closed state for a successful batch with no state change", () => { + const store = createRedisStore(); + store.set( + "ep:t1", + new Map([ + ["is_open", "0"], + ["switched_at", "0"], + ["sample_till", "9999999999"], + ]), + ); + + const [circuitState, circuitSwitched] = runRecordResult(store, { + consumedTokens: 5, + processingFailures: 0, + }); + + expect(circuitState).toBe("closed"); + expect(circuitSwitched).toBe(0); + }); + + it("increments cur_attempts without incrementing cur_failures", () => { + const store = createRedisStore(); + store.set( + "ep:t1", + new Map([ + ["is_open", "0"], + ["switched_at", "0"], + ["sample_till", "9999999999"], + ]), + ); + + runRecordResult(store, { consumedTokens: 3, processingFailures: 0 }); + + const epHash = store.get("ep:t1")!; + expect(epHash.get("cur_attempts")).toBe("3"); + expect(epHash.get("cur_failures")).toBe("0"); + }); + + it("increments both cur_attempts and cur_failures", () => { + const store = createRedisStore(); + store.set( + "ep:t1", + new Map([ + ["is_open", "0"], + ["switched_at", "0"], + ["sample_till", "9999999999"], + ]), + ); + + runRecordResult(store, { consumedTokens: 5, processingFailures: 1 }); + + const epHash = store.get("ep:t1")!; + expect(epHash.get("cur_attempts")).toBe("5"); + expect(epHash.get("cur_failures")).toBe("1"); + }); + + it("does not record attempts/failures when circuit is fully open", () => { + const store = createRedisStore(); + const now = 1_000_000; + const switchedAt = now - 10_000; + + store.set( + "ep:t1", + new Map([ + ["is_open", "1"], + ["switched_at", switchedAt.toString()], + ["sample_till", "9999999999"], + ["cur_attempts", "0"], + ["cur_failures", "0"], + ]), + ); + + runRecordResult(store, { + now, + cooldownPeriodMs: 120_000, + consumedTokens: 5, + processingFailures: 3, + }); + + const epHash = store.get("ep:t1")!; + expect(epHash.get("cur_attempts")).toBe("0"); + expect(epHash.get("cur_failures")).toBe("0"); + }); + + it("returns open when circuit is fully open and state unchanged", () => { + const store = createRedisStore(); + const now = 1_000_000; + const switchedAt = now - 10_000; + + store.set( + "ep:t1", + new Map([ + ["is_open", "1"], + ["switched_at", switchedAt.toString()], + ["sample_till", "9999999999"], + ]), + ); + + const [circuitState, circuitSwitched] = runRecordResult(store, { + now, + cooldownPeriodMs: 120_000, + consumedTokens: 1, + processingFailures: 0, + }); + + expect(circuitState).toBe("open"); + expect(circuitSwitched).toBe(0); + }); + + it("treats a fresh endpoint as fully open during cooldown and does not record", () => { + const store = createRedisStore(); + const now = 10_000; + + const [circuitState, circuitSwitched] = runRecordResult(store, { + now, + cooldownPeriodMs: 120_000, + consumedTokens: 5, + processingFailures: 3, + }); + + expect(circuitState).toBe("open"); + expect(circuitSwitched).toBe(0); + + const epHash = store.get("ep:t1")!; + expect(epHash.get("cur_attempts")).toBe("0"); + expect(epHash.get("cur_failures")).toBe("0"); + }); + + it("treats a fresh endpoint as half-open when now exceeds cooldown", () => { + const store = createRedisStore(); + const now = 1_000_000; + + const [circuitState, circuitSwitched] = runRecordResult(store, { + now, + cooldownPeriodMs: 120_000, + consumedTokens: 1, + processingFailures: 0, + minAttempts: 5, + }); + + expect(circuitState).toBe("closed_recovery"); + expect(circuitSwitched).toBe(1); + }); + + it("accumulates across multiple calls within the same sample window", () => { + const store = createRedisStore(); + const now = 1_000_000; + + store.set( + "ep:t1", + new Map([ + ["is_open", "0"], + ["switched_at", "0"], + ["sample_till", "9999999999"], + ]), + ); + + runRecordResult(store, { + now, + consumedTokens: 3, + processingFailures: 1, + }); + runRecordResult(store, { + now: now + 1000, + consumedTokens: 2, + processingFailures: 0, + }); + + const epHash = store.get("ep:t1")!; + expect(epHash.get("cur_attempts")).toBe("5"); + expect(epHash.get("cur_failures")).toBe("1"); + }); + }); + + describe("circuit opening", () => { + it("opens circuit when failure rate exceeds threshold", () => { + const store = createRedisStore(); + store.set( + "ep:t1", + new Map([ + ["is_open", "0"], + ["switched_at", "0"], + ["sample_till", "9999999999"], + ]), + ); + + const [circuitState, circuitSwitched] = runRecordResult(store, { + consumedTokens: 5, + processingFailures: 5, + minAttempts: 5, + failureThreshold: 0.3, + }); + expect(circuitState).toBe("open"); + expect(circuitSwitched).toBe(1); + }); + + it("does not open circuit when below minimum attempts", () => { + const store = createRedisStore(); + store.set( + "ep:t1", + new Map([ + ["is_open", "0"], + ["switched_at", "0"], + ["sample_till", "9999999999"], + ]), + ); + + const [circuitState, circuitSwitched] = runRecordResult(store, { + consumedTokens: 3, + processingFailures: 3, + minAttempts: 5, + failureThreshold: 0.3, + }); + expect(circuitState).toBe("closed"); + expect(circuitSwitched).toBe(0); + }); + + it("does not open circuit when failure rate equals threshold exactly", () => { + const store = createRedisStore(); + store.set( + "ep:t1", + new Map([ + ["is_open", "0"], + ["switched_at", "0"], + ["sample_till", "9999999999"], + ]), + ); + + const [circuitState, circuitSwitched] = runRecordResult(store, { + consumedTokens: 10, + processingFailures: 3, + minAttempts: 5, + failureThreshold: 0.3, + }); + expect(circuitState).toBe("closed"); + expect(circuitSwitched).toBe(0); + }); + + it("sets is_open and switched_at on open", () => { + const store = createRedisStore(); + const now = 1_000_000; + store.set( + "ep:t1", + new Map([ + ["is_open", "0"], + ["switched_at", "0"], + ["sample_till", "9999999999"], + ]), + ); + + runRecordResult(store, { + now, + consumedTokens: 5, + processingFailures: 5, + minAttempts: 5, + failureThreshold: 0.3, + }); + + const epHash = store.get("ep:t1")!; + expect(epHash.get("is_open")).toBe("1"); + expect(Number(epHash.get("switched_at"))).toBe(now); + }); + + it("resets all counters and sets sampleTill on open", () => { + const store = createRedisStore(); + const now = 1_000_000; + const samplePeriodMs = 300_000; + store.set( + "ep:t1", + new Map([ + ["is_open", "0"], + ["switched_at", "0"], + ["sample_till", "9999999999"], + ]), + ); + + runRecordResult(store, { + now, + consumedTokens: 5, + processingFailures: 5, + minAttempts: 5, + failureThreshold: 0.3, + samplePeriodMs, + }); + + const epHash = store.get("ep:t1")!; + expect(epHash.get("cur_failures")).toBe("0"); + expect(epHash.get("cur_attempts")).toBe("0"); + expect(epHash.get("prev_failures")).toBe("0"); + expect(epHash.get("prev_attempts")).toBe("0"); + expect(Number(epHash.get("sample_till"))).toBe(now + samplePeriodMs); + }); + + it("resets cooldown when a half-open probe succeeds but accumulated failures still exceed threshold", () => { + const store = createRedisStore(); + const now = 1_000_000; + const cooldownPeriodMs = 120_000; + const switchedAt = now - cooldownPeriodMs - 10_000; // past cooldown → half-open + + store.set( + "ep:t1", + new Map([ + ["is_open", "1"], + ["switched_at", switchedAt.toString()], + ["sample_till", (now + 300_000).toString()], + ["prev_attempts", "20"], + ["prev_failures", "20"], + ["cur_attempts", "0"], + ["cur_failures", "0"], + ]), + ); + + const [circuitState, circuitSwitched] = runRecordResult(store, { + now, + cooldownPeriodMs, + consumedTokens: 1, + processingFailures: 0, // probe succeeds + minAttempts: 5, + failureThreshold: 0.3, + }); + + expect(circuitState).toBe("open"); + expect(circuitSwitched).toBe(1); + + const epHash = store.get("ep:t1")!; + expect(Number(epHash.get("switched_at"))).toBe(now); // fresh cooldown + expect(epHash.get("cur_attempts")).toBe("0"); + expect(epHash.get("prev_attempts")).toBe("0"); + }); + }); + + describe("circuit closing", () => { + it("closes circuit when half-open and batch has successes", () => { + const store = createRedisStore(); + const now = 1_000_000; + const cooldownPeriodMs = 120_000; + const switchedAt = now - cooldownPeriodMs - 10_000; // past cooldown → half-open + + store.set( + "ep:t1", + new Map([ + ["is_open", "1"], + ["switched_at", switchedAt.toString()], + ["sample_till", "9999999999"], + ]), + ); + + const consumedTokens = 5; + const processingFailures = 1; // 4 successes out of 5 (rate 0.2 < 0.3 threshold, no re-open) + + const [circuitState, circuitSwitched] = runRecordResult(store, { + now, + cooldownPeriodMs, + consumedTokens, + processingFailures, + }); + + expect(circuitState).toBe("closed_recovery"); + expect(circuitSwitched).toBe(1); + + const epHash = store.get("ep:t1")!; + expect(epHash.get("is_open")).toBe("0"); + expect(Number(epHash.get("switched_at"))).toBe(now); + }); + + it("does not close when half-open but all attempts failed", () => { + const store = createRedisStore(); + const now = 1_000_000; + const switchedAt = now - 130_000; + + store.set( + "ep:t1", + new Map([ + ["is_open", "1"], + ["switched_at", switchedAt.toString()], + ["sample_till", "9999999999"], + ]), + ); + + const [circuitState, circuitSwitched] = runRecordResult(store, { + now, + cooldownPeriodMs: 120_000, + consumedTokens: 1, + processingFailures: 1, + }); + + expect(circuitState).toBe("open_half"); + expect(circuitSwitched).toBe(0); + }); + + it("does not close when processingFailures exceeds consumedTokens", () => { + const store = createRedisStore(); + const now = 1_000_000; + const switchedAt = now - 130_000; + + store.set( + "ep:t1", + new Map([ + ["is_open", "1"], + ["switched_at", switchedAt.toString()], + ["sample_till", "9999999999"], + ]), + ); + + const [circuitState, circuitSwitched] = runRecordResult(store, { + now, + cooldownPeriodMs: 120_000, + consumedTokens: 1, + processingFailures: 2, + }); + + expect(circuitState).toBe("open_half"); + expect(circuitSwitched).toBe(0); + }); + + it("reports closed when past recovery period", () => { + const store = createRedisStore(); + const now = 1_000_000; + const switchedAt = now - 700_000; + + store.set( + "ep:t1", + new Map([ + ["is_open", "0"], + ["switched_at", switchedAt.toString()], + ["sample_till", "9999999999"], + ]), + ); + + const [circuitState, circuitSwitched] = runRecordResult(store, { + now, + recoveryPeriodMs: 600_000, + consumedTokens: 1, + processingFailures: 0, + }); + + expect(circuitState).toBe("closed"); + expect(circuitSwitched).toBe(0); + }); + }); + + describe("sliding window", () => { + it("promotes current to previous when sampleTill expires", () => { + const store = createRedisStore(); + const now = 1_000_000; + const samplePeriodMs = 300_000; + const sampleTill = now - 1; + + store.set( + "ep:t1", + new Map([ + ["is_open", "0"], + ["switched_at", "0"], + ["sample_till", sampleTill.toString()], + ["cur_attempts", "10"], + ["cur_failures", "3"], + ["prev_attempts", "0"], + ["prev_failures", "0"], + ]), + ); + + runRecordResult(store, { now, samplePeriodMs, consumedTokens: 1 }); + + const epHash = store.get("ep:t1")!; + expect(epHash.get("prev_attempts")).toBe("10"); + expect(epHash.get("prev_failures")).toBe("3"); + expect(Number(epHash.get("sample_till"))).toBe( + sampleTill + samplePeriodMs, + ); + }); + + it("complete reset when window is too old", () => { + const store = createRedisStore(); + const now = 1_000_000; + const samplePeriodMs = 300_000; + const sampleTill = now - samplePeriodMs - 1; + + store.set( + "ep:t1", + new Map([ + ["is_open", "0"], + ["switched_at", "0"], + ["sample_till", sampleTill.toString()], + ["cur_attempts", "10"], + ["cur_failures", "3"], + ["prev_attempts", "5"], + ["prev_failures", "2"], + ]), + ); + + runRecordResult(store, { now, samplePeriodMs, consumedTokens: 1 }); + + const epHash = store.get("ep:t1")!; + expect(epHash.get("prev_attempts")).toBe("0"); + expect(epHash.get("prev_failures")).toBe("0"); + expect(Number(epHash.get("sample_till"))).toBe(now + samplePeriodMs); + }); + + it("does not promote when sampleTill equals now exactly", () => { + const store = createRedisStore(); + const now = 1_000_000; + + store.set( + "ep:t1", + new Map([ + ["is_open", "0"], + ["switched_at", "0"], + ["sample_till", now.toString()], + ["cur_attempts", "10"], + ["cur_failures", "3"], + ["prev_attempts", "0"], + ["prev_failures", "0"], + ]), + ); + + runRecordResult(store, { now, consumedTokens: 1, processingFailures: 0 }); + + const epHash = store.get("ep:t1")!; + expect(epHash.get("prev_attempts")).toBe("0"); + expect(epHash.get("cur_attempts")).toBe("11"); + }); + + it("interpolates using weight from sampleTill", () => { + const store = createRedisStore(); + const samplePeriodMs = 300_000; + const now = 1_000_000; + const sampleTill = now + samplePeriodMs / 2; // weight = 0.5 + const failureThreshold = 0.3; + const minAttempts = 10; + + store.set( + "ep:t1", + new Map([ + ["is_open", "0"], + ["switched_at", "0"], + ["sample_till", sampleTill.toString()], + ["prev_attempts", "20"], + ["prev_failures", "20"], // 100% failure in previous window + ]), + ); + + // Current batch: 1 attempt, 0 failures — not enough alone to trip + // weight = (sampleTill - now) / samplePeriodMs = 0.5 + // interpolated attempts = 20 * 0.5 + 1 = 11 (>= minAttempts 10) + // interpolated failures = 20 * 0.5 + 0 = 10 + // failure rate = 10/11 ≈ 0.91 > 0.3 → opens + const [circuitState, circuitSwitched] = runRecordResult(store, { + now, + samplePeriodMs, + consumedTokens: 1, + processingFailures: 0, + minAttempts, + failureThreshold, + }); + expect(circuitState).toBe("open"); + expect(circuitSwitched).toBe(1); + }); + }); + + describe("state persistence", () => { + it("writes all sampling fields to redis", () => { + const store = createRedisStore(); + store.set( + "ep:t1", + new Map([ + ["is_open", "0"], + ["switched_at", "0"], + ["sample_till", "9999999999"], + ]), + ); + runRecordResult(store); + + const epHash = store.get("ep:t1")!; + expect(epHash.has("cur_attempts")).toBe(true); + expect(epHash.has("cur_failures")).toBe(true); + expect(epHash.has("prev_attempts")).toBe(true); + expect(epHash.has("prev_failures")).toBe(true); + expect(epHash.has("sample_till")).toBe(true); + }); + + it("does not write is_open or switched_at when circuit does not switch", () => { + const store = createRedisStore(); + store.set( + "ep:t1", + new Map([ + ["is_open", "0"], + ["switched_at", "500"], + ["sample_till", "9999999999"], + ]), + ); + + runRecordResult(store, { consumedTokens: 1, processingFailures: 0 }); + + const epHash = store.get("ep:t1")!; + expect(epHash.get("switched_at")).toBe("500"); + }); + + it("isolates state between different targets", () => { + const store = createRedisStore(); + store.set( + "ep:target-a", + new Map([ + ["is_open", "0"], + ["switched_at", "0"], + ["sample_till", "9999999999"], + ]), + ); + store.set( + "ep:target-b", + new Map([ + ["is_open", "0"], + ["switched_at", "0"], + ["sample_till", "9999999999"], + ]), + ); + + runRecordResult( + store, + { consumedTokens: 5, processingFailures: 0 }, + "target-a", + ); + runRecordResult( + store, + { consumedTokens: 3, processingFailures: 2 }, + "target-b", + ); + + const hashA = store.get("ep:target-a")!; + const hashB = store.get("ep:target-b")!; + expect(hashA.get("cur_attempts")).toBe("5"); + expect(hashA.get("cur_failures")).toBe("0"); + expect(hashB.get("cur_attempts")).toBe("3"); + expect(hashB.get("cur_failures")).toBe("2"); + }); + }); +}); diff --git a/lambdas/https-client-lambda/src/__tests__/redis-client.test.ts b/lambdas/https-client-lambda/src/__tests__/redis-client.test.ts new file mode 100644 index 00000000..3cd9513f --- /dev/null +++ b/lambdas/https-client-lambda/src/__tests__/redis-client.test.ts @@ -0,0 +1,123 @@ +import { getRedisClient, resetRedisClient } from "services/redis-client"; + +jest.mock("@nhs-notify-client-callbacks/logger"); + +const mockPresign = jest.fn().mockResolvedValue({ + hostname: "cache.example.invalid", + path: "/", + query: { "X-Amz-Signature": "mock-sig" }, +}); + +jest.mock("@smithy/signature-v4", () => ({ + SignatureV4: jest.fn().mockImplementation(() => ({ presign: mockPresign })), +})); + +jest.mock("@aws-sdk/credential-providers", () => ({ + fromNodeProviderChain: jest.fn(), +})); + +const mockSendCommand = jest.fn(); +const mockConnect = jest.fn().mockResolvedValue(undefined); +const mockDisconnect = jest.fn().mockResolvedValue(undefined); +const mockOn = jest.fn(); + +jest.mock("@redis/client", () => ({ + createClient: jest.fn(() => ({ + sendCommand: mockSendCommand, + connect: mockConnect, + disconnect: mockDisconnect, + on: mockOn, + isOpen: true, + })), +})); + +beforeEach(() => { + jest.clearAllMocks(); + resetRedisClient(); + delete process.env.ELASTICACHE_ENDPOINT; + delete process.env.ELASTICACHE_CACHE_NAME; + delete process.env.ELASTICACHE_IAM_USERNAME; +}); + +describe("getRedisClient", () => { + it("throws when ELASTICACHE_ENDPOINT is not set", async () => { + await expect(getRedisClient()).rejects.toThrow( + "ELASTICACHE_ENDPOINT is required", + ); + }); + + it("throws when ELASTICACHE_IAM_USERNAME is not set", async () => { + process.env.ELASTICACHE_ENDPOINT = "cache.example.invalid"; + + await expect(getRedisClient()).rejects.toThrow( + "ELASTICACHE_IAM_USERNAME is required", + ); + }); + + it("throws when ELASTICACHE_CACHE_NAME is not set", async () => { + process.env.ELASTICACHE_ENDPOINT = "cache.example.invalid"; + process.env.ELASTICACHE_IAM_USERNAME = "iam-user"; + + await expect(getRedisClient()).rejects.toThrow( + "ELASTICACHE_CACHE_NAME, ELASTICACHE_ENDPOINT, and ELASTICACHE_IAM_USERNAME are required", + ); + }); + + it("creates and connects a Redis client with IAM token", async () => { + process.env.ELASTICACHE_ENDPOINT = "cache.example.invalid"; + process.env.ELASTICACHE_CACHE_NAME = "my-cache"; + process.env.ELASTICACHE_IAM_USERNAME = "iam-user"; + + const client = await getRedisClient(); + + expect(client).toBeDefined(); + expect(mockPresign).toHaveBeenCalled(); + expect(mockConnect).toHaveBeenCalled(); + }); + + it("returns cached client when already open and token is valid", async () => { + process.env.ELASTICACHE_ENDPOINT = "cache.example.invalid"; + process.env.ELASTICACHE_CACHE_NAME = "my-cache"; + process.env.ELASTICACHE_IAM_USERNAME = "iam-user"; + + const first = await getRedisClient(); + const second = await getRedisClient(); + + expect(first).toBe(second); + expect(mockConnect).toHaveBeenCalledTimes(1); + expect(mockPresign).toHaveBeenCalledTimes(1); + }); + + it("registers error handler on client", async () => { + process.env.ELASTICACHE_ENDPOINT = "cache.example.invalid"; + process.env.ELASTICACHE_CACHE_NAME = "my-cache"; + process.env.ELASTICACHE_IAM_USERNAME = "iam-user"; + + await getRedisClient(); + + expect(mockOn).toHaveBeenCalledWith("error", expect.any(Function)); + + const errorHandler = mockOn.mock.calls.find( + (c: unknown[]) => c[0] === "error", + )![1] as (err: Error) => void; + errorHandler(new Error("test error")); + }); + + it("disconnects existing client when token expires before reconnecting", async () => { + jest.useFakeTimers(); + process.env.ELASTICACHE_ENDPOINT = "cache.example.invalid"; + process.env.ELASTICACHE_CACHE_NAME = "my-cache"; + process.env.ELASTICACHE_IAM_USERNAME = "iam-user"; + + await getRedisClient(); + + jest.advanceTimersByTime(841_000); + + await getRedisClient(); + + expect(mockDisconnect).toHaveBeenCalledTimes(1); + expect(mockConnect).toHaveBeenCalledTimes(2); + + jest.useRealTimers(); + }); +}); diff --git a/lambdas/https-client-lambda/src/__tests__/retry-policy.test.ts b/lambdas/https-client-lambda/src/__tests__/retry-policy.test.ts new file mode 100644 index 00000000..31a83979 --- /dev/null +++ b/lambdas/https-client-lambda/src/__tests__/retry-policy.test.ts @@ -0,0 +1,253 @@ +import type { SQSRecord } from "aws-lambda"; +import { + exceedsSqsMaxVisibility, + handleRateLimitedRecord, + isWindowExhausted, + jitteredBackoffSeconds, + parseRetryAfter, +} from "services/delivery/retry-policy"; + +const mockSendToDlq = jest.fn(); +jest.mock("services/dlq-sender", () => ({ + sendToDlq: (...args: unknown[]) => mockSendToDlq(...args), +})); + +const mockChangeVisibility = jest.fn(); +jest.mock("services/sqs-visibility", () => ({ + changeVisibility: (...args: unknown[]) => mockChangeVisibility(...args), +})); + +jest.mock("@nhs-notify-client-callbacks/logger", () => ({ + logger: { info: jest.fn(), warn: jest.fn(), error: jest.fn() }, +})); + +describe("jitteredBackoffSeconds", () => { + it("produces value in [1, 5) at receiveCount=1", () => { + for (let i = 0; i < 100; i++) { + const val = jitteredBackoffSeconds(1); + expect(val).toBeGreaterThanOrEqual(1); + expect(val).toBeLessThan(5); + } + }); + + it("produces value in [1, 300) at receiveCount=10 (cap)", () => { + for (let i = 0; i < 100; i++) { + const val = jitteredBackoffSeconds(10); + expect(val).toBeGreaterThanOrEqual(1); + expect(val).toBeLessThan(300); + } + }); + + it("respects cap at very high receiveCount", () => { + for (let i = 0; i < 50; i++) { + const val = jitteredBackoffSeconds(100); + expect(val).toBeLessThan(300); + } + }); +}); + +describe("parseRetryAfter", () => { + beforeEach(() => { + jest.useFakeTimers({ now: 10_000_000 }); + }); + + afterEach(() => { + jest.useRealTimers(); + }); + + it("parses integer string", () => { + expect(parseRetryAfter("120")).toBe(120); + }); + + it("preserves negative values", () => { + expect(parseRetryAfter("-5")).toBe(-5); + }); + + it("parses HTTP date string", () => { + const futureDate = new Date(10_060_000); + expect(parseRetryAfter(futureDate.toUTCString())).toBe(60); + }); + + it("returns 0 for past HTTP date", () => { + const pastDate = new Date(9_940_000); + expect(parseRetryAfter(pastDate.toUTCString())).toBe(0); + }); + + it("returns 0 for garbage input", () => { + expect(parseRetryAfter("not-a-date-or-number")).toBe(0); + }); +}); + +describe("isWindowExhausted", () => { + beforeEach(() => { + jest.useFakeTimers({ now: 10_000 }); + }); + + afterEach(() => { + jest.useRealTimers(); + }); + + it("returns false just below limit", () => { + expect(isWindowExhausted(9001, 1000)).toBe(false); + }); + + it("returns true at limit", () => { + expect(isWindowExhausted(9000, 1000)).toBe(true); + }); + + it("returns true beyond limit", () => { + expect(isWindowExhausted(8000, 1000)).toBe(true); + }); +}); + +describe("exceedsSqsMaxVisibility", () => { + it("returns false at 43200", () => { + expect(exceedsSqsMaxVisibility(43_200)).toBe(false); + }); + + it("returns true at 43201", () => { + expect(exceedsSqsMaxVisibility(43_201)).toBe(true); + }); +}); + +const makeRecord = (overrides: Partial = {}): SQSRecord => ({ + messageId: "msg-1", + receiptHandle: "receipt-1", + body: JSON.stringify({ + payload: {}, + subscriptionId: "sub-1", + targetId: "target-1", + }), + attributes: { + ApproximateReceiveCount: "1", + SentTimestamp: "0", + SenderId: "sender", + ApproximateFirstReceiveTimestamp: "0", + }, + messageAttributes: {}, + md5OfBody: "abc", + eventSource: "aws:sqs", + eventSourceARN: "arn:aws:sqs:eu-west-2:123:queue", + awsRegion: "eu-west-2", + ...overrides, +}); + +describe("handleRateLimitedRecord", () => { + beforeEach(() => { + jest.clearAllMocks(); + mockSendToDlq.mockResolvedValue(undefined); + mockChangeVisibility.mockResolvedValue(undefined); + }); + + describe("429 under retry period", () => { + it("retries with Retry-After delay when within SQS max visibility", async () => { + const result = await handleRateLimitedRecord( + makeRecord(), + "client-1", + "target-1", + "120", + 1, + ); + + expect(result).toBe("retry"); + expect(mockChangeVisibility).toHaveBeenCalledWith("receipt-1", 120); + expect(mockSendToDlq).not.toHaveBeenCalled(); + }); + + it("retries with jittered backoff when no Retry-After header provided", async () => { + const result = await handleRateLimitedRecord( + makeRecord(), + "client-1", + "target-1", + undefined, + 1, + ); + + expect(result).toBe("retry"); + + expect(mockChangeVisibility).toHaveBeenCalled(); + const [, delaySec] = mockChangeVisibility.mock.calls[0] as [ + string, + number, + ]; + expect(delaySec).toBeGreaterThanOrEqual(1); + expect(delaySec).toBeLessThan(5); + expect(mockSendToDlq).not.toHaveBeenCalled(); + }); + }); + + describe("429 over retry period but under SQS limit", () => { + it("retries when Retry-After exceeds max retry duration but is within SQS max visibility", async () => { + const result = await handleRateLimitedRecord( + makeRecord(), + "client-1", + "target-1", + "10000", + 1, + ); + + expect(result).toBe("retry"); + expect(mockChangeVisibility).toHaveBeenCalledWith("receipt-1", 10_000); + expect(mockSendToDlq).not.toHaveBeenCalled(); + }); + + it("retries at SQS max visibility boundary (43200s)", async () => { + const result = await handleRateLimitedRecord( + makeRecord(), + "client-1", + "target-1", + "43200", + 1, + ); + + expect(result).toBe("retry"); + expect(mockChangeVisibility).toHaveBeenCalledWith("receipt-1", 43_200); + expect(mockSendToDlq).not.toHaveBeenCalled(); + }); + }); + + describe("429 over SQS limit", () => { + it("sends to DLQ when Retry-After exceeds SQS max visibility", async () => { + const result = await handleRateLimitedRecord( + makeRecord(), + "client-1", + "target-1", + "50000", + 1, + ); + + expect(result).toBe("dlq"); + expect(mockSendToDlq).toHaveBeenCalledWith(makeRecord().body); + expect(mockChangeVisibility).not.toHaveBeenCalled(); + }); + + it("returns dlq without throwing", async () => { + const result = await handleRateLimitedRecord( + makeRecord(), + "client-1", + "target-1", + "43201", + 1, + ); + + expect(result).toBe("dlq"); + expect(mockSendToDlq).toHaveBeenCalledWith(makeRecord().body); + }); + }); + + describe("429 with negative Retry-After", () => { + it("sends to DLQ when Retry-After is negative", async () => { + const result = await handleRateLimitedRecord( + makeRecord(), + "client-1", + "target-1", + "-1", + 1, + ); + + expect(result).toBe("dlq"); + expect(mockSendToDlq).toHaveBeenCalledWith(makeRecord().body); + expect(mockChangeVisibility).not.toHaveBeenCalled(); + }); + }); +}); diff --git a/lambdas/https-client-lambda/src/__tests__/sqs-visibility.test.ts b/lambdas/https-client-lambda/src/__tests__/sqs-visibility.test.ts new file mode 100644 index 00000000..2bdc0474 --- /dev/null +++ b/lambdas/https-client-lambda/src/__tests__/sqs-visibility.test.ts @@ -0,0 +1,72 @@ +import { ChangeMessageVisibilityCommand } from "@aws-sdk/client-sqs"; + +import { changeVisibility } from "services/sqs-visibility"; + +const mockSend = jest.fn(); +jest.mock("@aws-sdk/client-sqs", () => { + const actual = jest.requireActual("@aws-sdk/client-sqs"); + return { + ...actual, + SQSClient: jest.fn().mockImplementation(() => ({ + send: (...args: unknown[]) => mockSend(...args), + })), + }; +}); + +describe("changeVisibility", () => { + beforeEach(() => { + mockSend.mockReset(); + process.env.QUEUE_URL = + "https://sqs.eu-west-2.invalid/123456789/test-queue"; + }); + + afterEach(() => { + delete process.env.QUEUE_URL; + }); + + it("sends ChangeMessageVisibilityCommand with correct params", async () => { + mockSend.mockResolvedValue({}); + + await changeVisibility("receipt-handle-1", 30); + + expect(mockSend).toHaveBeenCalledTimes(1); + const command = mockSend.mock.calls[0][0]; + expect(command).toBeInstanceOf(ChangeMessageVisibilityCommand); + expect(command.input).toEqual({ + QueueUrl: "https://sqs.eu-west-2.invalid/123456789/test-queue", + ReceiptHandle: "receipt-handle-1", + VisibilityTimeout: 30, + }); + }); + + it("floors fractional visibility timeout", async () => { + mockSend.mockResolvedValue({}); + + await changeVisibility("receipt-handle-1", 30.7); + + const command = mockSend.mock.calls[0][0]; + expect(command.input.VisibilityTimeout).toBe(30); + }); + + it("surfaces SDK errors", async () => { + mockSend.mockRejectedValue(new Error("SQS error")); + + await expect(changeVisibility("receipt-handle-1", 30)).rejects.toThrow( + "SQS error", + ); + }); + + it("throws when QUEUE_URL is not set", async () => { + let changeFn: typeof changeVisibility; + delete process.env.QUEUE_URL; + + jest.isolateModules(() => { + // eslint-disable-next-line @typescript-eslint/no-require-imports -- jest.isolateModules requires synchronous require + changeFn = require("services/sqs-visibility").changeVisibility; + }); + + await expect(changeFn!("receipt-handle-1", 30)).rejects.toThrow( + "QUEUE_URL is required", + ); + }); +}); diff --git a/lambdas/https-client-lambda/src/__tests__/tls-agent-factory.test.ts b/lambdas/https-client-lambda/src/__tests__/tls-agent-factory.test.ts new file mode 100644 index 00000000..09a54802 --- /dev/null +++ b/lambdas/https-client-lambda/src/__tests__/tls-agent-factory.test.ts @@ -0,0 +1,409 @@ +import type { CallbackTarget } from "@nhs-notify-client-callbacks/models"; + +const mockS3Send = jest.fn(); +jest.mock("@aws-sdk/client-s3", () => { + const actual = jest.requireActual("@aws-sdk/client-s3"); + return { + ...actual, + S3Client: jest.fn().mockImplementation(() => ({ send: mockS3Send })), + }; +}); + +jest.mock("@nhs-notify-client-callbacks/logger", () => ({ + logger: { + info: jest.fn(), + warn: jest.fn(), + error: jest.fn(), + debug: jest.fn(), + }, +})); + +jest.mock("node-forge", () => ({ + pem: { + decode: jest.fn((input: string) => { + const matches = [ + ...(input ?? "").matchAll( + /-----BEGIN ([^-]+)-----[\s\S]*?-----END [^-]+-----/g, + ), + ]; + return matches.map((match) => ({ + type: (match[1] ?? "").trim(), + body: "", + })); + }), + encode: jest.fn( + (obj: { type: string }) => + `-----BEGIN ${obj.type}-----\nZmFrZQ==\n-----END ${obj.type}-----\n`, + ), + }, +})); + +const mockValidTo = new Date(Date.now() + 365 * 86_400_000).toISOString(); + +jest.mock("node:crypto", () => { + const actual = jest.requireActual("node:crypto"); + return { + ...actual, + X509Certificate: class MockX509Certificate { + validTo = mockValidTo; + + publicKey = { + export: () => Buffer.from("mock-spki-der"), + }; + }, + }; +}); + +const TEST_KEY = + "-----BEGIN PRIVATE KEY-----\nfake-key\n-----END PRIVATE KEY-----"; // gitleaks:allow +const TEST_CERT = + "-----BEGIN CERTIFICATE-----\nfake-cert\n-----END CERTIFICATE-----"; +const COMBINED_PEM = `${TEST_KEY}\n${TEST_CERT}`; + +const createTarget = ( + overrides: Partial = {}, +): CallbackTarget => ({ + targetId: "target-1", + type: "API", + invocationEndpoint: "https://webhook.example.invalid", + invocationMethod: "POST", + invocationRateLimit: 10, + apiKey: { headerName: "x-api-key", headerValue: "secret" }, + ...overrides, +}); + +const mockS3PemResponse = (pem: string) => { + mockS3Send.mockResolvedValue({ + Body: { transformToString: jest.fn().mockResolvedValue(pem) }, + }); +}; + +describe("tls-agent-factory", () => { + let buildAgent: typeof import("services/delivery/tls-agent-factory").buildAgent; + let resetCache: typeof import("services/delivery/tls-agent-factory").resetCache; + + beforeEach(async () => { + jest.resetModules(); + + process.env.MTLS_CERT_S3_BUCKET = "test-certs-bucket"; + process.env.MTLS_CERT_S3_KEY = "client.pem"; + delete process.env.MTLS_CA_S3_KEY; + process.env.CERT_EXPIRY_THRESHOLD_MS = "86400000"; + + // @ts-expect-error -- modulePaths resolves at runtime + const mod = await import("services/delivery/tls-agent-factory"); + buildAgent = mod.buildAgent; + resetCache = mod.resetCache; + + mockS3Send.mockReset(); + }); + + it("builds agent with key and cert when mtls is enabled", async () => { + mockS3PemResponse(COMBINED_PEM); + const agent = await buildAgent( + createTarget({ delivery: { mtls: { enabled: true } } }), + ); + + expect(agent).toBeDefined(); + expect(agent.options.keepAlive).toBe(false); + expect(mockS3Send).toHaveBeenCalled(); + }); + + it("builds agent without key and cert when mtls is disabled", async () => { + const agent = await buildAgent(createTarget()); + + expect(agent).toBeDefined(); + expect(agent.options.keepAlive).toBe(false); + expect(mockS3Send).not.toHaveBeenCalled(); + }); + + it("loads CA for server trust when MTLS_CA_S3_KEY is set and mtls is disabled", async () => { + process.env.MTLS_CA_S3_KEY = "test-ca.pem"; + jest.resetModules(); + // @ts-expect-error -- modulePaths resolves at runtime + const mod = await import("services/delivery/tls-agent-factory"); + + const caPem = + "-----BEGIN CERTIFICATE-----\ntest-ca\n-----END CERTIFICATE-----"; + mockS3Send + .mockResolvedValueOnce({ + Body: { + transformToString: jest.fn().mockResolvedValue(COMBINED_PEM), + }, + }) + .mockResolvedValueOnce({ + Body: { transformToString: jest.fn().mockResolvedValue(caPem) }, + }); + + const agent = await mod.buildAgent( + createTarget({ delivery: { mtls: { enabled: false } } }), + ); + + expect(agent).toBeDefined(); + expect(agent.options.ca).toBe(caPem); + expect(agent.options.key).toBeUndefined(); + expect(agent.options.cert).toBeUndefined(); + }); + + it("loads CA when MTLS_CA_S3_KEY is set", async () => { + process.env.MTLS_CA_S3_KEY = "test-ca.pem"; + jest.resetModules(); + // @ts-expect-error -- modulePaths resolves at runtime + const mod = await import("services/delivery/tls-agent-factory"); + + const caPem = + "-----BEGIN CERTIFICATE-----\ntest-ca\n-----END CERTIFICATE-----"; + mockS3Send + .mockResolvedValueOnce({ + Body: { + transformToString: jest.fn().mockResolvedValue(COMBINED_PEM), + }, + }) + .mockResolvedValueOnce({ + Body: { transformToString: jest.fn().mockResolvedValue(caPem) }, + }); + + const agent = await mod.buildAgent( + createTarget({ delivery: { mtls: { enabled: true } } }), + ); + + expect(agent).toBeDefined(); + expect(mockS3Send).toHaveBeenCalledTimes(2); + }); + + it("loads cert from S3", async () => { + mockS3PemResponse(COMBINED_PEM); + await buildAgent(createTarget({ delivery: { mtls: { enabled: true } } })); + + expect(mockS3Send).toHaveBeenCalledTimes(1); + }); + + it("caches cert material on subsequent calls", async () => { + mockS3PemResponse(COMBINED_PEM); + const target = createTarget({ delivery: { mtls: { enabled: true } } }); + + await buildAgent(target); + await buildAgent(target); + + expect(mockS3Send).toHaveBeenCalledTimes(1); + }); + + it("exports PERMANENT_TLS_ERROR_CODES set", async () => { + // @ts-expect-error -- modulePaths resolves at runtime + const mod = await import("services/delivery/tls-agent-factory"); + + expect(mod.PERMANENT_TLS_ERROR_CODES).toBeInstanceOf(Set); + expect(mod.PERMANENT_TLS_ERROR_CODES.has("CERT_HAS_EXPIRED")).toBe(true); + }); + + it("resets cached material via resetCache", async () => { + mockS3PemResponse(COMBINED_PEM); + const target = createTarget({ delivery: { mtls: { enabled: true } } }); + + await buildAgent(target); + resetCache(); + await buildAgent(target); + + expect(mockS3Send).toHaveBeenCalledTimes(2); + }); + + it("throws when S3 env vars are missing", async () => { + delete process.env.MTLS_CERT_S3_BUCKET; + delete process.env.MTLS_CERT_S3_KEY; + jest.resetModules(); + // @ts-expect-error -- modulePaths resolves at runtime + const mod = await import("services/delivery/tls-agent-factory"); + + await expect( + mod.buildAgent(createTarget({ delivery: { mtls: { enabled: true } } })), + ).rejects.toThrow("MTLS_CERT_S3_BUCKET and MTLS_CERT_S3_KEY are required"); + }); + + it("throws when S3 object body is empty", async () => { + mockS3Send.mockResolvedValue({ Body: undefined }); + + await expect( + buildAgent(createTarget({ delivery: { mtls: { enabled: true } } })), + ).rejects.toThrow("has no body"); + }); + + it("builds agent with checkServerIdentity when certPinning is enabled", async () => { + mockS3PemResponse(COMBINED_PEM); + const target = createTarget({ + delivery: { + mtls: { + enabled: true, + certPinning: { enabled: true, spkiHash: "abc123" }, + }, + }, + }); + + const agent = await buildAgent(target); + + expect(agent).toBeDefined(); + expect(agent.options.checkServerIdentity).toBeDefined(); + }); + + it("checkServerIdentity returns error when SPKI hash does not match", async () => { + mockS3PemResponse(COMBINED_PEM); + const target = createTarget({ + delivery: { + mtls: { + enabled: true, + certPinning: { enabled: true, spkiHash: "expected-hash" }, + }, + }, + }); + + const agent = await buildAgent(target); + const checkFn = agent.options.checkServerIdentity as ( + hostname: string, + cert: { raw: Buffer; subject: { CN: string } }, + ) => Error | undefined; + + const mockPeerCert = { + raw: Buffer.from("mock-cert-der"), + subject: { CN: "webhook.example.invalid" }, + subjectaltname: "DNS:webhook.example.invalid", + }; + + const result = checkFn("webhook.example.invalid", mockPeerCert); + + expect(result).toBeInstanceOf(Error); + expect(result!.message).toContain("Certificate pinning failed"); + expect((result as NodeJS.ErrnoException).code).toBe( + "ERR_CERT_PINNING_FAILED", + ); + }); + + it("checkServerIdentity returns undefined when SPKI hash matches", async () => { + const { createHash } = jest.requireActual("node:crypto"); + const expectedHash = createHash("sha256") + .update(Buffer.from("mock-spki-der")) + .digest("base64"); + + mockS3PemResponse(COMBINED_PEM); + const target = createTarget({ + delivery: { + mtls: { + enabled: true, + certPinning: { enabled: true, spkiHash: expectedHash }, + }, + }, + }); + + const agent = await buildAgent(target); + const checkFn = agent.options.checkServerIdentity as ( + hostname: string, + cert: { raw: Buffer; subject: { CN: string } }, + ) => Error | undefined; + + const mockPeerCert = { + raw: Buffer.from("mock-cert-der"), + subject: { CN: "webhook.example.invalid" }, + subjectaltname: "DNS:webhook.example.invalid", + }; + + const result = checkFn("webhook.example.invalid", mockPeerCert); + + expect(result).toBeUndefined(); + }); + + it("checkServerIdentity returns default error when hostname does not match", async () => { + mockS3PemResponse(COMBINED_PEM); + const target = createTarget({ + delivery: { + mtls: { + enabled: true, + certPinning: { enabled: true, spkiHash: "abc" }, + }, + }, + }); + + const agent = await buildAgent(target); + const checkFn = agent.options.checkServerIdentity as ( + hostname: string, + cert: { raw: Buffer; subject: { CN: string } }, + ) => Error | undefined; + + const mockPeerCert = { + raw: Buffer.from("mock-cert-der"), + subject: { CN: "other.example.invalid" }, + subjectaltname: "DNS:other.example.invalid", + }; + + const result = checkFn("webhook.example.invalid", mockPeerCert); + + expect(result).toBeDefined(); + expect(result!.message).toContain("does not match"); + }); + + it("does not load cert material when mtls is disabled", async () => { + const agent = await buildAgent(createTarget()); + + expect(agent).toBeDefined(); + expect(mockS3Send).not.toHaveBeenCalled(); + }); + + it("throws when certPinning.enabled is true but spkiHash is missing", async () => { + const target = createTarget({ + delivery: { + mtls: { + enabled: true, + certPinning: { enabled: true }, + }, + }, + }); + + await expect(buildAgent(target)).rejects.toThrow( + "certPinning.spkiHash is required when certPinning is enabled", + ); + expect(mockS3Send).not.toHaveBeenCalled(); + }); + + it("uses default CERT_EXPIRY_THRESHOLD_MS when env var is not set", async () => { + delete process.env.CERT_EXPIRY_THRESHOLD_MS; + jest.resetModules(); + // @ts-expect-error -- modulePaths resolves at runtime + const mod = await import("services/delivery/tls-agent-factory"); + + mockS3PemResponse(COMBINED_PEM); + const agent = await mod.buildAgent( + createTarget({ delivery: { mtls: { enabled: true } } }), + ); + + expect(agent).toBeDefined(); + }); + + it("handles PEM with no private key or certificate sections", async () => { + mockS3Send.mockResolvedValue({ + Body: { + transformToString: jest.fn().mockResolvedValue("no-pem-content"), + }, + }); + + const agent = await buildAgent( + createTarget({ delivery: { mtls: { enabled: true } } }), + ); + + expect(agent).toBeDefined(); + }); + + it("reloads cert material when cached cert is approaching expiry", async () => { + // Use a threshold larger than the mock cert's remaining validity (365 days) + process.env.CERT_EXPIRY_THRESHOLD_MS = String(400 * 86_400_000); + jest.resetModules(); + // @ts-expect-error -- modulePaths resolves at runtime + const mod = await import("services/delivery/tls-agent-factory"); + + mockS3PemResponse(COMBINED_PEM); + const target = createTarget({ delivery: { mtls: { enabled: true } } }); + + await mod.buildAgent(target); + + mockS3PemResponse(COMBINED_PEM); + await mod.buildAgent(target); + + expect(mockS3Send).toHaveBeenCalledTimes(2); + }); +}); diff --git a/lambdas/https-client-lambda/src/handler.ts b/lambdas/https-client-lambda/src/handler.ts new file mode 100644 index 00000000..89975f28 --- /dev/null +++ b/lambdas/https-client-lambda/src/handler.ts @@ -0,0 +1,520 @@ +import type { SQSBatchItemFailure, SQSRecord } from "aws-lambda"; +import type { ClientCallbackPayload } from "@nhs-notify-client-callbacks/models"; +import pMap from "p-map"; +import { logger } from "@nhs-notify-client-callbacks/logger"; +import { loadTargetConfig } from "services/config-loader"; +import { getApplicationId } from "services/applications-map"; +import { signPayload } from "services/payload-signer"; +import { buildAgent } from "services/delivery/tls-agent-factory"; +import { + OUTCOME_PERMANENT_FAILURE, + OUTCOME_RATE_LIMITED, + OUTCOME_SUCCESS, + deliverPayload, +} from "services/delivery/https-client"; +import { sendToDlq } from "services/dlq-sender"; +import { changeVisibility } from "services/sqs-visibility"; +import { + handleRateLimitedRecord, + isWindowExhausted, + jitteredBackoffSeconds, +} from "services/delivery/retry-policy"; +import { + type EndpointGateConfig, + admit, + recordResult, +} from "services/endpoint-gate"; +import { getRedisClient } from "services/redis-client"; +import { + recordAdmissionDenied, + recordCircuitBreakerClosed, + recordCircuitBreakerOpen, + recordDeliveryAttempt, + recordDeliveryDuration, + recordDeliveryFailure, + recordDeliveryPermanentFailure, + recordDeliveryRateLimited, + recordDeliverySuccess, + recordRetryWindowExhausted, +} from "services/delivery-observability"; +import { flushMetrics, resetMetrics } from "services/delivery-metrics"; + +type RedisClientType = Awaited>; + +const DEFAULT_MAX_RETRY_DURATION_MS = 7_200_000; // 2 hours +const DEFAULT_CONCURRENCY_LIMIT = 10; +const BURST_MULTIPLIER = 5; +const MAX_BURST_CAPACITY = Number( + process.env.TOKEN_BUCKET_BURST_CAPACITY ?? "2250", +); +const SQS_MAX_VISIBILITY_TIMEOUT_SEC = 43_200; // 12 hours + +const gateConfig: EndpointGateConfig = { + // Max tokens the bucket can hold — absorbs short traffic bursts without throttling + burstCapacity: MAX_BURST_CAPACITY, + // Probe rate to test endpoint recovery when half-open (default: 1/60 req/s) + probeRateLimit: Number(process.env.CB_PROBE_RATE_LIMIT ?? String(1 / 60)), + // Linear ramp-up after circuit closes, avoids flooding a freshly recovered endpoint (default: 10 min) + recoveryPeriodMs: Number(process.env.CB_RECOVERY_PERIOD_MS ?? "600000"), + // Sliding window over which failure rates are sampled (default: 5 min) + samplePeriodMs: Number(process.env.CB_SAMPLE_PERIOD_MS ?? "300000"), + // Failure rate within the sample window that triggers circuit open (default: 30%) + failureThreshold: Number(process.env.CB_FAILURE_THRESHOLD ?? "0.3"), + // Minimum attempts in the sample window before the failure rate is evaluated (default: 5 attempts) + minAttempts: Number(process.env.CB_MIN_ATTEMPTS ?? "5"), + // Full block after circuit opens, before half-open probes begin (default: 2 min) + cooldownPeriodMs: Number(process.env.CB_COOLDOWN_PERIOD_MS ?? "120000"), +}; + +type CallbackDeliveryMessage = { + payload: ClientCallbackPayload; + subscriptionId: string; + targetId: string; +}; + +type DeliveryOutcome = "success" | "retry" | "dlq"; + +type TargetBatch = { + targetId: string; + records: SQSRecord[]; + messages: CallbackDeliveryMessage[]; +}; + +type TargetBatchResult = { + failures: SQSBatchItemFailure[]; + deliveredCount: number; + dlqCount: number; +}; + +function groupByTarget( + records: SQSRecord[], + unparseable: SQSRecord[], +): TargetBatch[] { + const groups = new Map< + string, + { records: SQSRecord[]; messages: CallbackDeliveryMessage[] } + >(); + + for (const record of records) { + let message: CallbackDeliveryMessage | undefined; + try { + message = JSON.parse(record.body); + } catch { + unparseable.push(record); + } + if (message) { + const existing = groups.get(message.targetId); + if (existing) { + existing.records.push(record); + existing.messages.push(message); + } else { + groups.set(message.targetId, { + records: [record], + messages: [message], + }); + } + } + } + + return [...groups.entries()].map( + ([targetId, { messages, records: recs }]) => ({ + targetId, + records: recs, + messages, + }), + ); +} + +function extractCorrelationId( + message: CallbackDeliveryMessage, +): string | undefined { + return message.payload.data[0]?.attributes?.messageId; +} + +async function deliverRecord( + record: SQSRecord, + message: CallbackDeliveryMessage, + target: Awaited>, + applicationId: string, + clientId: string, +): Promise { + const correlationId = extractCorrelationId(message); + const receiveCount = Number(record.attributes.ApproximateReceiveCount); + + logger.info("Processing delivery record", { + correlationId, + receiveCount, + firstReceivedAt: new Date( + Number(record.attributes.ApproximateFirstReceiveTimestamp), + ).toISOString(), + }); + + const maxRetryDurationMs = + target.delivery?.maxRetryDurationSeconds === undefined + ? DEFAULT_MAX_RETRY_DURATION_MS + : target.delivery.maxRetryDurationSeconds * 1000; + + const firstReceivedMs = Number( + record.attributes.ApproximateFirstReceiveTimestamp, + ); + + if (isWindowExhausted(firstReceivedMs, maxRetryDurationMs)) { + recordRetryWindowExhausted(clientId, message.targetId, correlationId); + await sendToDlq(record.body); + return "dlq"; + } + + const agent = await buildAgent(target); + const signature = signPayload( + applicationId, + target.apiKey.headerValue, + message.payload, + ); + const payloadJson = JSON.stringify(message.payload); + + recordDeliveryAttempt( + clientId, + message.targetId, + correlationId, + record.messageId, + receiveCount, + ); + const deliveryStart = Date.now(); + const result = await deliverPayload(target, payloadJson, signature, agent); + recordDeliveryDuration(message.targetId, Date.now() - deliveryStart); + + if (result.outcome === OUTCOME_SUCCESS) { + recordDeliverySuccess(clientId, message.targetId, correlationId); + return "success"; + } + + if (result.outcome === OUTCOME_PERMANENT_FAILURE) { + recordDeliveryPermanentFailure( + clientId, + message.targetId, + result.statusCode, + result.errorCode, + correlationId, + ); + await sendToDlq(record.body, result); + return "dlq"; + } + + if (result.outcome === OUTCOME_RATE_LIMITED) { + recordDeliveryRateLimited(clientId, message.targetId, correlationId); + return handleRateLimitedRecord( + record, + clientId, + message.targetId, + result.retryAfterHeader, + receiveCount, + ); + } + + const backoffSec = jitteredBackoffSeconds(receiveCount); + recordDeliveryFailure( + clientId, + message.targetId, + result.statusCode, + backoffSec, + receiveCount, + correlationId, + ); + await changeVisibility(record.receiptHandle, backoffSec); + return "retry"; +} + +async function denyRecords( + records: SQSRecord[], + messages: CallbackDeliveryMessage[], + clientId: string, + targetId: string, + reason: string, + delaySec: (receiveCount: number) => number, +): Promise { + recordAdmissionDenied( + clientId, + targetId, + reason, + messages.map((m) => extractCorrelationId(m)), + ); + const failures: SQSBatchItemFailure[] = []; + for (const record of records) { + const receiveCount = Number(record.attributes.ApproximateReceiveCount); + await changeVisibility(record.receiptHandle, delaySec(receiveCount)); + failures.push({ itemIdentifier: record.messageId }); + } + return failures; +} + +async function handleBatchDenied( + batch: TargetBatch, + clientId: string, + reason: string, + retryAfterMs: number, +): Promise { + const baseDelaySec = Math.max(1, Math.ceil(retryAfterMs / 1000)); + const failures = await denyRecords( + batch.records, + batch.messages, + clientId, + batch.targetId, + reason, + (receiveCount) => + Math.min(receiveCount * baseDelaySec, SQS_MAX_VISIBILITY_TIMEOUT_SEC), + ); + return { failures, deliveredCount: 0, dlqCount: 0 }; +} + +async function deliverAdmittedRecords( + records: SQSRecord[], + messages: CallbackDeliveryMessage[], + target: Awaited>, + applicationId: string, + clientId: string, + concurrencyLimit: number, +): Promise<{ + deliveredCount: number; + dlqCount: number; + failures: SQSBatchItemFailure[]; +}> { + const pairs = records.map( + (record, i): { message: CallbackDeliveryMessage; record: SQSRecord } => ({ + message: messages[i], // eslint-disable-line security/detect-object-injection -- numeric .map() index + record, + }), + ); + + const results = await pMap( + pairs, + async ({ + message, + record, + }): Promise<{ outcome: DeliveryOutcome; record: SQSRecord }> => { + try { + const outcome = await deliverRecord( + record, + message, + target, + applicationId, + clientId, + ); + return { outcome, record }; + } catch (error) { + const correlationId = extractCorrelationId(message); + logger.error("Failed to process record", { + messageId: record.messageId, + correlationId, + err: error, + }); + + try { + await sendToDlq(record.body); + return { outcome: "dlq", record }; + } catch (dlqError) { + logger.error("DLQ send also failed — returning for retry", { + messageId: record.messageId, + err: dlqError, + }); + return { outcome: "retry", record }; + } + } + }, + { concurrency: concurrencyLimit }, + ); + + const failures: SQSBatchItemFailure[] = []; + let deliveredCount = 0; + let dlqCount = 0; + + for (const { outcome, record } of results) { + switch (outcome) { + case "retry": { + failures.push({ itemIdentifier: record.messageId }); + break; + } + case "dlq": { + dlqCount += 1; + break; + } + case "success": { + deliveredCount += 1; + break; + } + default: { + const exhaustiveCheck: never = outcome; + throw new Error(`Unexpected outcome: ${exhaustiveCheck}`); + } + } + } + + return { deliveredCount, dlqCount, failures }; +} + +async function reportCircuitBreaker( + redis: RedisClientType, + targetId: string, + consumedTokens: number, + processingFailures: number, +): Promise { + const cbOutcome = await recordResult( + redis, + targetId, + consumedTokens, + processingFailures, + gateConfig, + ); + if (cbOutcome.circuitSwitched && cbOutcome.circuitState === "open") { + recordCircuitBreakerOpen(targetId); + } + if ( + cbOutcome.circuitSwitched && + cbOutcome.circuitState === "closed_recovery" + ) { + recordCircuitBreakerClosed(targetId); + } +} + +async function processTargetBatch( + batch: TargetBatch, + redis: RedisClientType, + clientId: string, + concurrencyLimit: number, +): Promise { + const target = await loadTargetConfig(clientId, batch.targetId); + const cbEnabled = + target.delivery?.circuitBreaker?.enabled ?? Boolean(target.delivery); + + const targetBurstCapacity = Math.min( + target.invocationRateLimit * BURST_MULTIPLIER, + MAX_BURST_CAPACITY, + ); + + const gateResult = await admit( + redis, + batch.targetId, + target.invocationRateLimit, + cbEnabled, + batch.records.length, + { ...gateConfig, burstCapacity: targetBurstCapacity }, + ); + + if (!gateResult.allowed) { + return handleBatchDenied( + batch, + clientId, + gateResult.reason, + gateResult.retryAfterMs, + ); + } + + const { consumedTokens } = gateResult; + const applicationId = await getApplicationId(clientId); + + const { deliveredCount, dlqCount, failures } = await deliverAdmittedRecords( + batch.records.slice(0, consumedTokens), + batch.messages.slice(0, consumedTokens), + target, + applicationId, + clientId, + concurrencyLimit, + ); + + if (cbEnabled && consumedTokens > 0) { + try { + await reportCircuitBreaker( + redis, + batch.targetId, + consumedTokens, + failures.length, + ); + } catch (error) { + logger.error("Failed to report circuit breaker result", { + targetId: batch.targetId, + err: error, + }); + } + } + + if (consumedTokens < batch.records.length) { + failures.push( + ...(await denyRecords( + batch.records.slice(consumedTokens), + batch.messages.slice(consumedTokens), + clientId, + batch.targetId, + "rate_limited", + (n) => n, + )), + ); + } + + return { deliveredCount, dlqCount, failures }; +} + +export async function processRecords( + records: SQSRecord[], +): Promise { + const { CLIENT_ID } = process.env; + if (!CLIENT_ID) { + logger.error("CLIENT_ID is required — sending all records to DLQ"); + await Promise.all(records.map((record) => sendToDlq(record.body))); + return []; + } + + resetMetrics(); + + const concurrencyLimit = Number( + process.env.CONCURRENCY_LIMIT ?? String(DEFAULT_CONCURRENCY_LIMIT), + ); + + logger.info("Batch received", { batchSize: records.length }); + + const redis = await getRedisClient(); + const unparseable: SQSRecord[] = []; + const targetBatches = groupByTarget(records, unparseable); + + for (const record of unparseable) { + logger.error("Unparseable message body — sending to DLQ", { + messageId: record.messageId, + }); + await sendToDlq(record.body); + } + + const allFailures: SQSBatchItemFailure[] = []; + let totalDelivered = 0; + let totalDlq = unparseable.length; + + for (const batch of targetBatches) { + try { + const batchResult = await processTargetBatch( + batch, + redis, + CLIENT_ID, + concurrencyLimit, + ); + allFailures.push(...batchResult.failures); + totalDelivered += batchResult.deliveredCount; + totalDlq += batchResult.dlqCount; + } catch (error) { + logger.error("Target batch failed — sending all records to DLQ", { + targetId: batch.targetId, + err: error, + }); + for (const record of batch.records) { + await sendToDlq(record.body); + } + totalDlq += batch.records.length; + } + } + + logger.info("Batch complete", { + batchSize: records.length, + deliveredCount: totalDelivered, + dlqCount: totalDlq, + retryCount: allFailures.length, + }); + + await flushMetrics(); + return allFailures; +} diff --git a/lambdas/https-client-lambda/src/index.ts b/lambdas/https-client-lambda/src/index.ts new file mode 100644 index 00000000..d53608ff --- /dev/null +++ b/lambdas/https-client-lambda/src/index.ts @@ -0,0 +1,7 @@ +import type { SQSBatchResponse, SQSEvent } from "aws-lambda"; +import { processRecords } from "handler"; + +export async function handler(event: SQSEvent): Promise { + const batchItemFailures = await processRecords(event.Records); + return { batchItemFailures }; +} diff --git a/lambdas/https-client-lambda/src/lua.d.ts b/lambdas/https-client-lambda/src/lua.d.ts new file mode 100644 index 00000000..8fe49f84 --- /dev/null +++ b/lambdas/https-client-lambda/src/lua.d.ts @@ -0,0 +1,4 @@ +declare module "*.lua" { + const content: string; + export default content; +} diff --git a/lambdas/https-client-lambda/src/services/admit.lua b/lambdas/https-client-lambda/src/services/admit.lua new file mode 100644 index 00000000..ad084b68 --- /dev/null +++ b/lambdas/https-client-lambda/src/services/admit.lua @@ -0,0 +1,133 @@ +-- admit.lua — Pre-processing: determines rate limit and consumes tokens. +-- +-- Two sequential steps run atomically: +-- 1. Circuit breaker — determine effective rate from circuit state +-- 2. Token bucket — consume tokens for the target batch +-- +-- The circuit has four states: +-- Open (during cooldown): rate = 0, complete block, bucket untouched +-- Half-open (after cooldown): rate = probeRateLimit +-- Recovering (closed, during recovery period): linear ramp-up +-- Normal (closed): full configured rate +-- +-- Returns: { consumedTokens, reason, retryAfterMs, effectiveRate } +-- +-- consumedTokens: how many tokens were consumed for this batch +-- +-- reason: the reason for admission/non-admission, one of: +-- "rate_limited" — no tokens available for batch, no admission +-- "circuit_open" — circuit is open, no admission +-- "some_allowed" — 1 or more tokens consumed, some admission +-- +-- retryAfterMs: for any not admitted, how long to wait before retrying +-- +-- effectiveRate: the effective rate (tokens/s) applied to this batch + +-- Reason constants +local RATE_LIMITED = "rate_limited" +local CIRCUIT_OPEN = "circuit_open" +local SOME_ALLOWED = "some_allowed" + +-- Keys +local epKey = KEYS[1] -- ep:{targetId} combined endpoint state hash + +-- Arguments +local now = tonumber(ARGV[1]) or 0 +local capacity = tonumber(ARGV[2]) or 0 +local targetRateLimit = tonumber(ARGV[3]) or 0 +local cooldownMs = tonumber(ARGV[4]) or 0 +local recoveryPeriodMs = tonumber(ARGV[5]) or 0 +local probeRateLimit = tonumber(ARGV[6]) or 0 +local targetBatchSize = tonumber(ARGV[7]) or 0 +local cbEnabled = tonumber(ARGV[8]) == 1 + +-------------------------------------------------------------------------------- +-- LOAD STATE +-------------------------------------------------------------------------------- + +local state = redis.call("HMGET", epKey, + "is_open", "switched_at", "bucket_tokens", "bucket_refilled_at") +local cbNeedInit = state[1] == false or state[1] == nil +local rlNeedInit = state[4] == false or state[4] == nil +local isOpen = cbNeedInit or tonumber(state[1]) == 1 +local switchedAt = cbNeedInit and 0 or tonumber(state[2] or "0") +local bucketTokens = tonumber(state[3] or "0") +local bucketRefilledAt = rlNeedInit and now or tonumber(state[4]) + +if not cbEnabled then + isOpen = false + switchedAt = 0 +end + +-------------------------------------------------------------------------------- +-- 1. CIRCUIT BREAKER — determine effective rate +-------------------------------------------------------------------------------- + +local isHalfOpen = isOpen and now > switchedAt + cooldownMs +local isRecovering = (not isOpen) and now < switchedAt + recoveryPeriodMs + +local effectiveRate + +if isOpen then + if isHalfOpen then + effectiveRate = probeRateLimit + else + return { 0, CIRCUIT_OPEN, (switchedAt + cooldownMs) - now, 0 } + end +else + if isRecovering then + local rampRange = math.max(0, targetRateLimit - probeRateLimit) + local rampProgress = math.max(0, now - switchedAt) / recoveryPeriodMs + effectiveRate = probeRateLimit + rampProgress * rampRange + else + effectiveRate = targetRateLimit + end +end + +-------------------------------------------------------------------------------- +-- 2. TOKEN BUCKET — batch consumption +-- +-- Generate tokens based on elapsed time, then consume as many as needed for +-- the batch, up to the number available. +-- +-- bucketRefilledAt tracks the point in time up to which tokens have been +-- generated. We advance it by exactly the time needed to produce the whole +-- tokens we generated (generationTime), rather than setting it to `now`. +-- +-- Why not `now`? Token generation uses floor(), so any sub-token fractional +-- time is truncated. Setting bucketRefilledAt = now would discard that +-- remainder, meaning the next call starts its elapsed-time calculation from +-- a later point than it should. Over many calls this causes token leakage — +-- the bucket refills slower than the configured rate. By advancing only by +-- generationTime, the leftover fractional time carries over to the next call. +-------------------------------------------------------------------------------- + +if isOpen then + bucketTokens = 0 + if rlNeedInit and isHalfOpen then + bucketTokens = 1 + end +end + +local generatedTokens = math.floor((now - bucketRefilledAt) * effectiveRate / 1000) +local availTokens = math.min(capacity, bucketTokens + generatedTokens) +local consumedTokens = math.min(targetBatchSize, availTokens) + +bucketTokens = availTokens - consumedTokens +if generatedTokens > 0 and effectiveRate > 0 then + local generationTime = generatedTokens * 1000 / effectiveRate + bucketRefilledAt = bucketRefilledAt + generationTime +end + +-------------------------------------------------------------------------------- +-- 3. PERSIST STATE AND RETURN +-------------------------------------------------------------------------------- + +redis.call("HSET", epKey, + "bucket_tokens", bucketTokens, + "bucket_refilled_at", bucketRefilledAt +) + +local reason = consumedTokens < 1 and RATE_LIMITED or SOME_ALLOWED +local retryAfter = consumedTokens < 1 and 1000 or 0 +return { consumedTokens, reason, retryAfter, effectiveRate } diff --git a/lambdas/https-client-lambda/src/services/applications-map.ts b/lambdas/https-client-lambda/src/services/applications-map.ts new file mode 100644 index 00000000..9a86d86c --- /dev/null +++ b/lambdas/https-client-lambda/src/services/applications-map.ts @@ -0,0 +1,73 @@ +import { GetObjectCommand, S3Client } from "@aws-sdk/client-s3"; +import { logger } from "@nhs-notify-client-callbacks/logger"; + +const s3Client = new S3Client({}); + +const DEFAULT_CACHE_TTL_MS = 300_000; // 5 minutes + +let cachedMap: Map | undefined; +let cacheExpiresAt = 0; + +async function loadMap(): Promise> { + if (cachedMap && Date.now() < cacheExpiresAt) { + return cachedMap; + } + + const { APPLICATIONS_MAP_S3_BUCKET, APPLICATIONS_MAP_S3_KEY } = process.env; + if (!APPLICATIONS_MAP_S3_BUCKET || !APPLICATIONS_MAP_S3_KEY) { + throw new Error( + "APPLICATIONS_MAP_S3_BUCKET and APPLICATIONS_MAP_S3_KEY are required", + ); + } + + const response = await s3Client.send( + new GetObjectCommand({ + Bucket: APPLICATIONS_MAP_S3_BUCKET, + Key: APPLICATIONS_MAP_S3_KEY, + }), + ); + + const body = await response.Body?.transformToString(); + if (!body) { + throw new Error( + `S3 object 's3://${APPLICATIONS_MAP_S3_BUCKET}/${APPLICATIONS_MAP_S3_KEY}' is empty`, + ); + } + + let parsed: Record; + try { + parsed = JSON.parse(body) as Record; + } catch { + throw new Error( + `S3 object 's3://${APPLICATIONS_MAP_S3_BUCKET}/${APPLICATIONS_MAP_S3_KEY}' contains invalid JSON`, + ); + } + + cachedMap = new Map(Object.entries(parsed)); + const ttlMs = + Number(process.env.APPLICATIONS_MAP_CACHE_TTL_MS) || DEFAULT_CACHE_TTL_MS; + cacheExpiresAt = Date.now() + ttlMs; + logger.info("Applications map loaded from S3", { + bucket: APPLICATIONS_MAP_S3_BUCKET, + key: APPLICATIONS_MAP_S3_KEY, + }); + return cachedMap; +} + +export async function getApplicationId(clientId: string): Promise { + const map = await loadMap(); + const applicationId = map.get(clientId); + + if (!applicationId) { + throw new Error( + `No applicationId found for clientId '${clientId}' in applications map`, + ); + } + + return applicationId; +} + +export function resetCache(): void { + cachedMap = undefined; + cacheExpiresAt = 0; +} diff --git a/lambdas/https-client-lambda/src/services/config-loader.ts b/lambdas/https-client-lambda/src/services/config-loader.ts new file mode 100644 index 00000000..7f5b7bdc --- /dev/null +++ b/lambdas/https-client-lambda/src/services/config-loader.ts @@ -0,0 +1,54 @@ +import { S3Client } from "@aws-sdk/client-s3"; +import type { CallbackTarget } from "@nhs-notify-client-callbacks/models"; +import { ConfigSubscriptionCache } from "@nhs-notify-client-callbacks/config-subscription-cache"; + +const s3Client = new S3Client({}); +let cache: ConfigSubscriptionCache | undefined; + +function getCache(): ConfigSubscriptionCache { + if (!cache) { + const { + CLIENT_SUBSCRIPTION_CONFIG_BUCKET, + CLIENT_SUBSCRIPTION_CONFIG_PREFIX, + } = process.env; + if (!CLIENT_SUBSCRIPTION_CONFIG_BUCKET) { + throw new Error("CLIENT_SUBSCRIPTION_CONFIG_BUCKET is required"); + } + + const ttlMs = + (Number(process.env.CLIENT_SUBSCRIPTION_CACHE_TTL_SECONDS) || 300) * 1000; + + cache = new ConfigSubscriptionCache({ + s3Client, + bucketName: CLIENT_SUBSCRIPTION_CONFIG_BUCKET, + keyPrefix: CLIENT_SUBSCRIPTION_CONFIG_PREFIX ?? "client_subscriptions/", + ttlMs, + }); + } + return cache; +} + +export function resetCache(): void { + cache = undefined; +} + +export async function loadTargetConfig( + clientId: string, + targetId: string, +): Promise { + const clientConfig = await getCache().loadClientConfig(clientId); + + if (!clientConfig) { + throw new Error(`No configuration found for client '${clientId}'`); + } + + const target = clientConfig.targets.find((t) => t.targetId === targetId); + + if (!target) { + throw new Error( + `Target '${targetId}' not found in config for client '${clientId}'`, + ); + } + + return target; +} diff --git a/lambdas/https-client-lambda/src/services/delivery-metrics.ts b/lambdas/https-client-lambda/src/services/delivery-metrics.ts new file mode 100644 index 00000000..7b38dbb3 --- /dev/null +++ b/lambdas/https-client-lambda/src/services/delivery-metrics.ts @@ -0,0 +1,156 @@ +import { + StorageResolution, + Unit, + createMetricsLogger, +} from "aws-embedded-metrics"; +import type { MetricsLogger } from "aws-embedded-metrics"; + +let metricsInstance: MetricsLogger | undefined; + +function getMetrics(): MetricsLogger { + if (metricsInstance) { + return metricsInstance; + } + + const namespace = process.env.METRICS_NAMESPACE; + const environment = process.env.ENVIRONMENT; + const clientId = process.env.CLIENT_ID; + + if (!namespace) { + throw new Error("METRICS_NAMESPACE environment variable is not set"); + } + if (!environment) { + throw new Error("ENVIRONMENT environment variable is not set"); + } + if (!clientId) { + throw new Error("CLIENT_ID environment variable is not set"); + } + + metricsInstance = createMetricsLogger(); + metricsInstance.setNamespace(namespace); + metricsInstance.setDimensions({ + Environment: environment, + ClientId: clientId, + }); + + return metricsInstance; +} + +export function emitDeliveryAttempt(targetId: string): void { + const metrics = getMetrics(); + metrics.setProperty("targetId", targetId); + metrics.putMetric("DeliveryAttempt", 1, Unit.Count, StorageResolution.High); +} + +export function emitDeliverySuccess(targetId: string): void { + const metrics = getMetrics(); + metrics.setProperty("targetId", targetId); + metrics.putMetric("DeliverySuccess", 1, Unit.Count, StorageResolution.High); +} + +export function emitDeliveryFailure(targetId: string): void { + const metrics = getMetrics(); + metrics.setProperty("targetId", targetId); + metrics.putMetric("DeliveryFailure", 1, Unit.Count, StorageResolution.High); +} + +export function emitDeliveryPermanentFailure(targetId: string): void { + const metrics = getMetrics(); + metrics.setProperty("targetId", targetId); + metrics.putMetric( + "DeliveryPermanentFailure", + 1, + Unit.Count, + StorageResolution.Standard, + ); +} + +export function emitServerRateLimited(targetId: string): void { + const metrics = getMetrics(); + metrics.setProperty("targetId", targetId); + metrics.putMetric( + "DeliveryServerRateLimited", + 1, + Unit.Count, + StorageResolution.High, + ); +} + +export function emitCircuitBreakerOpen(targetId: string): void { + const metrics = getMetrics(); + metrics.setProperty("targetId", targetId); + metrics.putMetric( + "CircuitBreakerOpen", + 1, + Unit.Count, + StorageResolution.Standard, + ); +} + +export function emitCircuitBreakerClosed(targetId: string): void { + const metrics = getMetrics(); + metrics.setProperty("targetId", targetId); + metrics.putMetric( + "CircuitBreakerClosed", + 1, + Unit.Count, + StorageResolution.Standard, + ); +} + +export function emitRetryWindowExhausted(targetId: string): void { + const metrics = getMetrics(); + metrics.setProperty("targetId", targetId); + metrics.putMetric( + "DeliveryRetryWindowExhausted", + 1, + Unit.Count, + StorageResolution.Standard, + ); +} + +export function emitClientRateLimited(targetId: string, count = 1): void { + const metrics = getMetrics(); + metrics.setProperty("targetId", targetId); + metrics.putMetric( + "DeliveryRateLimited", + count, + Unit.Count, + StorageResolution.High, + ); +} + +export function emitCircuitBlocked(targetId: string, count = 1): void { + const metrics = getMetrics(); + metrics.setProperty("targetId", targetId); + metrics.putMetric( + "DeliveryCircuitBlocked", + count, + Unit.Count, + StorageResolution.High, + ); +} + +export function emitDeliveryDuration( + targetId: string, + durationMs: number, +): void { + const metrics = getMetrics(); + metrics.setProperty("targetId", targetId); + metrics.putMetric( + "DeliveryDurationMs", + durationMs, + Unit.Milliseconds, + StorageResolution.High, + ); +} + +export async function flushMetrics(): Promise { + if (metricsInstance) { + await metricsInstance.flush(); + } +} + +export function resetMetrics(): void { + metricsInstance = undefined; +} diff --git a/lambdas/https-client-lambda/src/services/delivery-observability.ts b/lambdas/https-client-lambda/src/services/delivery-observability.ts new file mode 100644 index 00000000..289ed3a9 --- /dev/null +++ b/lambdas/https-client-lambda/src/services/delivery-observability.ts @@ -0,0 +1,150 @@ +import { logger } from "@nhs-notify-client-callbacks/logger"; +import { + emitCircuitBlocked, + emitCircuitBreakerClosed, + emitCircuitBreakerOpen, + emitClientRateLimited, + emitDeliveryAttempt, + emitDeliveryDuration, + emitDeliveryFailure, + emitDeliveryPermanentFailure, + emitDeliverySuccess, + emitRetryWindowExhausted, + emitServerRateLimited, +} from "services/delivery-metrics"; + +export function recordDeliveryAttempt( + clientId: string, + targetId: string, + correlationId?: string, + sqsMessageId?: string, + receiveCount?: number, +): void { + emitDeliveryAttempt(targetId); + logger.info("Attempting delivery", { + clientId, + targetId, + correlationId, + sqsMessageId, + receiveCount, + }); +} + +export function recordDeliverySuccess( + clientId: string, + targetId: string, + correlationId?: string, +): void { + emitDeliverySuccess(targetId); + logger.info("Delivery succeeded", { clientId, targetId, correlationId }); +} + +export function recordDeliveryPermanentFailure( + clientId: string, + targetId: string, + statusCode?: number, + errorCode?: string, + correlationId?: string, +): void { + emitDeliveryPermanentFailure(targetId); + logger.warn("Permanent delivery failure — sending to DLQ", { + clientId, + targetId, + correlationId, + ...(statusCode !== undefined && { statusCode }), + ...(errorCode !== undefined && { errorCode }), + }); +} + +export function recordDeliveryRateLimited( + clientId: string, + targetId: string, + correlationId?: string, +): void { + emitServerRateLimited(targetId); + logger.info("Server rate limited (429)", { + clientId, + targetId, + correlationId, + }); +} + +export function recordDeliveryFailure( + clientId: string, + targetId: string, + statusCode: number, + backoffSec: number, + receiveCount: number, + correlationId?: string, +): void { + emitDeliveryFailure(targetId); + logger.warn("Transient delivery failure — requeuing", { + clientId, + targetId, + correlationId, + statusCode, + backoffSec, + receiveCount, + }); +} + +export function recordCircuitBreakerOpen( + targetId: string, + correlationId?: string, +): void { + emitCircuitBreakerOpen(targetId); + logger.warn("Circuit breaker opened", { targetId, correlationId }); +} + +export function recordCircuitBreakerClosed( + targetId: string, + correlationId?: string, +): void { + emitCircuitBreakerClosed(targetId); + logger.info("Circuit breaker closed", { targetId, correlationId }); +} + +export function recordRetryWindowExhausted( + clientId: string, + targetId: string, + correlationId?: string, +): void { + emitRetryWindowExhausted(targetId); + logger.warn("Retry window exhausted — sending to DLQ", { + clientId, + targetId, + correlationId, + }); +} + +export function recordAdmissionDenied( + clientId: string, + targetId: string, + reason: string, + correlationIds: (string | undefined)[], +): void { + if (reason === "circuit_open") { + emitCircuitBlocked(targetId, correlationIds.length); + logger.warn("Circuit blocked", { + clientId, + targetId, + deniedCount: correlationIds.length, + correlationIds, + }); + } else { + emitClientRateLimited(targetId, correlationIds.length); + logger.warn("Client rate limited", { + clientId, + targetId, + deniedCount: correlationIds.length, + correlationIds, + }); + } +} + +export function recordDeliveryDuration( + targetId: string, + durationMs: number, +): void { + emitDeliveryDuration(targetId, durationMs); +} diff --git a/lambdas/https-client-lambda/src/services/delivery/https-client.ts b/lambdas/https-client-lambda/src/services/delivery/https-client.ts new file mode 100644 index 00000000..4a04529d --- /dev/null +++ b/lambdas/https-client-lambda/src/services/delivery/https-client.ts @@ -0,0 +1,111 @@ +import https from "node:https"; +import type { Agent } from "node:https"; +import type { CallbackTarget } from "@nhs-notify-client-callbacks/models"; +import { PERMANENT_TLS_ERROR_CODES } from "services/delivery/tls-agent-factory"; + +export const OUTCOME_SUCCESS = "success" as const; +export const OUTCOME_PERMANENT_FAILURE = "permanent_failure" as const; +export const OUTCOME_RATE_LIMITED = "rate_limited" as const; +export const OUTCOME_TRANSIENT_FAILURE = "transient_failure" as const; + +const RETRYABLE_4XX = new Set([401, 407, 409]); + +export type DeliveryResult = + | { outcome: typeof OUTCOME_SUCCESS } + | { + outcome: typeof OUTCOME_PERMANENT_FAILURE; + statusCode?: number; + errorCode?: string; + responseBody?: string; + } + | { + outcome: typeof OUTCOME_RATE_LIMITED; + statusCode: 429; + retryAfterHeader: string | undefined; + } + | { outcome: typeof OUTCOME_TRANSIENT_FAILURE; statusCode: number }; + +export function deliverPayload( + target: CallbackTarget, + signedPayloadJson: string, + signatureHeader: string, + agent: Agent, +): Promise { + const requestTimeoutMs = Number(process.env.REQUEST_TIMEOUT_MS ?? "30000"); + + return new Promise((resolve) => { + const url = new URL(target.invocationEndpoint); + + const req = https.request( + url, + { + method: target.invocationMethod, + agent, + timeout: requestTimeoutMs, + headers: { + "Content-Type": "application/json", + "x-hmac-sha256-signature": signatureHeader, + [target.apiKey.headerName]: target.apiKey.headerValue, + }, + }, + (res) => { + const statusCode = res.statusCode ?? 0; + + if (statusCode >= 200 && statusCode < 300) { + res.resume(); + resolve({ outcome: OUTCOME_SUCCESS }); + return; + } + + if (statusCode === 429) { + res.resume(); + const retryAfterHeader = res.headers["retry-after"]; + resolve({ + outcome: OUTCOME_RATE_LIMITED, + statusCode, + retryAfterHeader, + }); + return; + } + + if (statusCode >= 400 && statusCode < 500) { + if (RETRYABLE_4XX.has(statusCode)) { + res.resume(); + resolve({ outcome: OUTCOME_TRANSIENT_FAILURE, statusCode }); + return; + } + + const chunks: Buffer[] = []; + res.on("data", (chunk: Buffer) => chunks.push(chunk)); + res.on("end", () => { + const responseBody = Buffer.concat(chunks).toString("utf8"); + resolve({ + outcome: OUTCOME_PERMANENT_FAILURE, + statusCode, + responseBody, + }); + }); + return; + } + + res.resume(); + resolve({ outcome: OUTCOME_TRANSIENT_FAILURE, statusCode }); + }, + ); + + req.on("timeout", () => { + req.destroy(new Error("Request timed out")); + }); + + req.on("error", (error: NodeJS.ErrnoException) => { + if (error.code && PERMANENT_TLS_ERROR_CODES.has(error.code)) { + resolve({ outcome: OUTCOME_PERMANENT_FAILURE, errorCode: error.code }); + return; + } + + resolve({ outcome: OUTCOME_TRANSIENT_FAILURE, statusCode: 0 }); + }); + + req.end(signedPayloadJson); + }); +} diff --git a/lambdas/https-client-lambda/src/services/delivery/retry-policy.ts b/lambdas/https-client-lambda/src/services/delivery/retry-policy.ts new file mode 100644 index 00000000..8d1ec2fd --- /dev/null +++ b/lambdas/https-client-lambda/src/services/delivery/retry-policy.ts @@ -0,0 +1,89 @@ +import type { SQSRecord } from "aws-lambda"; +import { logger } from "@nhs-notify-client-callbacks/logger"; +import { sendToDlq } from "services/dlq-sender"; +import { changeVisibility } from "services/sqs-visibility"; + +const BACKOFF_CAP_SECONDS = 300; +const SQS_MAX_VISIBILITY_SECONDS = 43_200; +const BASE_BACKOFF_MULTIPLIER = 5; +const BACKOFF_EXPONENT_BASE = 2; + +export function jitteredBackoffSeconds(receiveCount: number): number { + const ceiling = Math.min( + BASE_BACKOFF_MULTIPLIER * BACKOFF_EXPONENT_BASE ** (receiveCount - 1), + BACKOFF_CAP_SECONDS, + ); + // eslint-disable-next-line sonarjs/pseudo-random -- jitter for backoff, not security-sensitive + return Math.max(1, Math.floor(Math.random() * ceiling)); +} + +export function parseRetryAfter(header: string): number { + const asInt = Number(header); + + if (!Number.isNaN(asInt) && Number.isFinite(asInt)) { + return Math.floor(asInt); + } + + const date = new Date(header); + if (Number.isNaN(date.getTime())) { + return 0; + } + + return Math.max(0, Math.floor((date.getTime() - Date.now()) / 1000)); +} + +export function isWindowExhausted( + firstReceivedMs: number, + maxRetryDurationMs: number, +): boolean { + return Date.now() - firstReceivedMs >= maxRetryDurationMs; +} + +export function exceedsSqsMaxVisibility(retryAfterSeconds: number): boolean { + return retryAfterSeconds > SQS_MAX_VISIBILITY_SECONDS; +} + +export async function handleRateLimitedRecord( + record: SQSRecord, + clientId: string, + targetId: string, + retryAfterHeader: string | undefined, + receiveCount: number, +): Promise<"retry" | "dlq"> { + const retryAfterSeconds = retryAfterHeader + ? parseRetryAfter(retryAfterHeader) + : 0; + + if (retryAfterSeconds < 0) { + logger.warn("429 Retry-After is negative — sending to DLQ", { + clientId, + targetId, + retryAfterSeconds, + }); + await sendToDlq(record.body); + return "dlq"; + } + + if (exceedsSqsMaxVisibility(retryAfterSeconds)) { + logger.warn("429 Retry-After exceeds SQS max — sending to DLQ", { + clientId, + targetId, + retryAfterSeconds, + }); + await sendToDlq(record.body); + return "dlq"; + } + + const delaySec = + retryAfterSeconds > 0 + ? retryAfterSeconds + : jitteredBackoffSeconds(receiveCount); + + logger.warn("Rate limited (429) — requeuing", { + clientId, + targetId, + delaySec, + }); + await changeVisibility(record.receiptHandle, delaySec); + return "retry"; +} diff --git a/lambdas/https-client-lambda/src/services/delivery/tls-agent-factory.ts b/lambdas/https-client-lambda/src/services/delivery/tls-agent-factory.ts new file mode 100644 index 00000000..bb1e7334 --- /dev/null +++ b/lambdas/https-client-lambda/src/services/delivery/tls-agent-factory.ts @@ -0,0 +1,170 @@ +import { Agent } from "node:https"; +import { X509Certificate, createHash } from "node:crypto"; +import { checkServerIdentity } from "node:tls"; +import type { PeerCertificate } from "node:tls"; +import forge from "node-forge"; +import { GetObjectCommand, S3Client } from "@aws-sdk/client-s3"; +import type { CallbackTarget } from "@nhs-notify-client-callbacks/models"; +import { logger } from "@nhs-notify-client-callbacks/logger"; + +const { MTLS_CA_S3_KEY, MTLS_CERT_S3_BUCKET, MTLS_CERT_S3_KEY } = process.env; +const CERT_EXPIRY_THRESHOLD_MS = + Number(process.env.CERT_EXPIRY_THRESHOLD_MS) || 86_400_000; // 24 hours + +const s3Client = new S3Client({}); + +export const PERMANENT_TLS_ERROR_CODES = new Set([ + "CERT_HAS_EXPIRED", + "DEPTH_ZERO_SELF_SIGNED_CERT", + "ERR_CERT_PINNING_FAILED", + "ERR_TLS_CERT_ALTNAME_INVALID", + "SELF_SIGNED_CERT_IN_CHAIN", + "UNABLE_TO_VERIFY_LEAF_SIGNATURE", +]); + +type CertMaterial = { + key: string; + cert: string; + ca?: string; + validTo: Date; +}; + +let cachedMaterial: CertMaterial | undefined; + +async function loadS3Object(bucket: string, key: string): Promise { + const response = await s3Client.send( + new GetObjectCommand({ Bucket: bucket, Key: key }), + ); + + if (!response.Body) { + throw new Error(`S3 object s3://${bucket}/${key} has no body`); + } + + return response.Body.transformToString(); +} + +async function loadFromS3(): Promise<{ + key: string; + cert: string; + ca?: string; +}> { + if (!MTLS_CERT_S3_BUCKET || !MTLS_CERT_S3_KEY) { + throw new Error("MTLS_CERT_S3_BUCKET and MTLS_CERT_S3_KEY are required"); + } + + const pem = await loadS3Object(MTLS_CERT_S3_BUCKET, MTLS_CERT_S3_KEY); + + const pemObjects = forge.pem.decode(pem); + const keyObj = pemObjects.find((obj) => obj.type.includes("PRIVATE KEY")); + const certObj = pemObjects.find((obj) => obj.type.includes("CERTIFICATE")); + const key = keyObj ? forge.pem.encode(keyObj) : ""; + const cert = certObj ? forge.pem.encode(certObj) : ""; + + let ca: string | undefined; + if (MTLS_CA_S3_KEY) { + ca = await loadS3Object(MTLS_CERT_S3_BUCKET, MTLS_CA_S3_KEY); + } + + return { key, cert, ca }; +} + +async function loadCertMaterial(): Promise { + const raw = await loadFromS3(); + + const x509 = new X509Certificate(raw.cert); + const validTo = new Date(x509.validTo); + + logger.info("mTLS certificate loaded", { + source: "S3", + validTo: validTo.toISOString(), + }); + + return { + key: raw.key, + cert: raw.cert, + ca: raw.ca, + validTo, + }; +} + +function isExpiringSoon(material: CertMaterial): boolean { + return material.validTo.getTime() - Date.now() < CERT_EXPIRY_THRESHOLD_MS; +} + +async function getMaterial(): Promise { + if (cachedMaterial && !isExpiringSoon(cachedMaterial)) { + return cachedMaterial; + } + + cachedMaterial = await loadCertMaterial(); + return cachedMaterial; +} + +export async function buildAgent(target: CallbackTarget): Promise { + const agentOptions: Record = { + keepAlive: false, + }; + + const certPinning = target.delivery?.mtls?.certPinning; + + if (certPinning?.enabled && !certPinning.spkiHash) { + throw new Error( + `certPinning.spkiHash is required when certPinning is enabled for target '${target.targetId}'`, + ); + } + + // Load CA from S3 when configured so targets with mtls.enabled: false can + // still verify the server's cert chain. + if (target.delivery?.mtls?.enabled || MTLS_CA_S3_KEY) { + const material = await getMaterial(); + + if (material.ca) { + agentOptions.ca = material.ca; + } + + if (target.delivery?.mtls?.enabled) { + agentOptions.key = material.key; + agentOptions.cert = material.cert; + } + } + + if (certPinning?.enabled) { + const expectedHash = certPinning.spkiHash!; + + /* eslint-disable sonarjs/function-return-type -- checkServerIdentity requires Error|undefined return */ + agentOptions.checkServerIdentity = ( + hostname: string, + peerCert: PeerCertificate, + ) => { + const defaultResult = checkServerIdentity(hostname, peerCert); + if (defaultResult) { + return defaultResult; + } + + const rawDer = peerCert.raw; + const x509 = new X509Certificate(rawDer); + const spkiDer = x509.publicKey.export({ + type: "spki", + format: "der", + }) as Buffer; + const actualHash = createHash("sha256").update(spkiDer).digest("base64"); + + if (actualHash !== expectedHash) { + const error = new Error( + `Certificate pinning failed: expected SPKI hash '${expectedHash}', got '${actualHash}'`, + ); + (error as NodeJS.ErrnoException).code = "ERR_CERT_PINNING_FAILED"; + return error; + } + + return undefined; + }; + /* eslint-enable sonarjs/function-return-type */ + } + + return new Agent(agentOptions as ConstructorParameters[0]); +} + +export function resetCache(): void { + cachedMaterial = undefined; +} diff --git a/lambdas/https-client-lambda/src/services/dlq-sender.ts b/lambdas/https-client-lambda/src/services/dlq-sender.ts new file mode 100644 index 00000000..56b92405 --- /dev/null +++ b/lambdas/https-client-lambda/src/services/dlq-sender.ts @@ -0,0 +1,70 @@ +import { + type MessageAttributeValue, + SQSClient, + SendMessageCommand, +} from "@aws-sdk/client-sqs"; + +const sqsClient = new SQSClient({}); + +export type DlqErrorInfo = { + statusCode?: number; + errorCode?: string; + responseBody?: string; +}; + +function buildDlqAttributes( + errorInfo: DlqErrorInfo, +): Record { + const attrs: Record = {}; + + if (errorInfo.errorCode) { + attrs.ERROR_CODE = { + DataType: "String", + StringValue: errorInfo.errorCode, + }; + } else if (errorInfo.statusCode !== undefined) { + attrs.ERROR_CODE = { + DataType: "String", + StringValue: "HTTP_CLIENT_ERROR", + }; + } + + if (errorInfo.responseBody) { + let errorMessage = errorInfo.responseBody; + try { + const parsed = JSON.parse(errorInfo.responseBody) as { + message?: string; + }; + if (parsed.message) { + errorMessage = parsed.message; + } + } catch { + // use raw body if not valid JSON + } + attrs.ERROR_MESSAGE = { DataType: "String", StringValue: errorMessage }; + } + + return attrs; +} + +export async function sendToDlq( + messageBody: string, + errorInfo?: DlqErrorInfo, +): Promise { + const { DLQ_URL } = process.env; + if (!DLQ_URL) { + throw new Error("DLQ_URL is required"); + } + + const messageAttributes = errorInfo + ? buildDlqAttributes(errorInfo) + : undefined; + + await sqsClient.send( + new SendMessageCommand({ + QueueUrl: DLQ_URL, + MessageBody: messageBody, + ...(messageAttributes && { MessageAttributes: messageAttributes }), + }), + ); +} diff --git a/lambdas/https-client-lambda/src/services/endpoint-gate.ts b/lambdas/https-client-lambda/src/services/endpoint-gate.ts new file mode 100644 index 00000000..d5bebb17 --- /dev/null +++ b/lambdas/https-client-lambda/src/services/endpoint-gate.ts @@ -0,0 +1,172 @@ +import type { RedisClientType } from "services/redis-client"; +import { createHash } from "node:crypto"; +import admitLuaSrc from "services/admit.lua"; +import recordResultLuaSrc from "services/record-result.lua"; + +export type AdmitResultAllowed = { + allowed: true; + consumedTokens: number; + effectiveRate: number; +}; + +export type AdmitResultDenied = { + allowed: false; + reason: "circuit_open" | "rate_limited"; + retryAfterMs: number; + effectiveRate: number; +}; + +export type AdmitResult = AdmitResultAllowed | AdmitResultDenied; + +export type CircuitState = "open" | "open_half" | "closed_recovery" | "closed"; + +export type RecordResultOutcome = { + circuitState: CircuitState; + circuitSwitched: boolean; +}; + +export type EndpointGateConfig = { + burstCapacity: number; + probeRateLimit: number; + recoveryPeriodMs: number; + samplePeriodMs: number; + failureThreshold: number; + minAttempts: number; + cooldownPeriodMs: number; +}; + +let admitSha: string | undefined; +let recordResultSha: string | undefined; + +function computeSha1(script: string): string { + // eslint-disable-next-line sonarjs/hashing -- SHA-1 required by Redis EVALSHA protocol, not a security context + return createHash("sha1").update(script).digest("hex"); +} + +async function evalScript( + client: RedisClientType, + script: string, + sha: string, + keys: string[], + args: string[], +): Promise { + const keyCount = keys.length.toString(); + try { + return await client.sendCommand([ + "EVALSHA", + sha, + keyCount, + ...keys, + ...args, + ]); + } catch (error: unknown) { + const isNoScript = + error instanceof Error && error.message.includes("NOSCRIPT"); + if (!isNoScript) { + const message = + error instanceof Error ? error.message : JSON.stringify(error); + throw new Error(`Redis error in script ${script}: ${message}`, { + cause: error, + }); + } + return client.sendCommand(["EVAL", script, keyCount, ...keys, ...args]); + } +} + +export async function admit( + client: RedisClientType, + targetId: string, + refillPerSec: number, + cbEnabled: boolean, + targetBatchSize: number, + config: EndpointGateConfig, +): Promise { + const epKey = `ep:{${targetId}}`; + const now = Date.now().toString(); + + const args = [ + now, + config.burstCapacity.toString(), + String(refillPerSec), + config.cooldownPeriodMs.toString(), + config.recoveryPeriodMs.toString(), + config.probeRateLimit.toString(), + String(targetBatchSize), + cbEnabled ? "1" : "0", + ]; + + if (!admitSha) { + admitSha = computeSha1(admitLuaSrc); + } + + const raw = (await evalScript( + client, + admitLuaSrc, + admitSha, + [epKey], + args, + )) as [number, string, number, number]; + + const [consumedOrFlag, reason, retryAfterMs, effectiveRate] = raw; + + if (reason === "some_allowed") { + return { + allowed: true, + consumedTokens: Number(consumedOrFlag), + effectiveRate: Number(effectiveRate), + }; + } + + return { + allowed: false, + reason: reason as "circuit_open" | "rate_limited", + retryAfterMs: Number(retryAfterMs), + effectiveRate: Number(effectiveRate), + }; +} + +export async function recordResult( + client: RedisClientType, + targetId: string, + consumedTokens: number, + processingFailures: number, + config: EndpointGateConfig, +): Promise { + const epKey = `ep:{${targetId}}`; + const now = Date.now().toString(); + + const args = [ + now, + String(consumedTokens), + String(processingFailures), + config.cooldownPeriodMs.toString(), + config.recoveryPeriodMs.toString(), + config.failureThreshold.toString(), + config.minAttempts.toString(), + config.samplePeriodMs.toString(), + ]; + + if (!recordResultSha) { + recordResultSha = computeSha1(recordResultLuaSrc); + } + + const raw = (await evalScript( + client, + recordResultLuaSrc, + recordResultSha, + [epKey], + args, + )) as [string, number]; + + const [circuitState, circuitSwitched] = raw; + + return { + circuitState: circuitState as CircuitState, + circuitSwitched: circuitSwitched === 1, + }; +} + +export function resetAdmitSha(): void { + admitSha = undefined; + recordResultSha = undefined; +} diff --git a/lambdas/client-transform-filter-lambda/src/services/payload-signer.ts b/lambdas/https-client-lambda/src/services/payload-signer.ts similarity index 100% rename from lambdas/client-transform-filter-lambda/src/services/payload-signer.ts rename to lambdas/https-client-lambda/src/services/payload-signer.ts index cf69cac8..e2174b76 100644 --- a/lambdas/client-transform-filter-lambda/src/services/payload-signer.ts +++ b/lambdas/https-client-lambda/src/services/payload-signer.ts @@ -2,9 +2,9 @@ import { createHmac } from "node:crypto"; import type { ClientCallbackPayload } from "@nhs-notify-client-callbacks/models"; export function signPayload( - payload: ClientCallbackPayload, applicationId: string, apiKey: string, + payload: ClientCallbackPayload, ): string { return createHmac("sha256", `${applicationId}.${apiKey}`) .update(JSON.stringify(payload)) diff --git a/lambdas/https-client-lambda/src/services/record-result.lua b/lambdas/https-client-lambda/src/services/record-result.lua new file mode 100644 index 00000000..1ebb0d36 --- /dev/null +++ b/lambdas/https-client-lambda/src/services/record-result.lua @@ -0,0 +1,168 @@ +-- record-result.lua — Post-processing: updates sampling and circuit breaker. +-- +-- After processing a batch, this script: +-- 1. Manages the sliding window (rolling forward as necessary) +-- 2. Records new attempts and failures (unless fully open) +-- 3. Interpolates attempt/failure rates using the sliding window +-- 4. Checks whether to close the circuit (half-open + successes) +-- 5. Checks whether to open the circuit (closed + threshold exceeded) +-- +-- Returns: { circuitState, curcuitSwitched } +-- +-- circuitState: the current state of the circuit after this run +-- "open" — fully open (during cooldown, no probes) +-- "open_half" — open but past cooldown (probing) +-- "closed_recovery" — closed but ramping up (recovery period) +-- "closed" — closed, running at full rate +-- +-- curcuitSwitched: whether the circuit opened or closed during this run +-- 1 — the circuit opened or closed during this execution +-- 0 — no state transition + +-- Circuit state constants +local OPEN = "open" +local OPEN_HALF = "open_half" +local CLOSED_RECOVERY = "closed_recovery" +local CLOSED = "closed" + +-- Keys +local epKey = KEYS[1] -- ep:{targetId} combined endpoint state hash + +-- Arguments +local now = tonumber(ARGV[1]) or 0 +local consumedTokens = tonumber(ARGV[2]) or 0 +local processingFailures = tonumber(ARGV[3]) or 0 +local cooldownPeriodMs = tonumber(ARGV[4]) or 0 +local recoveryPeriodMs = tonumber(ARGV[5]) or 0 +local failureThreshold = tonumber(ARGV[6]) or 0 +local minAttempts = tonumber(ARGV[7]) or 0 +local samplePeriodMs = tonumber(ARGV[8]) or 0 + +-------------------------------------------------------------------------------- +-- LOAD CURRENT STATE +-------------------------------------------------------------------------------- + +local state = redis.call("HMGET", epKey, + "is_open", "switched_at", + "cur_attempts", "prev_attempts", "cur_failures", "prev_failures", + "sample_till") +local needInit = state[1] == false or state[1] == nil +local isOpen = needInit or tonumber(state[1]) == 1 +local switchedAt = needInit and 0 or tonumber(state[2] or "0") +local curAttempts = tonumber(state[3] or "0") +local prevAttempts = tonumber(state[4] or "0") +local curFailures = tonumber(state[5] or "0") +local prevFailures = tonumber(state[6] or "0") +local sampleTill = tonumber(state[7] or "0") + +-------------------------------------------------------------------------------- +-- 1. DETERMINE CIRCUIT SUB-STATE +-------------------------------------------------------------------------------- + +local isHalfOpen = isOpen and now > switchedAt + cooldownPeriodMs +local isFullyOpen = isOpen and not isHalfOpen + +-------------------------------------------------------------------------------- +-- 2. MANAGE SLIDING WINDOW +-------------------------------------------------------------------------------- + +if sampleTill < now then + if sampleTill + samplePeriodMs < now then + -- Complete reset — window is too old + prevAttempts = 0 + prevFailures = 0 + sampleTill = now + samplePeriodMs + else + -- Promote current to previous + prevAttempts = curAttempts + prevFailures = curFailures + sampleTill = sampleTill + samplePeriodMs + end + curAttempts = 0 + curFailures = 0 +end + +-------------------------------------------------------------------------------- +-- 3. RECORD NEW ATTEMPTS/FAILURES (unless fully open) +-------------------------------------------------------------------------------- + +if not isFullyOpen then + curAttempts = curAttempts + consumedTokens + curFailures = curFailures + processingFailures +end + +-------------------------------------------------------------------------------- +-- 4. INTERPOLATE VALUES +-------------------------------------------------------------------------------- + +local weight = (sampleTill - now) / samplePeriodMs +local attempts = prevAttempts * weight + curAttempts +local failures = prevFailures * weight + curFailures + +-------------------------------------------------------------------------------- +-- 5. CIRCUIT BREAKER LOGIC +-------------------------------------------------------------------------------- + +local processingSuccesses = consumedTokens - processingFailures +local circuitSwitched = false + +-- Close circuit when half-open and there are successes +if isHalfOpen and processingSuccesses > 0 then + isOpen = false + switchedAt = now + circuitSwitched = true + -- fall through, allow circuit to immediately re-open +end + +-- Open circuit when closed, enough samples, and threshold exceeded +local hasSampledEnough = attempts >= minAttempts +if not isOpen and hasSampledEnough and (failures / attempts) > failureThreshold then + isOpen = true + switchedAt = now + curAttempts = 0 + curFailures = 0 + prevAttempts = 0 + prevFailures = 0 + sampleTill = now + samplePeriodMs + circuitSwitched = true +end + +-------------------------------------------------------------------------------- +-- 6. DETERMINE CURRENT CIRCUIT STATE FOR REPORTING +-------------------------------------------------------------------------------- + +local circuitState +if isOpen then + if now > switchedAt + cooldownPeriodMs then + circuitState = OPEN_HALF + else + circuitState = OPEN + end +else + if now < switchedAt + recoveryPeriodMs then + circuitState = CLOSED_RECOVERY + else + circuitState = CLOSED + end +end + +-------------------------------------------------------------------------------- +-- 7. PERSIST STATE +-------------------------------------------------------------------------------- + +redis.call("HSET", epKey, + "cur_attempts", curAttempts, + "prev_attempts", prevAttempts, + "cur_failures", curFailures, + "prev_failures", prevFailures, + "sample_till", sampleTill +) + +if circuitSwitched then + redis.call("HSET", epKey, + "is_open", isOpen and 1 or 0, + "switched_at", switchedAt + ) +end + +return { circuitState, circuitSwitched and 1 or 0 } diff --git a/lambdas/https-client-lambda/src/services/redis-client.ts b/lambdas/https-client-lambda/src/services/redis-client.ts new file mode 100644 index 00000000..5dbc295c --- /dev/null +++ b/lambdas/https-client-lambda/src/services/redis-client.ts @@ -0,0 +1,107 @@ +import { type RedisClientType, createClient } from "@redis/client"; +import { SignatureV4 } from "@smithy/signature-v4"; +import { Sha256 } from "@aws-crypto/sha256-js"; +import { fromNodeProviderChain } from "@aws-sdk/credential-providers"; +import { logger } from "@nhs-notify-client-callbacks/logger"; + +const TOKEN_EXPIRY_SECONDS = 900; +const TOKEN_REFRESH_BUFFER_SECONDS = 60; + +let redisClient: RedisClientType | undefined; +let tokenExpiry = 0; + +async function generateElastiCacheIamToken(): Promise { + const cacheName = process.env.ELASTICACHE_CACHE_NAME; + const endpoint = process.env.ELASTICACHE_ENDPOINT; + const username = process.env.ELASTICACHE_IAM_USERNAME; + + if (!cacheName || !endpoint || !username) { + throw new Error( + "ELASTICACHE_CACHE_NAME, ELASTICACHE_ENDPOINT, and ELASTICACHE_IAM_USERNAME are required", + ); + } + + const region = process.env.AWS_REGION ?? "eu-west-2"; + + const signer = new SignatureV4({ + credentials: fromNodeProviderChain(), + region, + service: "elasticache", + sha256: Sha256, + }); + + const signed = await signer.presign( + { + protocol: "https:", + method: "GET", + hostname: cacheName, + path: "/", + query: { Action: "connect", User: username }, + headers: { host: cacheName }, + }, + { expiresIn: TOKEN_EXPIRY_SECONDS }, + ); + + tokenExpiry = Date.now() + TOKEN_EXPIRY_SECONDS * 1000; + + logger.info("ElastiCache IAM token generated", { + cacheName, + username, + region, + signingAlgorithm: signed.query?.["X-Amz-Algorithm"], + tokenExpiresAt: new Date(tokenExpiry).toISOString(), + }); + + const qs = new URLSearchParams( + signed.query as Record, + ).toString(); + return `${cacheName}/?${qs}`; +} + +export async function getRedisClient(): Promise { + const isTokenValid = + tokenExpiry > Date.now() + TOKEN_REFRESH_BUFFER_SECONDS * 1000; + + if (redisClient?.isOpen && isTokenValid) { + return redisClient; + } + + const endpoint = process.env.ELASTICACHE_ENDPOINT; + if (!endpoint) { + throw new Error("ELASTICACHE_ENDPOINT is required"); + } + + const username = process.env.ELASTICACHE_IAM_USERNAME; + if (!username) { + throw new Error("ELASTICACHE_IAM_USERNAME is required"); + } + + if (redisClient?.isOpen) { + logger.info("Disconnecting Redis client for token refresh"); + await redisClient.disconnect(); + } + + const token = await generateElastiCacheIamToken(); + + logger.info("Connecting to ElastiCache", { endpoint, username }); + + redisClient = createClient({ + url: `rediss://${endpoint}:6379`, + username, + password: token, + }); + + redisClient.on("error", (err) => { + logger.error("Redis connection error", { error: String(err) }); + }); + + await redisClient.connect(); + return redisClient; +} + +export function resetRedisClient(): void { + redisClient = undefined; + tokenExpiry = 0; +} + +export { type RedisClientType } from "@redis/client"; diff --git a/lambdas/https-client-lambda/src/services/sqs-visibility.ts b/lambdas/https-client-lambda/src/services/sqs-visibility.ts new file mode 100644 index 00000000..e6fe2720 --- /dev/null +++ b/lambdas/https-client-lambda/src/services/sqs-visibility.ts @@ -0,0 +1,21 @@ +import { ChangeMessageVisibilityCommand, SQSClient } from "@aws-sdk/client-sqs"; + +const sqsClient = new SQSClient({}); + +export async function changeVisibility( + receiptHandle: string, + visibilityTimeoutSeconds: number, +): Promise { + const { QUEUE_URL } = process.env; + if (!QUEUE_URL) { + throw new Error("QUEUE_URL is required"); + } + + await sqsClient.send( + new ChangeMessageVisibilityCommand({ + QueueUrl: QUEUE_URL, + ReceiptHandle: receiptHandle, + VisibilityTimeout: Math.floor(visibilityTimeoutSeconds), + }), + ); +} diff --git a/tests/performance/tsconfig.json b/lambdas/https-client-lambda/tsconfig.json similarity index 60% rename from tests/performance/tsconfig.json rename to lambdas/https-client-lambda/tsconfig.json index 2cc7bdfa..a50e6fc0 100644 --- a/tests/performance/tsconfig.json +++ b/lambdas/https-client-lambda/tsconfig.json @@ -2,16 +2,13 @@ "compilerOptions": { "isolatedModules": true, "paths": { - "helpers": [ - "./helpers/index" + "*": [ + "./src/*" ] } }, - "exclude": [ - "jest.config.ts" - ], "extends": "../../tsconfig.base.json", "include": [ - "**/*.ts" + "src/**/*" ] } diff --git a/lambdas/mock-webhook-lambda/src/__tests__/index.test.ts b/lambdas/mock-webhook-lambda/src/__tests__/index.test.ts index 6f3cc917..fefa87ed 100644 --- a/lambdas/mock-webhook-lambda/src/__tests__/index.test.ts +++ b/lambdas/mock-webhook-lambda/src/__tests__/index.test.ts @@ -1,6 +1,14 @@ +import { X509Certificate } from "node:crypto"; import type { APIGatewayProxyEvent } from "aws-lambda"; import { handler } from "index"; +jest.mock("node:crypto", () => ({ + ...jest.requireActual("node:crypto"), + X509Certificate: jest.fn(), +})); + +const mockX509Certificate = X509Certificate as unknown as jest.Mock; + const TEST_API_KEY = "test-api-key"; jest.mock("@nhs-notify-client-callbacks/logger", () => { @@ -32,6 +40,28 @@ const createMockEvent = ( ): APIGatewayProxyEvent => ({ body, headers, rawPath }) as unknown as APIGatewayProxyEvent; +const createAlbEvent = ( + body: string | null, + headers: Record = DEFAULT_HEADERS, + extraHeaders: Record = {}, +): APIGatewayProxyEvent => + ({ + body, + path: "/target-abc", + httpMethod: "POST", + headers: { ...headers, ...extraHeaders }, + requestContext: { + elb: { + targetGroupArn: + "arn:aws:elasticloadbalancing:eu-west-2:123456789012:targetgroup/mock/abc", + }, + }, + }) as unknown as APIGatewayProxyEvent; + +const FAKE_CERT_HEADER = encodeURIComponent( + "-----BEGIN CERTIFICATE-----\nZmFrZQ==\n-----END CERTIFICATE-----", +); + describe("Mock Webhook Lambda", () => { beforeAll(() => { process.env.API_KEY = TEST_API_KEY; @@ -337,6 +367,54 @@ describe("Mock Webhook Lambda", () => { const body = JSON.parse(result.body); expect(body.message).toBe("Forced status 500"); }); + + it("should return forced status code when messageId uses timed format and deadline is in the future", async () => { + const futureMs = Date.now() + 60_000; + const callback = { + data: [ + { + type: "MessageStatus", + attributes: { + messageId: `force-500-until-${futureMs}-some-uuid`, + messageStatus: "delivered", + }, + links: { message: "some-message-link" }, + meta: { idempotencyKey: "some-idempotency-key" }, + }, + ], + }; + + const event = createMockEvent(JSON.stringify(callback)); + const result = await handler(event); + + expect(result.statusCode).toBe(500); + const body = JSON.parse(result.body); + expect(body.message).toBe("Forced status 500"); + }); + + it("should return 200 when messageId uses timed format and deadline has passed", async () => { + const pastMs = Date.now() - 60_000; + const callback = { + data: [ + { + type: "MessageStatus", + attributes: { + messageId: `force-500-until-${pastMs}-some-uuid`, + messageStatus: "delivered", + }, + links: { message: "some-message-link" }, + meta: { idempotencyKey: "some-idempotency-key" }, + }, + ], + }; + + const event = createMockEvent(JSON.stringify(callback)); + const result = await handler(event); + + expect(result.statusCode).toBe(200); + const body = JSON.parse(result.body); + expect(body.message).toBe("Callback received"); + }); }); describe("Logging", () => { @@ -381,3 +459,144 @@ describe("Mock Webhook Lambda", () => { }); }); }); + +describe("ALB mTLS certificate logging", () => { + beforeAll(() => { + process.env.API_KEY = TEST_API_KEY; + }); + + afterAll(() => { + delete process.env.API_KEY; + }); + + beforeEach(() => { + mockX509Certificate.mockReset(); + mockX509Certificate.mockImplementation(() => ({ + validFrom: new Date(Date.now() - 86_400_000).toString(), + validTo: new Date(Date.now() + 86_400_000).toString(), + })); + }); + + it("logs isMtls=false and proceeds when ALB invocation has no client certificate header", async () => { + const event = createAlbEvent(JSON.stringify({ data: [] })); + const result = await handler(event); + + expect(result.statusCode).not.toBe(401); + expect(mockLogger.info).toHaveBeenCalledWith( + "Mock webhook invoked without mTLS", + expect.objectContaining({ isMtls: false }), + ); + }); + + it("logs isMtls=false and proceeds when client certificate header cannot be parsed", async () => { + mockX509Certificate.mockImplementationOnce(() => { + throw new Error("Invalid certificate"); + }); + const event = createAlbEvent( + JSON.stringify({ data: [] }), + DEFAULT_HEADERS, + { "x-amzn-mtls-clientcert": FAKE_CERT_HEADER }, + ); + const result = await handler(event); + + expect(result.statusCode).not.toBe(401); + expect(mockLogger.info).toHaveBeenCalledWith( + "Mock webhook invoked without mTLS", + expect.objectContaining({ isMtls: false }), + ); + }); + + it("logs isMtls=false and proceeds when client certificate is expired", async () => { + mockX509Certificate.mockImplementationOnce(() => ({ + validFrom: new Date(Date.now() - 172_800_000).toString(), + validTo: new Date(Date.now() - 86_400_000).toString(), + })); + const event = createAlbEvent( + JSON.stringify({ data: [] }), + DEFAULT_HEADERS, + { "x-amzn-mtls-clientcert": FAKE_CERT_HEADER }, + ); + const result = await handler(event); + + expect(result.statusCode).not.toBe(401); + expect(mockLogger.info).toHaveBeenCalledWith( + "Mock webhook invoked without mTLS", + expect.objectContaining({ isMtls: false }), + ); + }); + + it("logs isMtls=true and proceeds when certificate is valid", async () => { + const event = createAlbEvent( + JSON.stringify({ data: [] }), + { "x-api-key": "wrong-key" }, + { "x-amzn-mtls-clientcert": FAKE_CERT_HEADER }, + ); + const result = await handler(event); + + expect(mockLogger.info).toHaveBeenCalledWith( + "mTLS client certificate verified", + expect.objectContaining({ isMtls: true }), + ); + expect(result.statusCode).toBe(401); + const body = JSON.parse(result.body); + expect(body.message).toBe("Unauthorized"); + }); + + it("processes request successfully when certificate is valid and API key is correct", async () => { + const callback = { + data: [ + { + type: "MessageStatus", + attributes: { + messageId: "msg-alb-mtls", + messageReference: "ref-alb", + messageStatus: "delivered", + timestamp: "2026-01-01T00:00:00Z", + }, + links: { message: "some-link" }, + meta: { idempotencyKey: "idem-key-alb" }, + }, + ], + }; + const event = createAlbEvent(JSON.stringify(callback), DEFAULT_HEADERS, { + "x-amzn-mtls-clientcert": FAKE_CERT_HEADER, + }); + const result = await handler(event); + + expect(result.statusCode).toBe(200); + const body = JSON.parse(result.body); + expect(body.message).toBe("Callback received"); + }); + + it("processes non-mTLS ALB request successfully when API key is correct", async () => { + const callback = { + data: [ + { + type: "MessageStatus", + attributes: { + messageId: "msg-alb-no-mtls", + messageReference: "ref-alb", + messageStatus: "delivered", + timestamp: "2026-01-01T00:00:00Z", + }, + links: { message: "some-link" }, + meta: { idempotencyKey: "idem-key-alb-no-mtls" }, + }, + ], + }; + const event = createAlbEvent(JSON.stringify(callback), DEFAULT_HEADERS); + const result = await handler(event); + + expect(result.statusCode).toBe(200); + const body = JSON.parse(result.body); + expect(body.message).toBe("Callback received"); + }); + + it("non-ALB invocations skip certificate check", async () => { + const event = createMockEvent(JSON.stringify({ data: [] })); + const result = await handler(event); + + const body = JSON.parse(result.body); + expect(body.message).not.toBe("Mutual TLS authentication required"); + }); +}); diff --git a/lambdas/mock-webhook-lambda/src/index.ts b/lambdas/mock-webhook-lambda/src/index.ts index 68d41231..f17e3b10 100644 --- a/lambdas/mock-webhook-lambda/src/index.ts +++ b/lambdas/mock-webhook-lambda/src/index.ts @@ -1,10 +1,33 @@ -import { createHash, timingSafeEqual } from "node:crypto"; +import { X509Certificate, createHash, timingSafeEqual } from "node:crypto"; import type { APIGatewayProxyEvent, APIGatewayProxyResult } from "aws-lambda"; import { Logger } from "@nhs-notify-client-callbacks/logger"; import type { ClientCallbackPayload } from "@nhs-notify-client-callbacks/models"; const logger = new Logger(); +function verifyClientCertificate(certHeader: string | undefined): { + valid: boolean; + reason?: string; +} { + if (!certHeader) { + return { valid: false, reason: "No client certificate provided" }; + } + try { + const pem = decodeURIComponent(certHeader); + const cert = new X509Certificate(pem); + const now = new Date(); + if (now < new Date(cert.validFrom) || now > new Date(cert.validTo)) { + return { + valid: false, + reason: "Client certificate is not within its validity period", + }; + } + return { valid: true }; + } catch { + return { valid: false, reason: "Failed to parse client certificate" }; + } +} + function isClientCallbackPayload( value: unknown, ): value is ClientCallbackPayload { @@ -34,28 +57,52 @@ function isClientCallbackPayload( }); } -async function buildResponse( - event: APIGatewayProxyEvent, -): Promise { - const eventWithFunctionUrlFields = event as APIGatewayProxyEvent & { - rawPath?: string; - requestContext?: { http?: { method?: string } }; +type EventWithContextFields = APIGatewayProxyEvent & { + rawPath?: string; + requestContext?: { + http?: { method?: string }; + elb?: { targetGroupArn: string }; }; - const headers = Object.fromEntries( +}; + +function normalizeHeaders( + event: APIGatewayProxyEvent, +): Record { + return Object.fromEntries( Object.entries(event.headers).map(([k, v]) => [String(k).toLowerCase(), v]), ) as Record; +} - const path = event.path ?? eventWithFunctionUrlFields.rawPath; +function resolveMtlsStatus( + headers: Record, + isAlbInvocation: boolean, +): boolean { + if (!isAlbInvocation) { + return false; + } - logger.info("Mock webhook invoked", { - path, - method: event.httpMethod, - hasBody: Boolean(event.body), - "x-api-key": headers["x-api-key"], - "x-hmac-sha256-signature": headers["x-hmac-sha256-signature"], - payload: event.body, + const clientCertPresent = Boolean(headers["x-amzn-mtls-clientcert"]); + const certResult = verifyClientCertificate(headers["x-amzn-mtls-clientcert"]); + + if (certResult.valid) { + logger.info("mTLS client certificate verified", { + fingerprint: headers["x-amzn-mtls-clientcert-fingerprint"] ?? "", + isMtls: true, + }); + return true; + } + + logger.info("Mock webhook invoked without mTLS", { + isMtls: false, + clientCertPresent, + reason: certResult.reason, }); + return false; +} +function authenticateApiKey(headers: Record): { + error: APIGatewayProxyResult | undefined; +} { const expectedApiKey = process.env.API_KEY; const providedApiKey = headers["x-api-key"]; @@ -69,97 +116,180 @@ async function buildResponse( ) { logger.error("Unauthorized: invalid or missing x-api-key"); return { - statusCode: 401, - body: JSON.stringify({ message: "Unauthorized" }), + error: { + statusCode: 401, + body: JSON.stringify({ message: "Unauthorized" }), + }, }; } - if (!event.body) { - logger.error("No event body received"); + return { error: undefined }; +} - return { +type ParseResult = { + payload: ClientCallbackPayload | undefined; + error: APIGatewayProxyResult | undefined; +}; + +function parseError(response: APIGatewayProxyResult): ParseResult { + return { payload: undefined, error: response }; +} + +function parseAndValidateBody(body: string | null): ParseResult { + if (!body) { + logger.error("No event body received"); + return parseError({ statusCode: 400, body: JSON.stringify({ message: "No body" }), - }; + }); } try { - const parsed = JSON.parse(event.body) as unknown; - + const parsed = JSON.parse(body) as unknown; logger.info("Mock webhook parsed payload", { parsedPayload: parsed }); if (!isClientCallbackPayload(parsed)) { logger.error("Invalid message structure - missing or invalid data array"); - - return { + return parseError({ statusCode: 400, body: JSON.stringify({ message: "Invalid message structure" }), - }; + }); } if (parsed.data.length !== 1) { logger.error("Expected exactly 1 callback item in data array", { receivedCount: parsed.data.length, }); - - return { + return parseError({ statusCode: 400, body: JSON.stringify({ message: `Expected exactly 1 callback item, got ${parsed.data.length}`, }), - }; + }); + } + + return { payload: parsed, error: undefined }; + } catch (error) { + if (error instanceof SyntaxError) { + logger.error("Invalid JSON body", { error: error.message }); + return parseError({ + statusCode: 400, + body: JSON.stringify({ message: "Invalid JSON body" }), + }); } - const [item] = parsed.data; - const correlationId = item.meta.idempotencyKey; - const { messageId } = item.attributes; - const forcedStatusMatch = /^force-(\d{3})-/.exec(messageId); - if (forcedStatusMatch) { - const statusCode = Number(forcedStatusMatch[1]); - logger.info("Forced status code response", { + logger.error("Failed to process callback", { + error: error instanceof Error ? error.message : String(error), + }); + return parseError({ + statusCode: 500, + body: JSON.stringify({ message: "Internal server error" }), + }); + } +} + +function checkForcedStatusResponse( + messageId: string, + correlationId: string, +): { response: APIGatewayProxyResult | undefined } { + const timedMatch = /^force-(\d{3})-until-(\d+)-/.exec(messageId); + if (timedMatch) { + const statusCode = Number(timedMatch[1]); + const until = Number(timedMatch[2]); + if (Date.now() < until) { + logger.info("Timed forced status code response", { correlationId, messageId, statusCode, + until, }); return { - statusCode, - body: JSON.stringify({ message: `Forced status ${statusCode}` }), + response: { + statusCode, + body: JSON.stringify({ message: `Forced status ${statusCode}` }), + }, }; } + return { response: undefined }; + } - logger.info("Callback received", { + const permanentMatch = /^force-(\d{3})-/.exec(messageId); + if (permanentMatch) { + const statusCode = Number(permanentMatch[1]); + logger.info("Forced status code response", { correlationId, messageId, - callbackType: item.type, - path, - apiKey: providedApiKey, - signature: headers["x-hmac-sha256-signature"] ?? "", - payload: JSON.stringify(item), + statusCode, }); - return { - statusCode: 200, - body: JSON.stringify({ message: "Callback received" }), + response: { + statusCode, + body: JSON.stringify({ message: `Forced status ${statusCode}` }), + }, }; - } catch (error) { - if (error instanceof SyntaxError) { - logger.error("Invalid JSON body", { error: error.message }); + } - return { - statusCode: 400, - body: JSON.stringify({ message: "Invalid JSON body" }), - }; - } + return { response: undefined }; +} - logger.error("Failed to process callback", { - error: error instanceof Error ? error.message : String(error), - }); +async function buildResponse( + event: APIGatewayProxyEvent, +): Promise { + const eventWithContextFields = event as EventWithContextFields; + const headers = normalizeHeaders(event); + const path = event.path ?? eventWithContextFields.rawPath; + const isAlbInvocation = Boolean(eventWithContextFields.requestContext?.elb); + const clientCertPresent = Boolean(headers["x-amzn-mtls-clientcert"]); + const isMtls = resolveMtlsStatus(headers, isAlbInvocation); - return { - statusCode: 500, - body: JSON.stringify({ message: "Internal server error" }), - }; + logger.info("Mock webhook invoked", { + path, + method: event.httpMethod, + hasBody: Boolean(event.body), + isMtls, + clientCertPresent, + "x-api-key": headers["x-api-key"], + "x-hmac-sha256-signature": headers["x-hmac-sha256-signature"], + payload: event.body, + }); + + const authResult = authenticateApiKey(headers); + if (authResult.error) { + return authResult.error; } + + const bodyResult = parseAndValidateBody(event.body); + if (bodyResult.error) { + return bodyResult.error; + } + + const [item] = bodyResult.payload!.data; + const correlationId = item.meta.idempotencyKey; + const { messageId } = item.attributes; + + const { response: forcedResponse } = checkForcedStatusResponse( + messageId, + correlationId, + ); + if (forcedResponse) { + return forcedResponse; + } + + logger.info("Callback received", { + correlationId, + messageId, + callbackType: item.type, + path, + isMtls, + apiKey: headers["x-api-key"], + signature: headers["x-hmac-sha256-signature"] ?? "", + payload: JSON.stringify(item), + }); + + return { + statusCode: 200, + body: JSON.stringify({ message: "Callback received" }), + }; } export async function handler( diff --git a/lambdas/perf-runner-lambda/jest.config.ts b/lambdas/perf-runner-lambda/jest.config.ts new file mode 100644 index 00000000..218d8ffd --- /dev/null +++ b/lambdas/perf-runner-lambda/jest.config.ts @@ -0,0 +1,13 @@ +import { nodeJestConfig } from "../../jest.config.base.ts"; + +export default { + ...nodeJestConfig, + coverageThreshold: { + global: { + ...nodeJestConfig.coverageThreshold?.global, + branches: 100, + lines: 100, + statements: 100, + }, + }, +}; diff --git a/tests/performance/package.json b/lambdas/perf-runner-lambda/package.json similarity index 51% rename from tests/performance/package.json rename to lambdas/perf-runner-lambda/package.json index 5e2f6c2f..59d7691b 100644 --- a/tests/performance/package.json +++ b/lambdas/perf-runner-lambda/package.json @@ -1,30 +1,36 @@ { + "name": "nhs-notify-perf-runner-lambda", + "version": "0.0.1", + "private": true, "engines": { "node": ">=24.14.1" }, - "name": "nhs-notify-client-callbacks-performance-tests", - "version": "0.0.1", - "private": true, "scripts": { - "test:performance": "jest", - "test:unit": "echo 'No unit tests in performance workspace - skipping'", + "lambda-build": "rm -rf dist && pnpm exec esbuild --bundle --minify --sourcemap --target=es2020 --platform=node --entry-names=[name] --outdir=dist src/index.ts", "lint": "eslint .", "lint:fix": "eslint . --fix", + "test:unit": "jest", "typecheck": "tsc --noEmit" }, "dependencies": { + "@aws-crypto/sha256-js": "catalog:aws", "@aws-sdk/client-cloudwatch-logs": "catalog:aws", "@aws-sdk/client-sqs": "catalog:aws", + "@aws-sdk/credential-providers": "catalog:aws", + "@smithy/signature-v4": "catalog:aws", + "@nhs-notify-client-callbacks/logger": "workspace:*", "@nhs-notify-client-callbacks/models": "workspace:*", - "@nhs-notify-client-callbacks/test-support": "workspace:*", - "async-wait-until": "catalog:app" + "@redis/client": "catalog:app" }, "devDependencies": { + "esbuild": "catalog:tools", "@tsconfig/node22": "catalog:tools", + "@types/aws-lambda": "catalog:tools", "@types/jest": "catalog:test", "@types/node": "catalog:tools", "eslint": "catalog:lint", "jest": "catalog:test", + "ts-jest": "catalog:test", "typescript": "catalog:tools" } } diff --git a/lambdas/perf-runner-lambda/src/__tests__/cloudwatch.test.ts b/lambdas/perf-runner-lambda/src/__tests__/cloudwatch.test.ts new file mode 100644 index 00000000..a6f4d36c --- /dev/null +++ b/lambdas/perf-runner-lambda/src/__tests__/cloudwatch.test.ts @@ -0,0 +1,662 @@ +import type { CloudWatchLogsClient } from "@aws-sdk/client-cloudwatch-logs"; +import { + queryCircuitBreakerSnapshot, + queryDeliveryMetricsSnapshot, + queryMetricsSnapshot, + queryPerClientRateTimeline, +} from "cloudwatch"; + +const mockCloudWatchClient = { + send: jest.fn(), +} as unknown as jest.Mocked; + +beforeEach(() => { + jest.useFakeTimers(); +}); + +afterEach(() => { + jest.useRealTimers(); +}); + +describe("queryMetricsSnapshot", () => { + it("returns null when StartQuery returns no queryId", async () => { + mockCloudWatchClient.send.mockResolvedValueOnce({} as never); + + const result = await queryMetricsSnapshot( + mockCloudWatchClient, + "/aws/lambda/nhs-dev-cb-client-transform-filter", + 1_700_000_000, + 1_700_000_060, + ); + + expect(result).toBeNull(); + }); + + it("returns null when the query status is Failed", async () => { + mockCloudWatchClient.send + .mockResolvedValueOnce({ queryId: "qid-1" } as never) + .mockResolvedValueOnce({ status: "Failed" } as never); + + const promise = queryMetricsSnapshot( + mockCloudWatchClient, + "/aws/lambda/test", + 0, + 60, + ); + + await jest.runAllTimersAsync(); + const result = await promise; + + expect(result).toBeNull(); + }); + + it("returns null when the query status is Cancelled", async () => { + mockCloudWatchClient.send + .mockResolvedValueOnce({ queryId: "qid-2" } as never) + .mockResolvedValueOnce({ status: "Cancelled" } as never); + + const promise = queryMetricsSnapshot( + mockCloudWatchClient, + "/aws/lambda/test", + 0, + 60, + ); + + await jest.runAllTimersAsync(); + const result = await promise; + + expect(result).toBeNull(); + }); + + it("returns a snapshot with zeroed metrics when the result row is empty", async () => { + mockCloudWatchClient.send + .mockResolvedValueOnce({ queryId: "qid-3" } as never) + .mockResolvedValueOnce({ status: "Complete", results: [] } as never); + + const promise = queryMetricsSnapshot( + mockCloudWatchClient, + "/aws/lambda/test", + 0, + 60, + ); + + await jest.runAllTimersAsync(); + const result = await promise; + + expect(result).toMatchObject({ p50Ms: 0, p95Ms: 0, p99Ms: 0, count: 0 }); + expect(result?.snapshotAt).toBeGreaterThan(0); + }); + + it("returns a populated snapshot when query completes successfully", async () => { + mockCloudWatchClient.send + .mockResolvedValueOnce({ queryId: "qid-4" } as never) + .mockResolvedValueOnce({ + status: "Complete", + results: [ + [ + { field: "eventCount", value: "500" }, + { field: "p50", value: "42" }, + { field: "p95", value: "120" }, + { field: "p99", value: "250" }, + ], + ], + } as never); + + const promise = queryMetricsSnapshot( + mockCloudWatchClient, + "/aws/lambda/test", + 0, + 60, + ); + + await jest.runAllTimersAsync(); + const result = await promise; + + expect(result).toMatchObject({ + count: 500, + p50Ms: 42, + p95Ms: 120, + p99Ms: 250, + }); + }); + + it("polls until the query becomes Complete", async () => { + mockCloudWatchClient.send + .mockResolvedValueOnce({ queryId: "qid-5" } as never) + .mockResolvedValueOnce({ status: "Running" } as never) + .mockResolvedValueOnce({ status: "Running" } as never) + .mockResolvedValueOnce({ + status: "Complete", + results: [[{ field: "eventCount", value: "10" }]], + } as never); + + const promise = queryMetricsSnapshot( + mockCloudWatchClient, + "/aws/lambda/test", + 0, + 60, + ); + + await jest.runAllTimersAsync(); + const result = await promise; + + expect(result?.count).toBe(10); + expect(mockCloudWatchClient.send).toHaveBeenCalledTimes(4); + }); + + it("returns null when the query does not complete within the timeout", async () => { + mockCloudWatchClient.send + .mockResolvedValueOnce({ queryId: "qid-6" } as never) + .mockResolvedValue({ status: "Running" } as never); + + const promise = queryMetricsSnapshot( + mockCloudWatchClient, + "/aws/lambda/test", + 0, + 60, + ); + + await jest.advanceTimersByTimeAsync(60_000); + const result = await promise; + + expect(result).toBeNull(); + }); +}); + +describe("queryDeliveryMetricsSnapshot", () => { + it("returns null when logGroupNames is empty", async () => { + const result = await queryDeliveryMetricsSnapshot( + mockCloudWatchClient, + [], + 0, + 60, + ); + + expect(result).toBeNull(); + expect(mockCloudWatchClient.send).not.toHaveBeenCalled(); + }); + + it("returns null when StartQuery returns no queryId", async () => { + mockCloudWatchClient.send.mockResolvedValueOnce({} as never); + + const result = await queryDeliveryMetricsSnapshot( + mockCloudWatchClient, + ["/aws/lambda/test-https-client-perf-client-1"], + 0, + 60, + ); + + expect(result).toBeNull(); + }); + + it("sends logGroupNames to StartQuery", async () => { + mockCloudWatchClient.send + .mockResolvedValueOnce({ queryId: "qid-d1" } as never) + .mockResolvedValueOnce({ status: "Complete", results: [] } as never); + + const logGroups = [ + "/aws/lambda/test-https-client-perf-client-1", + "/aws/lambda/test-https-client-perf-client-2", + ]; + + const promise = queryDeliveryMetricsSnapshot( + mockCloudWatchClient, + logGroups, + 0, + 60, + ); + + await jest.runAllTimersAsync(); + await promise; + + const startCmd = mockCloudWatchClient.send.mock.calls[0][0] as { + input: { logGroupNames: string[] }; + }; + expect(startCmd.input.logGroupNames).toEqual(logGroups); + }); + + it("returns a snapshot with zeroed metrics when the result row is empty", async () => { + mockCloudWatchClient.send + .mockResolvedValueOnce({ queryId: "qid-d2" } as never) + .mockResolvedValueOnce({ status: "Complete", results: [] } as never); + + const promise = queryDeliveryMetricsSnapshot( + mockCloudWatchClient, + ["/aws/lambda/test-https-client-perf-client-1"], + 0, + 60, + ); + + await jest.runAllTimersAsync(); + const result = await promise; + + expect(result).toMatchObject({ + deliveryCount: 0, + p50Ms: 0, + p95Ms: 0, + p99Ms: 0, + }); + expect(result?.snapshotAt).toBeGreaterThan(0); + }); + + it("returns a populated snapshot when query completes successfully", async () => { + mockCloudWatchClient.send + .mockResolvedValueOnce({ queryId: "qid-d3" } as never) + .mockResolvedValueOnce({ + status: "Complete", + results: [ + [ + { field: "deliveryCount", value: "200" }, + { field: "p50", value: "85" }, + { field: "p95", value: "250" }, + { field: "p99", value: "450" }, + ], + ], + } as never); + + const promise = queryDeliveryMetricsSnapshot( + mockCloudWatchClient, + ["/aws/lambda/test-https-client-perf-client-1"], + 0, + 60, + ); + + await jest.runAllTimersAsync(); + const result = await promise; + + expect(result).toMatchObject({ + deliveryCount: 200, + p50Ms: 85, + p95Ms: 250, + p99Ms: 450, + }); + }); + + it("returns null when the query status is Failed", async () => { + mockCloudWatchClient.send + .mockResolvedValueOnce({ queryId: "qid-d4" } as never) + .mockResolvedValueOnce({ status: "Failed" } as never); + + const promise = queryDeliveryMetricsSnapshot( + mockCloudWatchClient, + ["/aws/lambda/test-https-client-perf-client-1"], + 0, + 60, + ); + + await jest.runAllTimersAsync(); + const result = await promise; + + expect(result).toBeNull(); + }); +}); + +describe("queryCircuitBreakerSnapshot", () => { + it("returns null when logGroupNames is empty", async () => { + const result = await queryCircuitBreakerSnapshot( + mockCloudWatchClient, + [], + 0, + 60, + ); + + expect(result).toBeNull(); + expect(mockCloudWatchClient.send).not.toHaveBeenCalled(); + }); + + it("returns null when StartQuery returns no queryId", async () => { + mockCloudWatchClient.send.mockResolvedValueOnce({} as never); + + const result = await queryCircuitBreakerSnapshot( + mockCloudWatchClient, + ["/aws/lambda/test-https-client-perf-client-1"], + 0, + 60, + ); + + expect(result).toBeNull(); + }); + + it("returns a snapshot with zeroed metrics when the result row is empty", async () => { + mockCloudWatchClient.send + .mockResolvedValueOnce({ queryId: "qid-cb1" } as never) + .mockResolvedValueOnce({ status: "Complete", results: [] } as never); + + const promise = queryCircuitBreakerSnapshot( + mockCloudWatchClient, + ["/aws/lambda/test-https-client-perf-client-1"], + 100, + 160, + ); + + await jest.runAllTimersAsync(); + const result = await promise; + + expect(result).toMatchObject({ + intervalStartSec: 100, + intervalEndSec: 160, + circuitOpenEvents: 0, + circuitCloseEvents: 0, + admissionDeniedCircuitOpen: 0, + admissionDeniedRateLimited: 0, + deliveryAttempts: 0, + deliverySuccesses: 0, + deliveryFailures: 0, + deliveryRateLimited: 0, + }); + expect(result?.snapshotAt).toBeGreaterThan(0); + }); + + it("returns a populated snapshot when query completes successfully", async () => { + mockCloudWatchClient.send + .mockResolvedValueOnce({ queryId: "qid-cb2" } as never) + .mockResolvedValueOnce({ + status: "Complete", + results: [ + [ + { field: "circuitOpenEvents", value: "3" }, + { field: "circuitCloseEvents", value: "2" }, + { field: "admissionDeniedCircuitOpen", value: "15" }, + { field: "admissionDeniedRateLimited", value: "8" }, + { field: "deliveryAttempts", value: "200" }, + { field: "deliverySuccesses", value: "180" }, + { field: "deliveryFailures", value: "12" }, + { field: "deliveryRateLimited", value: "8" }, + ], + ], + } as never); + + const promise = queryCircuitBreakerSnapshot( + mockCloudWatchClient, + ["/aws/lambda/test-https-client-perf-client-1"], + 100, + 160, + ); + + await jest.runAllTimersAsync(); + const result = await promise; + + expect(result).toMatchObject({ + intervalStartSec: 100, + intervalEndSec: 160, + circuitOpenEvents: 3, + circuitCloseEvents: 2, + admissionDeniedCircuitOpen: 15, + admissionDeniedRateLimited: 8, + deliveryAttempts: 200, + deliverySuccesses: 180, + deliveryFailures: 12, + deliveryRateLimited: 8, + }); + }); + + it("sends logGroupNames to StartQuery", async () => { + mockCloudWatchClient.send + .mockResolvedValueOnce({ queryId: "qid-cb3" } as never) + .mockResolvedValueOnce({ status: "Complete", results: [] } as never); + + const logGroups = [ + "/aws/lambda/test-https-client-perf-client-1", + "/aws/lambda/test-https-client-perf-client-2", + ]; + + const promise = queryCircuitBreakerSnapshot( + mockCloudWatchClient, + logGroups, + 0, + 60, + ); + + await jest.runAllTimersAsync(); + await promise; + + const startCmd = mockCloudWatchClient.send.mock.calls[0][0] as { + input: { logGroupNames: string[] }; + }; + expect(startCmd.input.logGroupNames).toEqual(logGroups); + }); + + it("returns null when the query status is Failed", async () => { + mockCloudWatchClient.send + .mockResolvedValueOnce({ queryId: "qid-cb4" } as never) + .mockResolvedValueOnce({ status: "Failed" } as never); + + const promise = queryCircuitBreakerSnapshot( + mockCloudWatchClient, + ["/aws/lambda/test-https-client-perf-client-1"], + 0, + 60, + ); + + await jest.runAllTimersAsync(); + const result = await promise; + + expect(result).toBeNull(); + }); +}); + +describe("queryPerClientRateTimeline", () => { + it("returns empty array when StartQuery returns no queryId", async () => { + mockCloudWatchClient.send.mockResolvedValueOnce({} as never); + + const result = await queryPerClientRateTimeline( + mockCloudWatchClient, + "/aws/lambda/test-https-client-perf-client-1", + 0, + 60, + ); + + expect(result).toEqual([]); + }); + + it("returns empty array when the query status is Failed", async () => { + mockCloudWatchClient.send + .mockResolvedValueOnce({ queryId: "qid-pcr1" } as never) + .mockResolvedValueOnce({ status: "Failed" } as never); + + const promise = queryPerClientRateTimeline( + mockCloudWatchClient, + "/aws/lambda/test-https-client-perf-client-1", + 0, + 60, + ); + + await jest.runAllTimersAsync(); + const result = await promise; + + expect(result).toEqual([]); + }); + + it("returns empty array when results are empty", async () => { + mockCloudWatchClient.send + .mockResolvedValueOnce({ queryId: "qid-pcr2" } as never) + .mockResolvedValueOnce({ status: "Complete", results: [] } as never); + + const promise = queryPerClientRateTimeline( + mockCloudWatchClient, + "/aws/lambda/test-https-client-perf-client-1", + 0, + 60, + ); + + await jest.runAllTimersAsync(); + const result = await promise; + + expect(result).toEqual([]); + }); + + it("returns empty array when results is undefined", async () => { + mockCloudWatchClient.send + .mockResolvedValueOnce({ queryId: "qid-pcr2b" } as never) + .mockResolvedValueOnce({ status: "Complete" } as never); + + const promise = queryPerClientRateTimeline( + mockCloudWatchClient, + "/aws/lambda/test-https-client-perf-client-1", + 0, + 60, + ); + + await jest.runAllTimersAsync(); + const result = await promise; + + expect(result).toEqual([]); + }); + + it("defaults missing fields to zero", async () => { + mockCloudWatchClient.send + .mockResolvedValueOnce({ queryId: "qid-pcr2c" } as never) + .mockResolvedValueOnce({ + status: "Complete", + results: [[{ field: "unknownField", value: "123" }]], + } as never); + + const promise = queryPerClientRateTimeline( + mockCloudWatchClient, + "/aws/lambda/test-https-client-perf-client-1", + 0, + 60, + ); + + await jest.runAllTimersAsync(); + const result = await promise; + + expect(result).toHaveLength(1); + expect(result[0].deliveryAttempts).toBe(0); + expect(result[0].timestampSec).toBe( + Math.floor(new Date("0").getTime() / 1000), + ); + }); + + it("returns entries sorted by time bin when query completes", async () => { + mockCloudWatchClient.send + .mockResolvedValueOnce({ queryId: "qid-pcr3" } as never) + .mockResolvedValueOnce({ + status: "Complete", + results: [ + [ + { field: "timeBin", value: "2026-04-09 10:00:00.000" }, + { field: "deliveryAttempts", value: "42" }, + ], + [ + { field: "timeBin", value: "2026-04-09 10:00:10.000" }, + { field: "deliveryAttempts", value: "38" }, + ], + ], + } as never); + + const promise = queryPerClientRateTimeline( + mockCloudWatchClient, + "/aws/lambda/test-https-client-perf-client-1", + 0, + 60, + ); + + await jest.runAllTimersAsync(); + const result = await promise; + + expect(result).toHaveLength(2); + expect(result[0]).toEqual({ + timestampSec: Math.floor( + new Date("2026-04-09 10:00:00.000").getTime() / 1000, + ), + deliveryAttempts: 42, + }); + expect(result[1]).toEqual({ + timestampSec: Math.floor( + new Date("2026-04-09 10:00:10.000").getTime() / 1000, + ), + deliveryAttempts: 38, + }); + }); + + it("sends logGroupName to StartQuery", async () => { + mockCloudWatchClient.send + .mockResolvedValueOnce({ queryId: "qid-pcr4" } as never) + .mockResolvedValueOnce({ status: "Complete", results: [] } as never); + + const promise = queryPerClientRateTimeline( + mockCloudWatchClient, + "/aws/lambda/test-https-client-perf-client-1", + 100, + 200, + ); + + await jest.runAllTimersAsync(); + await promise; + + const startCmd = mockCloudWatchClient.send.mock.calls[0][0] as { + input: { logGroupName: string; startTime: number; endTime: number }; + }; + expect(startCmd.input.logGroupName).toBe( + "/aws/lambda/test-https-client-perf-client-1", + ); + expect(startCmd.input.startTime).toBe(100); + expect(startCmd.input.endTime).toBe(200); + }); + + it("polls until the query becomes Complete", async () => { + mockCloudWatchClient.send + .mockResolvedValueOnce({ queryId: "qid-pcr5" } as never) + .mockResolvedValueOnce({ status: "Running" } as never) + .mockResolvedValueOnce({ + status: "Complete", + results: [ + [ + { field: "timeBin", value: "2026-04-09 10:00:00.000" }, + { field: "deliveryAttempts", value: "5" }, + ], + ], + } as never); + + const promise = queryPerClientRateTimeline( + mockCloudWatchClient, + "/aws/lambda/test-https-client-perf-client-1", + 0, + 60, + ); + + await jest.runAllTimersAsync(); + const result = await promise; + + expect(result).toHaveLength(1); + expect(result[0].deliveryAttempts).toBe(5); + expect(mockCloudWatchClient.send).toHaveBeenCalledTimes(3); + }); + + it("returns empty array when the query does not complete within the timeout", async () => { + mockCloudWatchClient.send + .mockResolvedValueOnce({ queryId: "qid-pcr6" } as never) + .mockResolvedValue({ status: "Running" } as never); + + const promise = queryPerClientRateTimeline( + mockCloudWatchClient, + "/aws/lambda/test-https-client-perf-client-1", + 0, + 60, + ); + + await jest.advanceTimersByTimeAsync(60_000); + const result = await promise; + + expect(result).toEqual([]); + }); + + it("returns empty array when the query status is Cancelled", async () => { + mockCloudWatchClient.send + .mockResolvedValueOnce({ queryId: "qid-pcr7" } as never) + .mockResolvedValueOnce({ status: "Cancelled" } as never); + + const promise = queryPerClientRateTimeline( + mockCloudWatchClient, + "/aws/lambda/test-https-client-perf-client-1", + 0, + 60, + ); + + await jest.runAllTimersAsync(); + const result = await promise; + + expect(result).toEqual([]); + }); +}); diff --git a/lambdas/perf-runner-lambda/src/__tests__/elasticache.test.ts b/lambdas/perf-runner-lambda/src/__tests__/elasticache.test.ts new file mode 100644 index 00000000..54c8e813 --- /dev/null +++ b/lambdas/perf-runner-lambda/src/__tests__/elasticache.test.ts @@ -0,0 +1,165 @@ +import { dumpRateLimitState, flushElastiCache } from "elasticache"; +import type { ElastiCacheDeps } from "types"; + +const mockConnect = jest.fn().mockResolvedValue(undefined); +const mockFlushAll = jest.fn().mockResolvedValue("OK"); +const mockDisconnect = jest.fn().mockResolvedValue(undefined); +const mockHmGet = jest.fn().mockResolvedValue([]); +let mockIsOpen = true; +let mockScanKeys: string[] = []; + +jest.mock("@redis/client", () => ({ + createClient: jest.fn(() => ({ + connect: mockConnect, + flushAll: mockFlushAll, + disconnect: mockDisconnect, + hmGet: mockHmGet, + get isOpen() { + return mockIsOpen; + }, + scanIterator: jest.fn(function scanIterator() { + return mockScanKeys[Symbol.iterator](); + }), + })), +})); + +jest.mock("@smithy/signature-v4", () => ({ + SignatureV4: jest.fn(() => ({ + presign: jest.fn().mockResolvedValue({ + query: { + "X-Amz-Algorithm": "AWS4-HMAC-SHA256", + "X-Amz-Credential": "test-credential", + }, + }), + })), +})); + +jest.mock("@aws-crypto/sha256-js", () => ({ + Sha256: jest.fn(), +})); + +jest.mock("@aws-sdk/credential-providers", () => ({ + fromNodeProviderChain: jest.fn(() => ({})), +})); + +const deps: ElastiCacheDeps = { + endpoint: "test-cache.example.invalid", + cacheName: "test-cache", + iamUsername: "test-user", + region: "eu-west-2", +}; + +beforeEach(() => { + jest.clearAllMocks(); + mockIsOpen = true; + mockScanKeys = []; +}); + +describe("flushElastiCache", () => { + it("connects, flushes all keys, and disconnects", async () => { + await flushElastiCache(deps); + + expect(mockConnect).toHaveBeenCalledTimes(1); + expect(mockFlushAll).toHaveBeenCalledTimes(1); + expect(mockDisconnect).toHaveBeenCalledTimes(1); + }); + + it("disconnects even when flushAll throws", async () => { + mockFlushAll.mockRejectedValueOnce(new Error("flush failed")); + + await expect(flushElastiCache(deps)).rejects.toThrow("flush failed"); + expect(mockDisconnect).toHaveBeenCalledTimes(1); + }); + + it("skips disconnect when client is not open", async () => { + mockIsOpen = false; + + await flushElastiCache(deps); + + expect(mockDisconnect).not.toHaveBeenCalled(); + }); +}); + +describe("dumpRateLimitState", () => { + it("returns empty array when no ep: keys exist", async () => { + mockScanKeys = []; + + const result = await dumpRateLimitState(deps); + + expect(result).toEqual([]); + expect(mockConnect).toHaveBeenCalledTimes(1); + expect(mockDisconnect).toHaveBeenCalledTimes(1); + }); + + it("returns state for each ep: key sorted alphabetically", async () => { + mockScanKeys = ["ep:{target-b}", "ep:{target-a}"]; + mockHmGet + .mockResolvedValueOnce([ + "1", + "500", + "0", + "400", + "20", + "15", + "5", + "3", + "1500", + ]) + .mockResolvedValueOnce([ + "0", + "1000", + "5", + "900", + "10", + "8", + "2", + "1", + "2000", + ]); + + const result = await dumpRateLimitState(deps); + + expect(result).toHaveLength(2); + expect(result[0].key).toBe("ep:{target-a}"); + expect(result[0].isOpen).toBe("1"); + expect(result[0].switchedAt).toBe("500"); + expect(result[0].bucketTokens).toBe("0"); + expect(result[0].bucketRefilledAt).toBe("400"); + expect(result[0].curAttempts).toBe("20"); + expect(result[0].prevAttempts).toBe("15"); + expect(result[0].curFailures).toBe("5"); + expect(result[0].prevFailures).toBe("3"); + expect(result[0].sampleTill).toBe("1500"); + + expect(result[1].key).toBe("ep:{target-b}"); + expect(result[1].isOpen).toBe("0"); + }); + + it("disconnects even when scan throws", async () => { + const mockClient = { + connect: mockConnect, + disconnect: mockDisconnect, + hmGet: mockHmGet, + get isOpen() { + return mockIsOpen; + }, + scanIterator: jest.fn(() => { + throw new Error("scan failed"); + }), + }; + const { createClient } = jest.requireMock("@redis/client"); + createClient.mockReturnValueOnce(mockClient); + + await expect(dumpRateLimitState(deps)).rejects.toThrow("scan failed"); + expect(mockDisconnect).toHaveBeenCalledTimes(1); + }); + + it("skips disconnect when client is not open", async () => { + mockIsOpen = false; + mockScanKeys = []; + + await dumpRateLimitState(deps); + + expect(mockDisconnect).not.toHaveBeenCalled(); + }); +}); diff --git a/lambdas/perf-runner-lambda/src/__tests__/event-factories.test.ts b/lambdas/perf-runner-lambda/src/__tests__/event-factories.test.ts new file mode 100644 index 00000000..dcecd707 --- /dev/null +++ b/lambdas/perf-runner-lambda/src/__tests__/event-factories.test.ts @@ -0,0 +1,121 @@ +import { EventTypes } from "@nhs-notify-client-callbacks/models"; +import { + createChannelStatusEvent, + createEvent, + createMessageStatusEvent, +} from "event-factories"; + +describe("createMessageStatusEvent", () => { + it("creates a valid message status CloudEvent with the given clientId and status", () => { + const event = createMessageStatusEvent("perf-client-1", "DELIVERED"); + + expect(event.specversion).toBe("1.0"); + expect(event.type).toBe(EventTypes.MESSAGE_STATUS_PUBLISHED); + expect(event.datacontenttype).toBe("application/json"); + expect(event.data.clientId).toBe("perf-client-1"); + expect(event.data.messageStatus).toBe("DELIVERED"); + expect(event.data.messageId).toBeTruthy(); + expect(event.id).toBeTruthy(); + }); + + it("assigns a unique id and messageId on each call", () => { + const a = createMessageStatusEvent("perf-client-1", "FAILED"); + const b = createMessageStatusEvent("perf-client-1", "FAILED"); + + expect(a.id).not.toBe(b.id); + expect(a.data.messageId).not.toBe(b.data.messageId); + }); + + it("prefixes messageId with force-{code}- when forcedStatusCode is set", () => { + const event = createMessageStatusEvent("perf-client-1", "DELIVERED", 500); + + expect(event.data.messageId).toMatch(/^force-500-[0-9a-f-]+$/); + }); + + it("prefixes messageId with force-{code}-until-{timestamp}- when both forced fields are set", () => { + const until = Date.now() + 60_000; + const event = createMessageStatusEvent( + "perf-client-1", + "DELIVERED", + 500, + until, + ); + + const prefix = `force-500-until-${until}-`; + expect(event.data.messageId.startsWith(prefix)).toBe(true); + }); +}); + +describe("createChannelStatusEvent", () => { + it("creates a valid channel status CloudEvent with the given clientId and status", () => { + const event = createChannelStatusEvent("perf-client-2", "DELIVERED"); + + expect(event.specversion).toBe("1.0"); + expect(event.type).toBe(EventTypes.CHANNEL_STATUS_PUBLISHED); + expect(event.datacontenttype).toBe("application/json"); + expect(event.data.clientId).toBe("perf-client-2"); + expect(event.data.channelStatus).toBe("DELIVERED"); + expect(event.data.messageId).toBeTruthy(); + expect(event.id).toBeTruthy(); + }); + + it("prefixes messageId with force-{code}- when forcedStatusCode is set", () => { + const event = createChannelStatusEvent("perf-client-2", "DELIVERED", 503); + + expect(event.data.messageId).toMatch(/^force-503-[0-9a-f-]+$/); + }); + + it("prefixes messageId with force-{code}-until-{timestamp}- when both forced fields are set", () => { + const until = Date.now() + 60_000; + const event = createChannelStatusEvent( + "perf-client-2", + "DELIVERED", + 503, + until, + ); + + const prefix = `force-503-until-${until}-`; + expect(event.data.messageId.startsWith(prefix)).toBe(true); + }); +}); + +describe("createEvent", () => { + it("delegates to createMessageStatusEvent for messageStatus factory entries", () => { + const event = createEvent({ + weight: 1, + factory: "messageStatus", + clientId: "perf-client-1", + messageStatus: "SENDING", + }); + + expect(event.type).toBe(EventTypes.MESSAGE_STATUS_PUBLISHED); + expect(event.data.clientId).toBe("perf-client-1"); + }); + + it("delegates to createChannelStatusEvent for channelStatus factory entries", () => { + const event = createEvent({ + weight: 1, + factory: "channelStatus", + clientId: "perf-client-2", + channelStatus: "FAILED", + }); + + expect(event.type).toBe(EventTypes.CHANNEL_STATUS_PUBLISHED); + expect(event.data.clientId).toBe("perf-client-2"); + }); + + it("forwards forcedStatusCode and forcedStatusCodeUntilMs from the mix entry", () => { + const until = Date.now() + 60_000; + const event = createEvent({ + weight: 1, + factory: "messageStatus", + clientId: "perf-client-1", + messageStatus: "DELIVERED", + forcedStatusCode: 500, + forcedStatusCodeUntilMs: until, + }); + + const prefix = `force-500-until-${until}-`; + expect(event.data.messageId.startsWith(prefix)).toBe(true); + }); +}); diff --git a/lambdas/perf-runner-lambda/src/__tests__/index.test.ts b/lambdas/perf-runner-lambda/src/__tests__/index.test.ts new file mode 100644 index 00000000..4c99f195 --- /dev/null +++ b/lambdas/perf-runner-lambda/src/__tests__/index.test.ts @@ -0,0 +1,234 @@ +import { handler } from "index"; +import type { PerformanceResult, Scenario } from "types"; + +import { runPerformanceTest } from "runner"; + +jest.mock("@aws-sdk/client-sqs", () => ({ + SQSClient: jest.fn(() => ({ destroy: jest.fn() })), +})); + +jest.mock("@aws-sdk/client-cloudwatch-logs", () => ({ + CloudWatchLogsClient: jest.fn(() => ({ destroy: jest.fn() })), +})); + +jest.mock("runner"); +jest.mock("@nhs-notify-client-callbacks/logger", () => ({ + Logger: jest.fn(() => ({ + info: jest.fn(), + error: jest.fn(), + })), +})); + +const mockRunPerformanceTest = runPerformanceTest as jest.MockedFunction< + typeof runPerformanceTest +>; + +const testScenario: Scenario = { + phases: [{ durationSecs: 5, targetEps: 100 }], + eventMix: [ + { + weight: 1, + factory: "messageStatus", + clientId: "perf-client-1", + messageStatus: "DELIVERED", + }, + ], + metricsIntervalSecs: 15, +}; + +const mockResult: PerformanceResult = { + testId: "test-id", + scenario: testScenario, + startedAt: "2026-04-09T10:00:00.000Z", + completedAt: "2026-04-09T10:02:00.000Z", + phases: [], + metrics: [], + deliveryMetrics: [], + circuitBreakerMetrics: [], +}; + +beforeEach(() => { + jest.clearAllMocks(); + mockRunPerformanceTest.mockResolvedValue(mockResult); + process.env.INBOUND_QUEUE_URL = "https://sqs.example.invalid/queue"; + process.env.DELIVERY_QUEUE_URL_PREFIX = + "https://sqs.example.invalid/nhs-dev-cbc-"; + process.env.TRANSFORM_FILTER_LOG_GROUP = + "/aws/lambda/nhs-dev-cb-client-transform-filter"; + process.env.DELIVERY_LOG_GROUP_PREFIX = + "/aws/lambda/nhs-dev-cbc-https-client-"; + process.env.MOCK_WEBHOOK_LOG_GROUP = "/aws/lambda/nhs-dev-cb-mock-webhook"; + process.env.ELASTICACHE_ENDPOINT = "cache.example.invalid"; + process.env.ELASTICACHE_CACHE_NAME = "test-cache"; + process.env.ELASTICACHE_IAM_USERNAME = "test-user"; + process.env.AWS_REGION = "eu-west-2"; +}); + +describe("handler", () => { + it("calls runPerformanceTest with the provided testId and scenario", async () => { + const result = await handler({ testId: "test-id", scenario: testScenario }); + + expect(result).toEqual(mockResult); + expect(mockRunPerformanceTest).toHaveBeenCalledWith( + expect.objectContaining({ + queueUrl: "https://sqs.example.invalid/queue", + deliveryQueueUrlPrefix: "https://sqs.example.invalid/nhs-dev-cbc-", + logGroupName: "/aws/lambda/nhs-dev-cb-client-transform-filter", + deliveryLogGroupPrefix: "/aws/lambda/nhs-dev-cbc-https-client-", + mockWebhookLogGroup: "/aws/lambda/nhs-dev-cb-mock-webhook", + }), + testScenario, + "test-id", + undefined, + expect.objectContaining({ + endpoint: "cache.example.invalid", + cacheName: "test-cache", + iamUsername: "test-user", + region: "eu-west-2", + }), + undefined, + undefined, + ); + }); + + it("uses a custom scenario when one is provided in the event", async () => { + const customScenario = { + ...testScenario, + phases: [{ durationSecs: 5, targetEps: 500 }], + }; + + await handler({ testId: "custom-test", scenario: customScenario }); + + expect(mockRunPerformanceTest).toHaveBeenCalledWith( + expect.anything(), + customScenario, + "custom-test", + undefined, + expect.anything(), + undefined, + undefined, + ); + }); + + it("destroys AWS clients even when runPerformanceTest throws", async () => { + const { SQSClient } = jest.requireMock("@aws-sdk/client-sqs"); + const mockDestroy = jest.fn(); + SQSClient.mockReturnValue({ destroy: mockDestroy }); + + mockRunPerformanceTest.mockRejectedValue(new Error("test failure")); + + await expect( + handler({ testId: "failing-test", scenario: testScenario }), + ).rejects.toThrow("test failure"); + expect(mockDestroy).toHaveBeenCalled(); + }); + + it("throws when INBOUND_QUEUE_URL is missing", async () => { + delete process.env.INBOUND_QUEUE_URL; + + await expect( + handler({ testId: "missing-queue-test", scenario: testScenario }), + ).rejects.toThrow( + "Missing required environment variable: INBOUND_QUEUE_URL", + ); + }); + + it("throws when TRANSFORM_FILTER_LOG_GROUP is missing", async () => { + delete process.env.TRANSFORM_FILTER_LOG_GROUP; + delete process.env.AWS_REGION; + + await expect( + handler({ testId: "missing-log-group-test", scenario: testScenario }), + ).rejects.toThrow( + "Missing required environment variable: TRANSFORM_FILTER_LOG_GROUP", + ); + }); + + it("passes undefined deliveryLogGroupPrefix when env var is not set", async () => { + delete process.env.DELIVERY_LOG_GROUP_PREFIX; + + await handler({ testId: "no-prefix-test", scenario: testScenario }); + + expect(mockRunPerformanceTest).toHaveBeenCalledWith( + expect.objectContaining({ + deliveryLogGroupPrefix: undefined, + }), + testScenario, + "no-prefix-test", + undefined, + expect.anything(), + undefined, + undefined, + ); + }); + + it("passes undefined elastiCacheDeps when ElastiCache env vars are missing", async () => { + delete process.env.ELASTICACHE_ENDPOINT; + delete process.env.ELASTICACHE_CACHE_NAME; + delete process.env.ELASTICACHE_IAM_USERNAME; + + await handler({ testId: "no-cache-test", scenario: testScenario }); + + expect(mockRunPerformanceTest).toHaveBeenCalledWith( + expect.anything(), + testScenario, + "no-cache-test", + undefined, + undefined, + undefined, + undefined, + ); + }); + + it("passes mockWebhookLogGroup from env var", async () => { + await handler({ testId: "webhook-test", scenario: testScenario }); + + expect(mockRunPerformanceTest).toHaveBeenCalledWith( + expect.objectContaining({ + mockWebhookLogGroup: "/aws/lambda/nhs-dev-cb-mock-webhook", + }), + expect.anything(), + "webhook-test", + undefined, + expect.anything(), + undefined, + undefined, + ); + }); + + it("passes cloudWatchSettlingMs when provided in the event", async () => { + await handler({ + testId: "settling-test", + scenario: testScenario, + cloudWatchSettlingMs: 5000, + }); + + expect(mockRunPerformanceTest).toHaveBeenCalledWith( + expect.anything(), + expect.anything(), + "settling-test", + undefined, + expect.anything(), + 5000, + undefined, + ); + }); + + it("passes skipPurge when provided in the event", async () => { + await handler({ + testId: "skip-purge-test", + scenario: testScenario, + skipPurge: true, + }); + + expect(mockRunPerformanceTest).toHaveBeenCalledWith( + expect.anything(), + expect.anything(), + "skip-purge-test", + undefined, + expect.anything(), + undefined, + true, + ); + }); +}); diff --git a/lambdas/perf-runner-lambda/src/__tests__/purge.test.ts b/lambdas/perf-runner-lambda/src/__tests__/purge.test.ts new file mode 100644 index 00000000..3f70037b --- /dev/null +++ b/lambdas/perf-runner-lambda/src/__tests__/purge.test.ts @@ -0,0 +1,128 @@ +import type { SQSClient } from "@aws-sdk/client-sqs"; +import { deriveQueueUrls, purgeQueues } from "purge"; +import type { Scenario } from "types"; + +const scenario: Scenario = { + phases: [{ durationSecs: 1, targetEps: 10 }], + eventMix: [ + { + weight: 1, + factory: "messageStatus", + clientId: "perf-client-1", + messageStatus: "DELIVERED", + }, + { + weight: 1, + factory: "channelStatus", + clientId: "perf-client-2", + channelStatus: "DELIVERED", + }, + ], + metricsIntervalSecs: 5, +}; + +const inboundQueueUrl = + "https://sqs.eu-west-2.amazonaws.com/123456789/nhs-dev-cb-inbound-event-queue"; + +const deliveryQueueUrlPrefix = + "https://sqs.eu-west-2.amazonaws.com/123456789/nhs-dev-cbc-"; + +describe("deriveQueueUrls", () => { + it("derives all queue URLs from the inbound queue URL and scenario", () => { + const urls = deriveQueueUrls( + inboundQueueUrl, + scenario, + deliveryQueueUrlPrefix, + ); + + expect(urls).toEqual([ + "https://sqs.eu-west-2.amazonaws.com/123456789/nhs-dev-cb-inbound-event-queue", + "https://sqs.eu-west-2.amazonaws.com/123456789/nhs-dev-cb-inbound-event-dlq", + "https://sqs.eu-west-2.amazonaws.com/123456789/nhs-dev-cbc-perf-client-1-delivery-queue", + "https://sqs.eu-west-2.amazonaws.com/123456789/nhs-dev-cbc-perf-client-1-delivery-dlq-queue", + "https://sqs.eu-west-2.amazonaws.com/123456789/nhs-dev-cbc-perf-client-2-delivery-queue", + "https://sqs.eu-west-2.amazonaws.com/123456789/nhs-dev-cbc-perf-client-2-delivery-dlq-queue", + ]); + }); + + it("falls back to inbound base URL when no delivery prefix provided", () => { + const urls = deriveQueueUrls(inboundQueueUrl, scenario); + + expect(urls).toEqual([ + "https://sqs.eu-west-2.amazonaws.com/123456789/nhs-dev-cb-inbound-event-queue", + "https://sqs.eu-west-2.amazonaws.com/123456789/nhs-dev-cb-inbound-event-dlq", + "https://sqs.eu-west-2.amazonaws.com/123456789/nhs-dev-cb-perf-client-1-delivery-queue", + "https://sqs.eu-west-2.amazonaws.com/123456789/nhs-dev-cb-perf-client-1-delivery-dlq-queue", + "https://sqs.eu-west-2.amazonaws.com/123456789/nhs-dev-cb-perf-client-2-delivery-queue", + "https://sqs.eu-west-2.amazonaws.com/123456789/nhs-dev-cb-perf-client-2-delivery-dlq-queue", + ]); + }); + + it("deduplicates client IDs that appear multiple times in eventMix", () => { + const duplicateScenario: Scenario = { + ...scenario, + eventMix: [ + { + weight: 1, + factory: "messageStatus", + clientId: "perf-client-1", + messageStatus: "DELIVERED", + }, + { + weight: 1, + factory: "channelStatus", + clientId: "perf-client-1", + channelStatus: "DELIVERED", + }, + ], + }; + + const urls = deriveQueueUrls( + inboundQueueUrl, + duplicateScenario, + deliveryQueueUrlPrefix, + ); + + expect(urls).toEqual([ + "https://sqs.eu-west-2.amazonaws.com/123456789/nhs-dev-cb-inbound-event-queue", + "https://sqs.eu-west-2.amazonaws.com/123456789/nhs-dev-cb-inbound-event-dlq", + "https://sqs.eu-west-2.amazonaws.com/123456789/nhs-dev-cbc-perf-client-1-delivery-queue", + "https://sqs.eu-west-2.amazonaws.com/123456789/nhs-dev-cbc-perf-client-1-delivery-dlq-queue", + ]); + }); +}); + +describe("purgeQueues", () => { + const mockSend = jest.fn().mockResolvedValue({}); + const mockSqsClient = { send: mockSend } as unknown as SQSClient; + + beforeEach(() => { + jest.clearAllMocks(); + mockSend.mockResolvedValue({}); + }); + + it("sends a PurgeQueueCommand for each queue URL", async () => { + const urls = [ + "https://sqs.example.invalid/queue-a", + "https://sqs.example.invalid/queue-b", + ]; + + await purgeQueues(mockSqsClient, urls); + + expect(mockSend).toHaveBeenCalledTimes(2); + }); + + it("throws when a purge fails", async () => { + mockSend.mockRejectedValueOnce(new Error("Access denied")); + + await expect( + purgeQueues(mockSqsClient, ["https://sqs.example.invalid/queue"]), + ).rejects.toThrow("Access denied"); + }); + + it("handles an empty queue URL list without sending any commands", async () => { + await purgeQueues(mockSqsClient, []); + + expect(mockSend).not.toHaveBeenCalled(); + }); +}); diff --git a/lambdas/perf-runner-lambda/src/__tests__/runner.test.ts b/lambdas/perf-runner-lambda/src/__tests__/runner.test.ts new file mode 100644 index 00000000..89933812 --- /dev/null +++ b/lambdas/perf-runner-lambda/src/__tests__/runner.test.ts @@ -0,0 +1,761 @@ +import type { SQSClient } from "@aws-sdk/client-sqs"; +import type { CloudWatchLogsClient } from "@aws-sdk/client-cloudwatch-logs"; +import type { + CircuitBreakerSnapshot, + DeliveryMetricsSnapshot, + MetricsSnapshot, + PhaseResult, + RunnerDeps, + Scenario, +} from "types"; +import { defaultSleep, runPerformanceTest } from "runner"; + +import { generatePhaseLoad } from "sqs"; +import { deriveQueueUrls, purgeQueues } from "purge"; +import { getQueueDepths } from "sqs-stats"; +import { dumpRateLimitState, flushElastiCache } from "elasticache"; +import { verifyMockWebhook } from "webhook-verify"; +import { + queryCircuitBreakerSnapshot, + queryDeliveryMetricsSnapshot, + queryMetricsSnapshot, + queryPerClientRateTimeline, +} from "cloudwatch"; + +jest.mock("sqs"); +jest.mock("cloudwatch"); +jest.mock("purge"); +jest.mock("elasticache"); +jest.mock("webhook-verify"); +jest.mock("sqs-stats"); + +const mockGeneratePhaseLoad = jest.mocked(generatePhaseLoad); +const mockQueryMetricsSnapshot = jest.mocked(queryMetricsSnapshot); +const mockQueryDeliveryMetricsSnapshot = jest.mocked( + queryDeliveryMetricsSnapshot, +); +const mockQueryCircuitBreakerSnapshot = jest.mocked( + queryCircuitBreakerSnapshot, +); +const mockQueryPerClientRateTimeline = jest.mocked(queryPerClientRateTimeline); +const mockDeriveQueueUrls = jest.mocked(deriveQueueUrls); +const mockPurgeQueues = jest.mocked(purgeQueues); +const mockFlushElastiCache = jest.mocked(flushElastiCache); +const mockDumpRateLimitState = jest.mocked(dumpRateLimitState); +const mockVerifyMockWebhook = jest.mocked(verifyMockWebhook); +const mockGetQueueDepths = jest.mocked(getQueueDepths); + +const immediateSleep = jest.fn().mockResolvedValue(undefined); + +const mockPhaseResult: PhaseResult = { + targetEps: 1000, + achievedEps: 980, + sent: 1000, + durationMs: 1020, +}; + +const mockSnapshot: MetricsSnapshot = { + snapshotAt: Date.now(), + p50Ms: 30, + p95Ms: 80, + p99Ms: 150, + count: 100, +}; + +const mockDeliverySnapshot: DeliveryMetricsSnapshot = { + snapshotAt: Date.now(), + deliveryCount: 50, + p50Ms: 120, + p95Ms: 300, + p99Ms: 500, +}; + +const mockCbSnapshot: CircuitBreakerSnapshot = { + snapshotAt: Date.now(), + intervalStartSec: 0, + intervalEndSec: 60, + circuitOpenEvents: 1, + circuitCloseEvents: 0, + admissionDeniedCircuitOpen: 5, + admissionDeniedRateLimited: 3, + deliveryAttempts: 100, + deliverySuccesses: 92, + deliveryFailures: 5, + deliveryRateLimited: 3, +}; + +const scenario: Scenario = { + phases: [{ durationSecs: 1, targetEps: 1000 }], + eventMix: [ + { + weight: 1, + factory: "messageStatus", + clientId: "perf-client-1", + messageStatus: "DELIVERED", + }, + ], + metricsIntervalSecs: 1, +}; + +const deps: RunnerDeps = { + sqsClient: {} as SQSClient, + cloudWatchClient: {} as CloudWatchLogsClient, + queueUrl: "https://sqs.example.invalid/queue", + deliveryQueueUrlPrefix: "https://sqs.example.invalid/nhs-dev-cbc-", + logGroupName: "/aws/lambda/nhs-dev-cb-client-transform-filter", + deliveryLogGroupPrefix: "/aws/lambda/nhs-dev-cbc-https-client-", +}; + +beforeEach(() => { + jest.clearAllMocks(); + mockGeneratePhaseLoad.mockResolvedValue(mockPhaseResult); + mockQueryDeliveryMetricsSnapshot.mockResolvedValue(null); + mockQueryCircuitBreakerSnapshot.mockResolvedValue(null); + mockQueryPerClientRateTimeline.mockResolvedValue([]); + mockDeriveQueueUrls.mockReturnValue([ + "https://sqs.example.invalid/inbound-event-queue", + ]); + mockPurgeQueues.mockResolvedValue(undefined); + mockFlushElastiCache.mockResolvedValue(undefined); + mockDumpRateLimitState.mockResolvedValue([]); + mockVerifyMockWebhook.mockResolvedValue({ + receivedCallbacks: 0, + verified: false, + }); + mockGetQueueDepths.mockResolvedValue({ + timestampMs: Date.now(), + queues: [ + { + queueUrl: "https://sqs.example.invalid/inbound-event-queue", + visible: 100, + notVisible: 10, + }, + ], + }); + immediateSleep.mockResolvedValue(undefined); +}); + +describe("runPerformanceTest", () => { + it("returns a PerformanceResult with phase results and snapshots from polling and final query", async () => { + mockQueryMetricsSnapshot.mockResolvedValue(mockSnapshot); + mockQueryDeliveryMetricsSnapshot.mockResolvedValue(mockDeliverySnapshot); + mockQueryCircuitBreakerSnapshot.mockResolvedValue(mockCbSnapshot); + + const result = await runPerformanceTest( + deps, + scenario, + "test-id-1", + immediateSleep, + ); + + expect(result.testId).toBe("test-id-1"); + expect(result.scenario).toBe(scenario); + expect(result.phases).toHaveLength(1); + expect(result.phases[0]).toEqual(mockPhaseResult); + expect(result.metrics).toHaveLength(2); // one mid-test, one final + expect(result.deliveryMetrics).toHaveLength(2); // one mid-test, one final + expect(result.circuitBreakerMetrics).toHaveLength(2); // one mid-test, one final + expect(result.startedAt).toBeTruthy(); + expect(result.completedAt).toBeTruthy(); + }); + + it("excludes null snapshots from the metrics array", async () => { + mockQueryMetricsSnapshot + .mockResolvedValueOnce(null) // mid-test poll returns null + .mockResolvedValueOnce(mockSnapshot); // final query returns snapshot + + const result = await runPerformanceTest( + deps, + scenario, + "test-id-2", + immediateSleep, + ); + + expect(result.metrics).toHaveLength(1); + expect(result.metrics[0]).toEqual(mockSnapshot); + expect(result.deliveryMetrics).toHaveLength(0); + expect(result.circuitBreakerMetrics).toHaveLength(0); + }); + + it("produces an empty metrics array when all queries return null", async () => { + mockQueryMetricsSnapshot.mockResolvedValue(null); + + const result = await runPerformanceTest( + deps, + scenario, + "test-id-3", + immediateSleep, + ); + + expect(result.metrics).toHaveLength(0); + expect(result.deliveryMetrics).toHaveLength(0); + expect(result.circuitBreakerMetrics).toHaveLength(0); + }); + + it("runs all phases and collects each result", async () => { + const multiPhaseScenario: Scenario = { + ...scenario, + phases: [ + { durationSecs: 1, targetEps: 500 }, + { durationSecs: 1, targetEps: 1000 }, + ], + }; + + const phase1Result = { ...mockPhaseResult, targetEps: 500 }; + const phase2Result = { ...mockPhaseResult, targetEps: 1000 }; + + mockGeneratePhaseLoad + .mockResolvedValueOnce(phase1Result) + .mockResolvedValueOnce(phase2Result); + mockQueryMetricsSnapshot.mockResolvedValue(null); + + const result = await runPerformanceTest( + deps, + multiPhaseScenario, + "test-id-4", + immediateSleep, + ); + + expect(result.phases).toHaveLength(2); + expect(result.phases[0]).toEqual(phase1Result); + expect(result.phases[1]).toEqual(phase2Result); + }); + + it("collects delivery metrics across multiple poll iterations", async () => { + let resolvePhase!: (value: PhaseResult) => void; + mockGeneratePhaseLoad.mockImplementation( + () => + new Promise((r) => { + resolvePhase = r; + }), + ); + mockQueryMetricsSnapshot.mockResolvedValue(mockSnapshot); + mockQueryDeliveryMetricsSnapshot.mockResolvedValue(mockDeliverySnapshot); + + let sleepCount = 0; + const controlledSleep = jest.fn(async () => { + sleepCount += 1; + if (sleepCount >= 3) { + resolvePhase(mockPhaseResult); + } + }); + + const result = await runPerformanceTest( + deps, + scenario, + "test-id-poll", + controlledSleep, + ); + + expect(result.deliveryMetrics.length).toBeGreaterThanOrEqual(1); + }); + + it("throws when scenario.eventMix is empty", async () => { + const emptyMixScenario: Scenario = { ...scenario, eventMix: [] }; + + await expect( + runPerformanceTest( + deps, + emptyMixScenario, + "empty-mix-test", + immediateSleep, + ), + ).rejects.toThrow("scenario.eventMix must contain at least one entry"); + }); + + it("throws when a phase has durationSecs of zero", async () => { + const badScenario: Scenario = { + ...scenario, + phases: [{ durationSecs: 0, targetEps: 1000 }], + }; + + await expect( + runPerformanceTest( + deps, + badScenario, + "zero-duration-test", + immediateSleep, + ), + ).rejects.toThrow("scenario.phases[0].durationSecs must be greater than 0"); + }); + + it("throws when a phase has targetEps of zero", async () => { + const badScenario: Scenario = { + ...scenario, + phases: [{ durationSecs: 1, targetEps: 0 }], + }; + + await expect( + runPerformanceTest(deps, badScenario, "zero-eps-test", immediateSleep), + ).rejects.toThrow("scenario.phases[0].targetEps must be greater than 0"); + }); + + it("throws when a later phase has an invalid value", async () => { + const badScenario: Scenario = { + ...scenario, + phases: [ + { durationSecs: 1, targetEps: 1000 }, + { durationSecs: 1, targetEps: 0 }, + ], + }; + + await expect( + runPerformanceTest(deps, badScenario, "later-phase-test", immediateSleep), + ).rejects.toThrow("scenario.phases[1].targetEps must be greater than 0"); + }); + + it("calls generatePhaseLoad with the correct phase and deps", async () => { + mockQueryMetricsSnapshot.mockResolvedValue(null); + + await runPerformanceTest(deps, scenario, "test-id-5", immediateSleep); + + expect(mockGeneratePhaseLoad).toHaveBeenCalledWith( + deps.sqsClient, + deps.queueUrl, + scenario.phases[0], + scenario.eventMix, + ); + }); + + it("skips delivery metrics when deliveryLogGroupPrefix is undefined", async () => { + const depsWithoutPrefix: RunnerDeps = { + ...deps, + deliveryLogGroupPrefix: undefined, + }; + mockQueryMetricsSnapshot.mockResolvedValue(mockSnapshot); + + const result = await runPerformanceTest( + depsWithoutPrefix, + scenario, + "test-id-6", + immediateSleep, + ); + + expect(mockQueryDeliveryMetricsSnapshot).not.toHaveBeenCalled(); + expect(mockQueryCircuitBreakerSnapshot).not.toHaveBeenCalled(); + expect(result.deliveryMetrics).toHaveLength(0); + expect(result.circuitBreakerMetrics).toHaveLength(0); + }); + + it("builds delivery log group names from prefix and event mix client IDs", async () => { + mockQueryMetricsSnapshot.mockResolvedValue(null); + mockQueryDeliveryMetricsSnapshot.mockResolvedValue(null); + + const multiClientScenario: Scenario = { + ...scenario, + eventMix: [ + { + weight: 1, + factory: "messageStatus", + clientId: "perf-client-1", + messageStatus: "DELIVERED", + }, + { + weight: 1, + factory: "channelStatus", + clientId: "perf-client-2", + channelStatus: "DELIVERED", + }, + ], + }; + + await runPerformanceTest( + deps, + multiClientScenario, + "test-id-7", + immediateSleep, + ); + + expect(mockQueryDeliveryMetricsSnapshot).toHaveBeenCalledWith( + deps.cloudWatchClient, + expect.arrayContaining([ + "/aws/lambda/nhs-dev-cbc-https-client-perf-client-1", + "/aws/lambda/nhs-dev-cbc-https-client-perf-client-2", + ]), + expect.any(Number), + expect.any(Number), + ); + }); + + it("collects circuit breaker metrics when deliveryLogGroupPrefix is set", async () => { + mockQueryMetricsSnapshot.mockResolvedValue(mockSnapshot); + mockQueryDeliveryMetricsSnapshot.mockResolvedValue(mockDeliverySnapshot); + mockQueryCircuitBreakerSnapshot.mockResolvedValue(mockCbSnapshot); + + const result = await runPerformanceTest( + deps, + scenario, + "test-cb-1", + immediateSleep, + ); + + expect(result.circuitBreakerMetrics.length).toBeGreaterThanOrEqual(1); + expect(mockQueryCircuitBreakerSnapshot).toHaveBeenCalled(); + }); + + it("returns empty circuitBreakerMetrics when CB queries return null", async () => { + mockQueryMetricsSnapshot.mockResolvedValue(mockSnapshot); + mockQueryDeliveryMetricsSnapshot.mockResolvedValue(mockDeliverySnapshot); + mockQueryCircuitBreakerSnapshot.mockResolvedValue(null); + + const result = await runPerformanceTest( + deps, + scenario, + "test-cb-null", + immediateSleep, + ); + + expect(result.circuitBreakerMetrics).toHaveLength(0); + }); + + it("uses per-interval windowing for circuit breaker snapshots", async () => { + mockQueryMetricsSnapshot.mockResolvedValue(mockSnapshot); + mockQueryDeliveryMetricsSnapshot.mockResolvedValue(mockDeliverySnapshot); + mockQueryCircuitBreakerSnapshot.mockResolvedValue(mockCbSnapshot); + + let resolvePhase!: (value: PhaseResult) => void; + mockGeneratePhaseLoad.mockImplementation( + () => + new Promise((r) => { + resolvePhase = r; + }), + ); + + let sleepCount = 0; + const controlledSleep = jest.fn(async () => { + sleepCount += 1; + if (sleepCount >= 3) { + resolvePhase(mockPhaseResult); + } + }); + + await runPerformanceTest( + deps, + scenario, + "test-cb-interval", + controlledSleep, + ); + + const cbCalls = mockQueryCircuitBreakerSnapshot.mock.calls; + expect(cbCalls.length).toBeGreaterThanOrEqual(2); + const firstCallEndSec = cbCalls[0][3]; + const secondCallStartSec = cbCalls[1][2]; + expect(secondCallStartSec).toBe(firstCallEndSec); + }); + + it("collects per-client rate timelines when deliveryLogGroupPrefix is set", async () => { + mockQueryMetricsSnapshot.mockResolvedValue(mockSnapshot); + mockQueryDeliveryMetricsSnapshot.mockResolvedValue(mockDeliverySnapshot); + mockQueryPerClientRateTimeline.mockResolvedValue([ + { timestampSec: 1000, deliveryAttempts: 10 }, + ]); + + const result = await runPerformanceTest( + deps, + scenario, + "test-pcr-1", + immediateSleep, + ); + + expect(result.perClientRateTimelines).toHaveLength(1); + expect(result.perClientRateTimelines![0].clientId).toBe("perf-client-1"); + expect(result.perClientRateTimelines![0].entries).toHaveLength(1); + }); + + it("queries each client log group individually for rate timelines", async () => { + mockQueryMetricsSnapshot.mockResolvedValue(null); + mockQueryPerClientRateTimeline.mockResolvedValue([ + { timestampSec: 1000, deliveryAttempts: 5 }, + ]); + + const multiClientScenario: Scenario = { + ...scenario, + eventMix: [ + { + weight: 1, + factory: "messageStatus", + clientId: "perf-client-1", + messageStatus: "DELIVERED", + }, + { + weight: 1, + factory: "channelStatus", + clientId: "perf-client-2", + channelStatus: "DELIVERED", + }, + ], + }; + + const result = await runPerformanceTest( + deps, + multiClientScenario, + "test-pcr-multi", + immediateSleep, + ); + + expect(mockQueryPerClientRateTimeline).toHaveBeenCalledTimes(2); + expect(mockQueryPerClientRateTimeline).toHaveBeenCalledWith( + deps.cloudWatchClient, + "/aws/lambda/nhs-dev-cbc-https-client-perf-client-1", + expect.any(Number), + expect.any(Number), + ); + expect(mockQueryPerClientRateTimeline).toHaveBeenCalledWith( + deps.cloudWatchClient, + "/aws/lambda/nhs-dev-cbc-https-client-perf-client-2", + expect.any(Number), + expect.any(Number), + ); + expect(result.perClientRateTimelines).toHaveLength(2); + }); + + it("excludes clients with empty rate timelines", async () => { + mockQueryMetricsSnapshot.mockResolvedValue(null); + mockQueryPerClientRateTimeline + .mockResolvedValueOnce([{ timestampSec: 1000, deliveryAttempts: 5 }]) + .mockResolvedValueOnce([]); + + const multiClientScenario: Scenario = { + ...scenario, + eventMix: [ + { + weight: 1, + factory: "messageStatus", + clientId: "perf-client-1", + messageStatus: "DELIVERED", + }, + { + weight: 1, + factory: "channelStatus", + clientId: "perf-client-2", + channelStatus: "DELIVERED", + }, + ], + }; + + const result = await runPerformanceTest( + deps, + multiClientScenario, + "test-pcr-filter", + immediateSleep, + ); + + expect(result.perClientRateTimelines).toHaveLength(1); + expect(result.perClientRateTimelines![0].clientId).toBe("perf-client-1"); + }); + + it("skips per-client rate timelines when deliveryLogGroupPrefix is undefined", async () => { + const depsWithoutPrefix: RunnerDeps = { + ...deps, + deliveryLogGroupPrefix: undefined, + }; + mockQueryMetricsSnapshot.mockResolvedValue(mockSnapshot); + + const result = await runPerformanceTest( + depsWithoutPrefix, + scenario, + "test-pcr-skip", + immediateSleep, + ); + + expect(mockQueryPerClientRateTimeline).not.toHaveBeenCalled(); + expect(result.perClientRateTimelines).toHaveLength(0); + }); + + it("purges queues before and after the test run", async () => { + mockQueryMetricsSnapshot.mockResolvedValue(null); + + await runPerformanceTest(deps, scenario, "test-purge", immediateSleep); + + expect(mockDeriveQueueUrls).toHaveBeenCalledWith( + deps.queueUrl, + scenario, + deps.deliveryQueueUrlPrefix, + ); + expect(mockPurgeQueues).toHaveBeenCalledTimes(2); + }); + + it("skips both purges when skipPurge is true", async () => { + mockQueryMetricsSnapshot.mockResolvedValue(null); + + await runPerformanceTest( + deps, + scenario, + "test-skip-purge", + immediateSleep, + undefined, + undefined, + true, + ); + + expect(mockPurgeQueues).not.toHaveBeenCalled(); + }); + + it("flushes ElastiCache before and after when deps are provided", async () => { + mockQueryMetricsSnapshot.mockResolvedValue(null); + const elastiCacheDeps = { + endpoint: "cache.example.invalid", + cacheName: "test-cache", + iamUsername: "test-user", + region: "eu-west-2", + }; + + await runPerformanceTest( + deps, + scenario, + "test-flush", + immediateSleep, + elastiCacheDeps, + ); + + expect(mockFlushElastiCache).toHaveBeenCalledTimes(1); + expect(mockFlushElastiCache).toHaveBeenCalledWith(elastiCacheDeps); + }); + + it("dumps rate limit state before and after when elasticache deps are provided", async () => { + mockQueryMetricsSnapshot.mockResolvedValue(null); + const elastiCacheDeps = { + endpoint: "cache.example.invalid", + cacheName: "test-cache", + iamUsername: "test-user", + region: "eu-west-2", + }; + const mockState = [ + { + key: "ep:{target-1}", + isOpen: "0", + switchedAt: "0", + bucketTokens: "10", + bucketRefilledAt: "1000", + curAttempts: "5", + prevAttempts: "3", + curFailures: "0", + prevFailures: "0", + sampleTill: "2000", + }, + ]; + mockDumpRateLimitState.mockResolvedValue(mockState); + + const result = await runPerformanceTest( + deps, + scenario, + "test-dump", + immediateSleep, + elastiCacheDeps, + ); + + expect(mockDumpRateLimitState).toHaveBeenCalledTimes(2); + expect(mockDumpRateLimitState).toHaveBeenCalledWith(elastiCacheDeps); + expect(result.rateLimitStateBefore).toEqual(mockState); + expect(result.rateLimitStateAfter).toEqual(mockState); + }); + + it("omits rate limit state when elasticache deps are not provided", async () => { + mockQueryMetricsSnapshot.mockResolvedValue(null); + + const result = await runPerformanceTest( + deps, + scenario, + "test-no-dump", + immediateSleep, + ); + + expect(mockDumpRateLimitState).not.toHaveBeenCalled(); + expect(result.rateLimitStateBefore).toBeUndefined(); + expect(result.rateLimitStateAfter).toBeUndefined(); + }); + + it("skips ElastiCache flush when deps are not provided", async () => { + mockQueryMetricsSnapshot.mockResolvedValue(null); + + await runPerformanceTest(deps, scenario, "test-no-flush", immediateSleep); + + expect(mockFlushElastiCache).not.toHaveBeenCalled(); + }); + + it("verifies mock webhook when log group is configured", async () => { + const depsWithWebhook: RunnerDeps = { + ...deps, + mockWebhookLogGroup: "/aws/lambda/test-mock-webhook", + }; + mockQueryMetricsSnapshot.mockResolvedValue(null); + mockVerifyMockWebhook.mockResolvedValue({ + receivedCallbacks: 25, + verified: true, + }); + + const result = await runPerformanceTest( + depsWithWebhook, + scenario, + "test-webhook", + immediateSleep, + ); + + expect(mockVerifyMockWebhook).toHaveBeenCalledWith( + depsWithWebhook.cloudWatchClient, + "/aws/lambda/test-mock-webhook", + expect.any(Number), + expect.any(Number), + ); + expect(result.webhookVerification).toEqual({ + receivedCallbacks: 25, + verified: true, + }); + }); + + it("omits webhook verification when log group is not configured", async () => { + mockQueryMetricsSnapshot.mockResolvedValue(null); + + const result = await runPerformanceTest( + deps, + scenario, + "test-no-webhook", + immediateSleep, + ); + + expect(mockVerifyMockWebhook).not.toHaveBeenCalled(); + expect(result.webhookVerification).toBeUndefined(); + }); + + it("samples queue depths during polling and at final snapshot", async () => { + mockQueryMetricsSnapshot.mockResolvedValue(null); + + await runPerformanceTest( + deps, + scenario, + "test-queue-depths", + immediateSleep, + ); + + expect(mockGetQueueDepths).toHaveBeenCalledTimes(2); // one mid-test, one final + expect(mockGetQueueDepths).toHaveBeenCalledWith(deps.sqsClient, [ + "https://sqs.example.invalid/inbound-event-queue", + ]); + }); + + it("uses the provided cloudWatchSettlingMs instead of the default", async () => { + mockQueryMetricsSnapshot.mockResolvedValue(null); + + await runPerformanceTest( + deps, + scenario, + "test-settling", + immediateSleep, + undefined, + 5000, + ); + + expect(immediateSleep).toHaveBeenCalledWith(5000); + }); +}); + +describe("defaultSleep", () => { + beforeEach(() => jest.useFakeTimers()); + afterEach(() => jest.useRealTimers()); + + it("resolves after the specified delay", async () => { + const promise = defaultSleep(500); + await jest.advanceTimersByTimeAsync(500); + await expect(promise).resolves.toBeUndefined(); + }); +}); diff --git a/lambdas/perf-runner-lambda/src/__tests__/sqs-stats.test.ts b/lambdas/perf-runner-lambda/src/__tests__/sqs-stats.test.ts new file mode 100644 index 00000000..8d2900b8 --- /dev/null +++ b/lambdas/perf-runner-lambda/src/__tests__/sqs-stats.test.ts @@ -0,0 +1,75 @@ +import type { SQSClient } from "@aws-sdk/client-sqs"; +import { getQueueDepths } from "sqs-stats"; + +describe("getQueueDepths", () => { + const mockSend = jest.fn(); + const mockSqsClient = { send: mockSend } as unknown as SQSClient; + + beforeEach(() => { + jest.clearAllMocks(); + }); + + it("returns visible and notVisible counts for each queue URL", async () => { + mockSend + .mockResolvedValueOnce({ + Attributes: { + ApproximateNumberOfMessages: "42", + ApproximateNumberOfMessagesNotVisible: "8", + }, + }) + .mockResolvedValueOnce({ + Attributes: { + ApproximateNumberOfMessages: "10", + ApproximateNumberOfMessagesNotVisible: "2", + }, + }); + + const result = await getQueueDepths(mockSqsClient, [ + "https://sqs.example.invalid/queue-a", + "https://sqs.example.invalid/queue-b", + ]); + + expect(result.queues).toHaveLength(2); + expect(result.queues[0]).toEqual({ + queueUrl: "https://sqs.example.invalid/queue-a", + visible: 42, + notVisible: 8, + }); + expect(result.queues[1]).toEqual({ + queueUrl: "https://sqs.example.invalid/queue-b", + visible: 10, + notVisible: 2, + }); + expect(result.timestampMs).toBeGreaterThan(0); + }); + + it("defaults to 0 when attributes are missing", async () => { + mockSend.mockResolvedValueOnce({ Attributes: undefined }); + + const result = await getQueueDepths(mockSqsClient, [ + "https://sqs.example.invalid/queue-a", + ]); + + expect(result.queues[0].visible).toBe(0); + expect(result.queues[0].notVisible).toBe(0); + }); + + it("sends GetQueueAttributesCommand with correct attributes requested", async () => { + mockSend.mockResolvedValueOnce({ Attributes: {} }); + + await getQueueDepths(mockSqsClient, [ + "https://sqs.example.invalid/queue-a", + ]); + + const command = mockSend.mock.calls[0][0] as { + input: { QueueUrl: string; AttributeNames: string[] }; + }; + expect(command.input.QueueUrl).toBe("https://sqs.example.invalid/queue-a"); + expect(command.input.AttributeNames).toContain( + "ApproximateNumberOfMessages", + ); + expect(command.input.AttributeNames).toContain( + "ApproximateNumberOfMessagesNotVisible", + ); + }); +}); diff --git a/lambdas/perf-runner-lambda/src/__tests__/sqs.test.ts b/lambdas/perf-runner-lambda/src/__tests__/sqs.test.ts new file mode 100644 index 00000000..63ab41df --- /dev/null +++ b/lambdas/perf-runner-lambda/src/__tests__/sqs.test.ts @@ -0,0 +1,141 @@ +import type { SQSClient } from "@aws-sdk/client-sqs"; +import type { EventMixEntry, Phase } from "types"; +import { generatePhaseLoad, selectWeighted, sendSqsBatch } from "sqs"; + +jest.mock("event-factories", () => ({ + createEvent: jest.fn(() => ({ + specversion: "1.0", + id: "mock-event-id", + type: "mock.type", + data: {}, + })), +})); + +const mockSqsClient = { + send: jest.fn(), +} as unknown as jest.Mocked; + +beforeEach(() => { + mockSqsClient.send.mockResolvedValue({} as never); +}); + +describe("selectWeighted", () => { + it("returns the only entry when there is one", () => { + const entries = [{ weight: 1, value: "a" }]; + const result = selectWeighted(entries); + expect(result).toBe(entries[0]); + }); + + it("distributes selections according to weight over many draws", () => { + const entries = [ + { weight: 9, label: "heavy" }, + { weight: 1, label: "light" }, + ]; + + const counts = { heavy: 0, light: 0 }; + for (let i = 0; i < 1000; i += 1) { + const selected = selectWeighted(entries); + counts[selected.label as keyof typeof counts] += 1; + } + + expect(counts.heavy).toBeGreaterThan(counts.light); + }); + + it("returns the last entry via fallback when no earlier entry matches", () => { + // With Math.random = 0.5, remaining = 0.5 * 10 = 5. + // First entry has weight 1; 5 - 1 = 4 > 0, so loop skips it. + // Fallback returns the last entry. + jest.spyOn(Math, "random").mockReturnValue(0.5); + const entries = [ + { weight: 1, label: "light" }, + { weight: 9, label: "heavy" }, + ]; + + const result = selectWeighted(entries); + expect(result.label).toBe("heavy"); + jest.restoreAllMocks(); + }); +}); + +describe("sendSqsBatch", () => { + it("sends a SendMessageBatchCommand with serialised event bodies", async () => { + const events = [ + { specversion: "1.0", id: "a", type: "t", data: {} }, + { specversion: "1.0", id: "b", type: "t", data: {} }, + ] as never[]; + + await sendSqsBatch( + mockSqsClient, + "https://sqs.example.invalid/queue", + events, + ); + + expect(mockSqsClient.send).toHaveBeenCalledTimes(1); + const command = mockSqsClient.send.mock.calls[0][0] as { + input: { + QueueUrl: string; + Entries: { Id: string; MessageBody: string }[]; + }; + }; + expect(command.input.QueueUrl).toBe("https://sqs.example.invalid/queue"); + expect(command.input.Entries).toHaveLength(2); + expect(command.input.Entries[0].Id).toBe("0"); + expect(JSON.parse(command.input.Entries[0].MessageBody)).toMatchObject({ + id: "a", + }); + }); +}); + +describe("generatePhaseLoad", () => { + it("returns a PhaseResult with sent count and timing", async () => { + const phase: Phase = { durationSecs: 1, targetEps: 10 }; + const eventMix: EventMixEntry[] = [ + { + weight: 1, + factory: "messageStatus", + clientId: "perf-client-1", + messageStatus: "DELIVERED", + }, + ]; + + const result = await generatePhaseLoad( + mockSqsClient, + "https://sqs.example.invalid/queue", + phase, + eventMix, + ); + + expect(result.targetEps).toBe(10); + expect(result.sent).toBeGreaterThan(0); + expect(result.durationMs).toBeGreaterThanOrEqual(0); + expect(result.achievedEps).toBeGreaterThan(0); + expect(mockSqsClient.send).toHaveBeenCalled(); + }); + + it("throttles between seconds when the wave completes early", async () => { + jest.useFakeTimers(); + + const phase: Phase = { durationSecs: 2, targetEps: 10 }; + const eventMix: EventMixEntry[] = [ + { + weight: 1, + factory: "messageStatus", + clientId: "perf-client-1", + messageStatus: "DELIVERED", + }, + ]; + + const resultPromise = generatePhaseLoad( + mockSqsClient, + "https://sqs.example.invalid/queue", + phase, + eventMix, + ); + + await jest.runAllTimersAsync(); + const result = await resultPromise; + + expect(result.sent).toBeGreaterThan(0); + jest.useRealTimers(); + }); +}); diff --git a/lambdas/perf-runner-lambda/src/__tests__/webhook-verify.test.ts b/lambdas/perf-runner-lambda/src/__tests__/webhook-verify.test.ts new file mode 100644 index 00000000..72c49870 --- /dev/null +++ b/lambdas/perf-runner-lambda/src/__tests__/webhook-verify.test.ts @@ -0,0 +1,173 @@ +import type { CloudWatchLogsClient } from "@aws-sdk/client-cloudwatch-logs"; +import { verifyMockWebhook } from "webhook-verify"; + +const mockSend = jest.fn(); +const mockClient = { send: mockSend } as unknown as CloudWatchLogsClient; + +beforeEach(() => { + jest.clearAllMocks(); +}); + +describe("verifyMockWebhook", () => { + it("returns verified=true when callbacks are found", async () => { + mockSend.mockResolvedValueOnce({ queryId: "q-1" }).mockResolvedValueOnce({ + status: "Complete", + results: [[{ field: "callbackCount", value: "42" }]], + }); + + const result = await verifyMockWebhook( + mockClient, + "/aws/lambda/test-mock-webhook", + 1000, + 2000, + ); + + expect(result).toEqual({ receivedCallbacks: 42, verified: true }); + }); + + it("returns verified=false when no callbacks are found", async () => { + mockSend.mockResolvedValueOnce({ queryId: "q-2" }).mockResolvedValueOnce({ + status: "Complete", + results: [[{ field: "callbackCount", value: "0" }]], + }); + + const result = await verifyMockWebhook( + mockClient, + "/aws/lambda/test-mock-webhook", + 1000, + 2000, + ); + + expect(result).toEqual({ receivedCallbacks: 0, verified: false }); + }); + + it("returns verified=false when query fails", async () => { + mockSend + .mockResolvedValueOnce({ queryId: "q-3" }) + .mockResolvedValueOnce({ status: "Failed" }); + + const result = await verifyMockWebhook( + mockClient, + "/aws/lambda/test-mock-webhook", + 1000, + 2000, + ); + + expect(result).toEqual({ receivedCallbacks: 0, verified: false }); + }); + + it("returns verified=false when no queryId is returned", async () => { + mockSend.mockResolvedValueOnce({}); + + const result = await verifyMockWebhook( + mockClient, + "/aws/lambda/test-mock-webhook", + 1000, + 2000, + ); + + expect(result).toEqual({ receivedCallbacks: 0, verified: false }); + }); + + it("returns verified=false when results are empty", async () => { + mockSend.mockResolvedValueOnce({ queryId: "q-4" }).mockResolvedValueOnce({ + status: "Complete", + results: [], + }); + + const result = await verifyMockWebhook( + mockClient, + "/aws/lambda/test-mock-webhook", + 1000, + 2000, + ); + + expect(result).toEqual({ receivedCallbacks: 0, verified: false }); + }); + + it("returns verified=false when results field is undefined", async () => { + mockSend.mockResolvedValueOnce({ queryId: "q-4b" }).mockResolvedValueOnce({ + status: "Complete", + results: undefined, + }); + + const result = await verifyMockWebhook( + mockClient, + "/aws/lambda/test-mock-webhook", + 1000, + 2000, + ); + + expect(result).toEqual({ receivedCallbacks: 0, verified: false }); + }); + + it("polls until the query completes", async () => { + mockSend + .mockResolvedValueOnce({ queryId: "q-5" }) + .mockResolvedValueOnce({ status: "Running" }) + .mockResolvedValueOnce({ + status: "Complete", + results: [[{ field: "callbackCount", value: "10" }]], + }); + + const result = await verifyMockWebhook( + mockClient, + "/aws/lambda/test-mock-webhook", + 1000, + 2000, + ); + + expect(result).toEqual({ receivedCallbacks: 10, verified: true }); + expect(mockSend).toHaveBeenCalledTimes(3); + }); + + it("returns verified=false when query is cancelled", async () => { + mockSend + .mockResolvedValueOnce({ queryId: "q-6" }) + .mockResolvedValueOnce({ status: "Cancelled" }); + + const result = await verifyMockWebhook( + mockClient, + "/aws/lambda/test-mock-webhook", + 1000, + 2000, + ); + + expect(result).toEqual({ receivedCallbacks: 0, verified: false }); + }); + + it("returns verified=false when polling times out", async () => { + jest.useFakeTimers(); + + mockSend.mockResolvedValueOnce({ queryId: "q-7" }).mockImplementation( + () => + new Promise((resolve) => { + setTimeout(() => resolve({ status: "Running" }), 1000); + }), + ); + + const originalDateNow = Date.now; + let callCount = 0; + jest.spyOn(Date, "now").mockImplementation(() => { + callCount += 1; + if (callCount <= 1) return originalDateNow.call(Date); + return originalDateNow.call(Date) + 60_000; + }); + + const promise = verifyMockWebhook( + mockClient, + "/aws/lambda/test-mock-webhook", + 1000, + 2000, + ); + + await jest.advanceTimersByTimeAsync(60_000); + + const result = await promise; + + expect(result).toEqual({ receivedCallbacks: 0, verified: false }); + + jest.useRealTimers(); + jest.restoreAllMocks(); + }); +}); diff --git a/lambdas/perf-runner-lambda/src/cloudwatch.ts b/lambdas/perf-runner-lambda/src/cloudwatch.ts new file mode 100644 index 00000000..598f5f3f --- /dev/null +++ b/lambdas/perf-runner-lambda/src/cloudwatch.ts @@ -0,0 +1,214 @@ +import { + type CloudWatchLogsClient, + GetQueryResultsCommand, + StartQueryCommand, +} from "@aws-sdk/client-cloudwatch-logs"; +import type { + CircuitBreakerSnapshot, + DeliveryMetricsSnapshot, + MetricsSnapshot, + PerClientRateEntry, +} from "types"; + +const INSIGHTS_POLL_INTERVAL_MS = 2000; +const INSIGHTS_TIMEOUT_MS = 30_000; + +type ResultField = { field?: string; value?: string }; + +async function pollInsightsQuery( + client: CloudWatchLogsClient, + queryId: string, +): Promise { + const deadline = Date.now() + INSIGHTS_TIMEOUT_MS; + + while (Date.now() < deadline) { + await new Promise((resolve) => { + setTimeout(resolve, INSIGHTS_POLL_INTERVAL_MS); + }); + + const response = await client.send(new GetQueryResultsCommand({ queryId })); + + if (response.status === "Failed" || response.status === "Cancelled") { + return null; + } + + if (response.status === "Complete") { + return (response.results as ResultField[][]) ?? []; + } + } + + return null; +} + +async function pollQueryResults( + client: CloudWatchLogsClient, + queryId: string, + mapRow: (row: ResultField[]) => T, +): Promise { + const rows = await pollInsightsQuery(client, queryId); + if (rows === null) return null; + return mapRow(rows[0] ?? []); +} + +async function pollAllQueryResults( + client: CloudWatchLogsClient, + queryId: string, + mapRow: (row: ResultField[]) => T, +): Promise { + const rows = await pollInsightsQuery(client, queryId); + if (rows === null) return []; + return rows.map((row) => mapRow(row)); +} + +export async function queryMetricsSnapshot( + client: CloudWatchLogsClient, + logGroupName: string, + startTimeSec: number, + endTimeSec: number, +): Promise { + const { queryId } = await client.send( + new StartQueryCommand({ + logGroupName, + startTime: startTimeSec, + endTime: endTimeSec, + queryString: [ + 'filter msg = "Callback lifecycle: batch-processing-completed"', + "| stats count(*) as eventCount, pct(processingTimeMs, 50) as p50, pct(processingTimeMs, 95) as p95, pct(processingTimeMs, 99) as p99", + ].join("\n"), + }), + ); + + if (!queryId) return null; + + return pollQueryResults(client, queryId, (row) => { + const getField = (name: string): number => + Number(row.find((f) => f.field === name)?.value ?? 0); + + return { + snapshotAt: Date.now(), + p50Ms: getField("p50"), + p95Ms: getField("p95"), + p99Ms: getField("p99"), + count: getField("eventCount"), + }; + }); +} + +export async function queryDeliveryMetricsSnapshot( + client: CloudWatchLogsClient, + logGroupNames: string[], + startTimeSec: number, + endTimeSec: number, +): Promise { + if (logGroupNames.length === 0) return null; + + const { queryId } = await client.send( + new StartQueryCommand({ + logGroupNames, + startTime: startTimeSec, + endTime: endTimeSec, + queryString: [ + "filter ispresent(DeliveryDurationMs)", + "| stats count(DeliveryDurationMs) as deliveryCount, pct(DeliveryDurationMs, 50) as p50, pct(DeliveryDurationMs, 95) as p95, pct(DeliveryDurationMs, 99) as p99", + ].join("\n"), + }), + ); + + if (!queryId) return null; + + return pollQueryResults(client, queryId, (row) => { + const getField = (name: string): number => + Number(row.find((f) => f.field === name)?.value ?? 0); + + return { + snapshotAt: Date.now(), + deliveryCount: getField("deliveryCount"), + p50Ms: getField("p50"), + p95Ms: getField("p95"), + p99Ms: getField("p99"), + }; + }); +} + +export async function queryCircuitBreakerSnapshot( + client: CloudWatchLogsClient, + logGroupNames: string[], + startTimeSec: number, + endTimeSec: number, +): Promise { + if (logGroupNames.length === 0) return null; + + const { queryId } = await client.send( + new StartQueryCommand({ + logGroupNames, + startTime: startTimeSec, + endTime: endTimeSec, + queryString: [ + 'filter msg in ["Circuit breaker opened", "Circuit breaker closed", "Admission denied", "Attempting delivery", "Delivery succeeded", "Transient delivery failure \u2014 requeuing", "Permanent delivery failure \u2014 sending to DLQ", "Rate limited (429)"]', + '| stats sum(msg = "Circuit breaker opened") as circuitOpenEvents,', + ' sum(msg = "Circuit breaker closed") as circuitCloseEvents,', + ' sum(msg = "Admission denied" and reason = "circuit_open") as admissionDeniedCircuitOpen,', + ' sum(msg = "Admission denied" and reason = "rate_limited") as admissionDeniedRateLimited,', + ' sum(msg = "Attempting delivery") as deliveryAttempts,', + ' sum(msg = "Delivery succeeded") as deliverySuccesses,', + ' sum(msg in ["Transient delivery failure \u2014 requeuing", "Permanent delivery failure \u2014 sending to DLQ"]) as deliveryFailures,', + ' sum(msg = "Rate limited (429)") as deliveryRateLimited', + ].join("\n"), + }), + ); + + if (!queryId) return null; + + return pollQueryResults(client, queryId, (row) => { + const getField = (name: string): number => + Number(row.find((f) => f.field === name)?.value ?? 0); + + return { + snapshotAt: Date.now(), + intervalStartSec: startTimeSec, + intervalEndSec: endTimeSec, + circuitOpenEvents: getField("circuitOpenEvents"), + circuitCloseEvents: getField("circuitCloseEvents"), + admissionDeniedCircuitOpen: getField("admissionDeniedCircuitOpen"), + admissionDeniedRateLimited: getField("admissionDeniedRateLimited"), + deliveryAttempts: getField("deliveryAttempts"), + deliverySuccesses: getField("deliverySuccesses"), + deliveryFailures: getField("deliveryFailures"), + deliveryRateLimited: getField("deliveryRateLimited"), + }; + }); +} + +const RATE_TIMELINE_BIN_SECONDS = 10; + +export async function queryPerClientRateTimeline( + client: CloudWatchLogsClient, + logGroupName: string, + startTimeSec: number, + endTimeSec: number, +): Promise { + const { queryId } = await client.send( + new StartQueryCommand({ + logGroupName, + startTime: startTimeSec, + endTime: endTimeSec, + queryString: [ + 'filter msg in ["Attempting delivery", "Admission denied"]', + `| stats sum(msg = "Attempting delivery") as deliveryAttempts by bin(@timestamp, ${RATE_TIMELINE_BIN_SECONDS}s) as timeBin`, + "| sort timeBin asc", + ].join("\n"), + }), + ); + + if (!queryId) return []; + + return pollAllQueryResults(client, queryId, (row) => { + const timeBinStr = row.find((f) => f.field === "timeBin")?.value ?? "0"; + const timestampSec = Math.floor(new Date(timeBinStr).getTime() / 1000); + const deliveryAttempts = Number( + row.find((f) => f.field === "deliveryAttempts")?.value ?? 0, + ); + + return { timestampSec, deliveryAttempts }; + }); +} diff --git a/lambdas/perf-runner-lambda/src/elasticache.ts b/lambdas/perf-runner-lambda/src/elasticache.ts new file mode 100644 index 00000000..8f41ad9e --- /dev/null +++ b/lambdas/perf-runner-lambda/src/elasticache.ts @@ -0,0 +1,110 @@ +import { type RedisClientType, createClient } from "@redis/client"; +import { SignatureV4 } from "@smithy/signature-v4"; +import { Sha256 } from "@aws-crypto/sha256-js"; +import { fromNodeProviderChain } from "@aws-sdk/credential-providers"; +import type { ElastiCacheDeps, EndpointRateLimitState } from "types"; + +const TOKEN_EXPIRY_SECONDS = 900; + +async function generateIamToken(deps: ElastiCacheDeps): Promise { + const signer = new SignatureV4({ + credentials: fromNodeProviderChain(), + region: deps.region, + service: "elasticache", + sha256: Sha256, + }); + + const signed = await signer.presign( + { + protocol: "https:", + method: "GET", + hostname: deps.cacheName, + path: "/", + query: { Action: "connect", User: deps.iamUsername }, + headers: { host: deps.cacheName }, + }, + { expiresIn: TOKEN_EXPIRY_SECONDS }, + ); + + const qs = new URLSearchParams( + signed.query as Record, + ).toString(); + return `${deps.cacheName}/?${qs}`; +} + +export async function flushElastiCache(deps: ElastiCacheDeps): Promise { + const token = await generateIamToken(deps); + + const client: RedisClientType = createClient({ + url: `rediss://${deps.endpoint}:6379`, + username: deps.iamUsername, + password: token, + }); + + try { + await client.connect(); + await client.flushAll(); + } finally { + if (client.isOpen) { + await client.disconnect(); + } + } +} + +const RATE_LIMIT_HASH_FIELDS = [ + "is_open", + "switched_at", + "bucket_tokens", + "bucket_refilled_at", + "cur_attempts", + "prev_attempts", + "cur_failures", + "prev_failures", + "sample_till", +] as const; + +export async function dumpRateLimitState( + deps: ElastiCacheDeps, +): Promise { + const token = await generateIamToken(deps); + + const client: RedisClientType = createClient({ + url: `rediss://${deps.endpoint}:6379`, + username: deps.iamUsername, + password: token, + }); + + try { + await client.connect(); + + const keys: string[] = []; + for await (const key of client.scanIterator({ MATCH: "ep:*" })) { + keys.push(key); + } + + // eslint-disable-next-line sonarjs/null-dereference -- false positive: keys is string[] + keys.sort((a, b) => a.localeCompare(b)); + const states: EndpointRateLimitState[] = []; + for (const key of keys) { + const values = await client.hmGet(key, [...RATE_LIMIT_HASH_FIELDS]); + states.push({ + key, + isOpen: values[0], + switchedAt: values[1], + bucketTokens: values[2], + bucketRefilledAt: values[3], + curAttempts: values[4], + prevAttempts: values[5], + curFailures: values[6], + prevFailures: values[7], + sampleTill: values[8], + }); + } + + return states; + } finally { + if (client.isOpen) { + await client.disconnect(); + } + } +} diff --git a/tests/performance/helpers/event-factories.ts b/lambdas/perf-runner-lambda/src/event-factories.ts similarity index 53% rename from tests/performance/helpers/event-factories.ts rename to lambdas/perf-runner-lambda/src/event-factories.ts index c31571e4..c32cb9f7 100644 --- a/tests/performance/helpers/event-factories.ts +++ b/lambdas/perf-runner-lambda/src/event-factories.ts @@ -1,22 +1,45 @@ import type { + ChannelStatus, ChannelStatusData, + MessageStatus, MessageStatusData, StatusPublishEvent, } from "@nhs-notify-client-callbacks/models"; import { EventTypes } from "@nhs-notify-client-callbacks/models"; +import type { EventMixEntry } from "types"; -export function createMessageStatusPublishEvent( - overrides?: Partial, +function buildMessageId( + uuid: string, + forcedStatusCode?: number, + forcedStatusCodeUntilMs?: number, +): string { + if (forcedStatusCode === undefined) { + return uuid; + } + if (forcedStatusCodeUntilMs === undefined) { + return `force-${forcedStatusCode}-${uuid}`; + } + return `force-${forcedStatusCode}-until-${forcedStatusCodeUntilMs}-${uuid}`; +} + +export function createMessageStatusEvent( + clientId: string, + messageStatus: MessageStatus, + forcedStatusCode?: number, + forcedStatusCodeUntilMs?: number, ): StatusPublishEvent { - const messageId = overrides?.messageId ?? crypto.randomUUID(); - const messageReference = - overrides?.messageReference ?? `ref-${crypto.randomUUID()}`; + const uuid = crypto.randomUUID(); + const messageId = buildMessageId( + uuid, + forcedStatusCode, + forcedStatusCodeUntilMs, + ); const data: MessageStatusData = { - clientId: "mock-client-1", + clientId, messageId, - messageReference, - messageStatus: "DELIVERED", + messageReference: `ref-${crypto.randomUUID()}`, + messageStatus, channels: [{ type: "NHSAPP", channelStatus: "DELIVERED" }], timestamp: new Date().toISOString(), routingPlan: { @@ -25,7 +48,6 @@ export function createMessageStatusPublishEvent( version: "v1.0.0", createdDate: new Date().toISOString(), }, - ...overrides, }; return { @@ -43,26 +65,30 @@ export function createMessageStatusPublishEvent( }; } -export function createChannelStatusPublishEvent( - overrides?: Partial, +export function createChannelStatusEvent( + clientId: string, + channelStatus: ChannelStatus, + forcedStatusCode?: number, + forcedStatusCodeUntilMs?: number, ): StatusPublishEvent { - const messageId = overrides?.messageId ?? crypto.randomUUID(); - const messageReference = - overrides?.messageReference ?? `ref-${crypto.randomUUID()}`; + const uuid = crypto.randomUUID(); + const messageId = buildMessageId( + uuid, + forcedStatusCode, + forcedStatusCodeUntilMs, + ); const data: ChannelStatusData = { - clientId: "mock-client-1", + clientId, messageId, - messageReference, + messageReference: `ref-${crypto.randomUUID()}`, channel: "NHSAPP", - channelStatus: "DELIVERED", - channelStatusDescription: "perf-test", + channelStatus, supplierStatus: "delivered", cascadeType: "primary", cascadeOrder: 0, timestamp: new Date().toISOString(), retryCount: 0, - ...overrides, }; return { @@ -79,3 +105,21 @@ export function createChannelStatusPublishEvent( data, }; } + +export function createEvent(entry: EventMixEntry): StatusPublishEvent { + if (entry.factory === "messageStatus") { + return createMessageStatusEvent( + entry.clientId, + entry.messageStatus, + entry.forcedStatusCode, + entry.forcedStatusCodeUntilMs, + ); + } + + return createChannelStatusEvent( + entry.clientId, + entry.channelStatus, + entry.forcedStatusCode, + entry.forcedStatusCodeUntilMs, + ); +} diff --git a/lambdas/perf-runner-lambda/src/index.ts b/lambdas/perf-runner-lambda/src/index.ts new file mode 100644 index 00000000..12def2d6 --- /dev/null +++ b/lambdas/perf-runner-lambda/src/index.ts @@ -0,0 +1,79 @@ +import { CloudWatchLogsClient } from "@aws-sdk/client-cloudwatch-logs"; +import { SQSClient } from "@aws-sdk/client-sqs"; +import { Logger } from "@nhs-notify-client-callbacks/logger"; +import { runPerformanceTest } from "runner"; +import type { + ElastiCacheDeps, + PerfRunnerPayload, + PerformanceResult, +} from "types"; + +const logger = new Logger(); + +export async function handler( + event: PerfRunnerPayload, +): Promise { + const { cloudWatchSettlingMs, scenario, skipPurge, testId } = event; + + const region = process.env.AWS_REGION ?? "eu-west-2"; + const queueUrl = process.env.INBOUND_QUEUE_URL; + const deliveryQueueUrlPrefix = process.env.DELIVERY_QUEUE_URL_PREFIX; + const logGroupName = process.env.TRANSFORM_FILTER_LOG_GROUP; + const deliveryLogGroupPrefix = process.env.DELIVERY_LOG_GROUP_PREFIX; + const mockWebhookLogGroup = process.env.MOCK_WEBHOOK_LOG_GROUP; + const elasticacheEndpoint = process.env.ELASTICACHE_ENDPOINT; + const elasticacheCacheName = process.env.ELASTICACHE_CACHE_NAME; + const elasticacheIamUsername = process.env.ELASTICACHE_IAM_USERNAME; + + if (!queueUrl) { + throw new Error("Missing required environment variable: INBOUND_QUEUE_URL"); + } + + if (!logGroupName) { + throw new Error( + "Missing required environment variable: TRANSFORM_FILTER_LOG_GROUP", + ); + } + + const sqsClient = new SQSClient({ region }); + const cloudWatchClient = new CloudWatchLogsClient({ region }); + + const elastiCacheDeps: ElastiCacheDeps | undefined = + elasticacheEndpoint && elasticacheCacheName && elasticacheIamUsername + ? { + endpoint: elasticacheEndpoint, + cacheName: elasticacheCacheName, + iamUsername: elasticacheIamUsername, + region, + } + : undefined; + + logger.info("Performance test started", { testId }); + + try { + const result = await runPerformanceTest( + { + sqsClient, + cloudWatchClient, + queueUrl, + deliveryQueueUrlPrefix, + logGroupName, + deliveryLogGroupPrefix, + mockWebhookLogGroup, + }, + scenario, + testId, + undefined, + elastiCacheDeps, + cloudWatchSettlingMs, + skipPurge, + ); + + logger.info("Performance test completed", { testId }); + + return result; + } finally { + sqsClient.destroy(); + cloudWatchClient.destroy(); + } +} diff --git a/lambdas/perf-runner-lambda/src/purge.ts b/lambdas/perf-runner-lambda/src/purge.ts new file mode 100644 index 00000000..40de78e5 --- /dev/null +++ b/lambdas/perf-runner-lambda/src/purge.ts @@ -0,0 +1,33 @@ +import { PurgeQueueCommand, type SQSClient } from "@aws-sdk/client-sqs"; +import type { Scenario } from "types"; + +export function deriveQueueUrls( + inboundQueueUrl: string, + scenario: Scenario, + deliveryQueueUrlPrefix?: string, +): string[] { + // eslint-disable-next-line sonarjs/null-dereference -- String.replace always returns a string + const inboundBaseUrl = inboundQueueUrl.replace(/inbound-event-queue$/, ""); + const deliveryBaseUrl = deliveryQueueUrlPrefix ?? inboundBaseUrl; + const clientIds = [...new Set(scenario.eventMix.map((e) => e.clientId))]; + + return [ + inboundQueueUrl, + `${inboundBaseUrl}inbound-event-dlq`, + ...clientIds.flatMap((id) => [ + `${deliveryBaseUrl}${id}-delivery-queue`, + `${deliveryBaseUrl}${id}-delivery-dlq-queue`, + ]), + ]; +} + +export async function purgeQueues( + client: SQSClient, + queueUrls: string[], +): Promise { + await Promise.all( + queueUrls.map((url) => + client.send(new PurgeQueueCommand({ QueueUrl: url })), + ), + ); +} diff --git a/lambdas/perf-runner-lambda/src/runner.ts b/lambdas/perf-runner-lambda/src/runner.ts new file mode 100644 index 00000000..c2726987 --- /dev/null +++ b/lambdas/perf-runner-lambda/src/runner.ts @@ -0,0 +1,305 @@ +import type { + CircuitBreakerSnapshot, + DeliveryMetricsSnapshot, + ElastiCacheDeps, + EndpointRateLimitState, + MetricsSnapshot, + PerClientRateTimeline, + PerformanceResult, + PhaseResult, + RunnerDeps, + Scenario, + WebhookVerificationResult, +} from "types"; +import { Logger } from "@nhs-notify-client-callbacks/logger"; +import { generatePhaseLoad } from "sqs"; +import { deriveQueueUrls, purgeQueues } from "purge"; +import { getQueueDepths } from "sqs-stats"; +import { dumpRateLimitState, flushElastiCache } from "elasticache"; +import { verifyMockWebhook } from "webhook-verify"; +import { + queryCircuitBreakerSnapshot, + queryDeliveryMetricsSnapshot, + queryMetricsSnapshot, + queryPerClientRateTimeline, +} from "cloudwatch"; + +const logger = new Logger(); + +const CLOUDWATCH_SETTLING_MS = 60_000; + +export const defaultSleep = (ms: number): Promise => + new Promise((resolve) => { + setTimeout(resolve, ms); + }); + +function buildDeliveryLogGroupNames( + prefix: string | undefined, + scenario: Scenario, +): string[] { + if (!prefix) return []; + const clientIds = new Set(scenario.eventMix.map((e) => e.clientId)); + return [...clientIds].map((id) => `${prefix}${id}`); +} + +async function collectSnapshots( + deps: RunnerDeps, + deliveryLogGroupNames: string[], + startSec: number, + endSec: number, + cbStartSec: number, + out: { + snapshots: MetricsSnapshot[]; + deliverySnapshots: DeliveryMetricsSnapshot[]; + cbSnapshots: CircuitBreakerSnapshot[]; + }, +): Promise { + const snap = await queryMetricsSnapshot( + deps.cloudWatchClient, + deps.logGroupName, + startSec, + endSec, + ); + if (snap !== null) out.snapshots.push(snap); + + if (deliveryLogGroupNames.length > 0) { + const deliverySnap = await queryDeliveryMetricsSnapshot( + deps.cloudWatchClient, + deliveryLogGroupNames, + startSec, + endSec, + ); + if (deliverySnap !== null) out.deliverySnapshots.push(deliverySnap); + + const cbSnap = await queryCircuitBreakerSnapshot( + deps.cloudWatchClient, + deliveryLogGroupNames, + cbStartSec, + endSec, + ); + if (cbSnap !== null) { + out.cbSnapshots.push(cbSnap); + return endSec; + } + } + + return cbStartSec; +} + +async function collectPerClientRateTimelines( + deps: RunnerDeps, + scenario: Scenario, + startSec: number, + endSec: number, +): Promise { + if (!deps.deliveryLogGroupPrefix) { + return []; + } + + const clientIds = [...new Set(scenario.eventMix.map((e) => e.clientId))]; + const timelinePromises = clientIds.map(async (clientId) => { + const logGroupName = `${deps.deliveryLogGroupPrefix}${clientId}`; + const entries = await queryPerClientRateTimeline( + deps.cloudWatchClient, + logGroupName, + startSec, + endSec, + ); + return { clientId, entries }; + }); + const timelines = await Promise.all(timelinePromises); + return timelines.filter((t) => t.entries.length > 0); +} + +async function collectWebhookVerification( + deps: RunnerDeps, + startSec: number, + endSec: number, +): Promise { + if (!deps.mockWebhookLogGroup) { + return undefined; + } + return verifyMockWebhook( + deps.cloudWatchClient, + deps.mockWebhookLogGroup, + startSec, + endSec, + ); +} + +export async function runPerformanceTest( + deps: RunnerDeps, + scenario: Scenario, + testId: string, + sleepFn: (ms: number) => Promise = defaultSleep, + elastiCacheDeps?: ElastiCacheDeps, + cloudWatchSettlingMs: number = CLOUDWATCH_SETTLING_MS, + skipPurge = false, +): Promise { + if (scenario.eventMix.length === 0) { + throw new Error("scenario.eventMix must contain at least one entry"); + } + + for (const [index, phase] of scenario.phases.entries()) { + if (phase.durationSecs <= 0) { + throw new Error( + `scenario.phases[${index}].durationSecs must be greater than 0`, + ); + } + if (phase.targetEps <= 0) { + throw new Error( + `scenario.phases[${index}].targetEps must be greater than 0`, + ); + } + } + + const testStartMs = Date.now(); + + const queueUrls = deriveQueueUrls( + deps.queueUrl, + scenario, + deps.deliveryQueueUrlPrefix, + ); + + if (skipPurge) { + logger.info("Skipping queue purge", { queueUrls }); + } else { + logger.info("Purging queues", { queueUrls }); + await purgeQueues(deps.sqsClient, queueUrls); + } + if (elastiCacheDeps) { + logger.info("Clearing rate limit and circuit breaker state"); + await flushElastiCache(elastiCacheDeps); + } + + let rateLimitStateBefore: EndpointRateLimitState[] | undefined; + if (elastiCacheDeps) { + rateLimitStateBefore = await dumpRateLimitState(elastiCacheDeps); + } + + const startedAt = new Date(testStartMs).toISOString(); + const phaseResults: PhaseResult[] = []; + const snapshots: MetricsSnapshot[] = []; + const deliverySnapshots: DeliveryMetricsSnapshot[] = []; + const cbSnapshots: CircuitBreakerSnapshot[] = []; + let lastCbSnapshotSec = Math.floor(testStartMs / 1000); + let stopPolling = false; + + const deliveryLogGroupNames = buildDeliveryLogGroupNames( + deps.deliveryLogGroupPrefix, + scenario, + ); + + const out = { snapshots, deliverySnapshots, cbSnapshots }; + + const pollLoop = async (): Promise => { + await sleepFn(scenario.metricsIntervalSecs * 1000); + while (!stopPolling) { + const startSec = Math.floor(testStartMs / 1000); + const endSec = Math.floor(Date.now() / 1000); + + lastCbSnapshotSec = await collectSnapshots( + deps, + deliveryLogGroupNames, + startSec, + endSec, + lastCbSnapshotSec, + out, + ); + logger.info("Sampling queue depths", { queueUrls }); + const depthSample = await getQueueDepths(deps.sqsClient, queueUrls); + logger.info("Queue depth sample", { queues: depthSample.queues }); + + if (!stopPolling) { + await sleepFn(scenario.metricsIntervalSecs * 1000); + } + } + }; + + const pollPromise = pollLoop(); + + for (const [index, phase] of scenario.phases.entries()) { + logger.info("Starting phase", { + index, + targetEps: phase.targetEps, + durationSecs: phase.durationSecs, + }); + const result = await generatePhaseLoad( + deps.sqsClient, + deps.queueUrl, + phase, + phase.eventMix ?? scenario.eventMix, + ); + logger.info("Phase complete", { + index, + targetEps: result.targetEps, + achievedEps: result.achievedEps, + sent: result.sent, + durationMs: result.durationMs, + }); + phaseResults.push(result); + } + + stopPolling = true; + await pollPromise; + + logger.info("Waiting for CloudWatch logs to settle", { + settlingMs: cloudWatchSettlingMs, + }); + await sleepFn(cloudWatchSettlingMs); + + const finalStartSec = Math.floor(testStartMs / 1000); + const finalEndSec = Math.floor(Date.now() / 1000); + + await collectSnapshots( + deps, + deliveryLogGroupNames, + finalStartSec, + finalEndSec, + lastCbSnapshotSec, + out, + ); + logger.info("Sampling queue depths", { queueUrls }); + const finalDepthSample = await getQueueDepths(deps.sqsClient, queueUrls); + logger.info("Final queue depth sample", { queues: finalDepthSample.queues }); + + const perClientRateTimelines = await collectPerClientRateTimelines( + deps, + scenario, + finalStartSec, + finalEndSec, + ); + + const webhookVerification = await collectWebhookVerification( + deps, + finalStartSec, + finalEndSec, + ); + + let rateLimitStateAfter: EndpointRateLimitState[] | undefined; + if (elastiCacheDeps) { + rateLimitStateAfter = await dumpRateLimitState(elastiCacheDeps); + } + + if (skipPurge) { + logger.info("Skipping final queue purge", { queueUrls }); + } else { + await purgeQueues(deps.sqsClient, queueUrls); + logger.info("Final queue purge complete", { queueUrls }); + } + + return { + testId, + scenario, + startedAt, + completedAt: new Date().toISOString(), + phases: phaseResults, + metrics: snapshots, + deliveryMetrics: deliverySnapshots, + circuitBreakerMetrics: cbSnapshots, + perClientRateTimelines, + webhookVerification, + rateLimitStateBefore, + rateLimitStateAfter, + }; +} diff --git a/lambdas/perf-runner-lambda/src/sqs-stats.ts b/lambdas/perf-runner-lambda/src/sqs-stats.ts new file mode 100644 index 00000000..5d573793 --- /dev/null +++ b/lambdas/perf-runner-lambda/src/sqs-stats.ts @@ -0,0 +1,29 @@ +import { GetQueueAttributesCommand, type SQSClient } from "@aws-sdk/client-sqs"; +import type { QueueDepthSample } from "types"; + +export async function getQueueDepths( + client: SQSClient, + queueUrls: string[], +): Promise { + const queues = await Promise.all( + queueUrls.map(async (url) => { + const response = await client.send( + new GetQueueAttributesCommand({ + QueueUrl: url, + AttributeNames: [ + "ApproximateNumberOfMessages", + "ApproximateNumberOfMessagesNotVisible", + ], + }), + ); + const attrs = response.Attributes ?? {}; + return { + queueUrl: url, + visible: Number(attrs.ApproximateNumberOfMessages ?? "0"), + notVisible: Number(attrs.ApproximateNumberOfMessagesNotVisible ?? "0"), + }; + }), + ); + + return { timestampMs: Date.now(), queues }; +} diff --git a/lambdas/perf-runner-lambda/src/sqs.ts b/lambdas/perf-runner-lambda/src/sqs.ts new file mode 100644 index 00000000..154ce2e3 --- /dev/null +++ b/lambdas/perf-runner-lambda/src/sqs.ts @@ -0,0 +1,78 @@ +import { type SQSClient, SendMessageBatchCommand } from "@aws-sdk/client-sqs"; +import type { StatusPublishEvent } from "@nhs-notify-client-callbacks/models"; +import type { EventMixEntry, Phase, PhaseResult } from "types"; +import { createEvent } from "event-factories"; + +const SQS_MAX_BATCH_SIZE = 10; + +export function selectWeighted(entries: T[]): T { + const totalWeight = entries.reduce((sum, entry) => sum + entry.weight, 0); + // eslint-disable-next-line sonarjs/pseudo-random -- weighted selection for load test event distribution + let remaining = Math.random() * totalWeight; + + for (const entry of entries.slice(0, -1)) { + remaining -= entry.weight; + if (remaining <= 0) return entry; + } + + // Safe: selectWeighted is only called with non-empty arrays + return entries.at(-1)!; +} + +export async function sendSqsBatch( + client: SQSClient, + queueUrl: string, + events: StatusPublishEvent[], +): Promise { + await client.send( + new SendMessageBatchCommand({ + QueueUrl: queueUrl, + Entries: events.map((event, index) => ({ + Id: String(index), + MessageBody: JSON.stringify(event), + })), + }), + ); +} + +export async function generatePhaseLoad( + client: SQSClient, + queueUrl: string, + phase: Phase, + eventMix: EventMixEntry[], +): Promise { + const batchesPerSecond = Math.ceil(phase.targetEps / SQS_MAX_BATCH_SIZE); + const start = Date.now(); + let sent = 0; + + for (let second = 0; second < phase.durationSecs; second++) { + const waveStart = Date.now(); + + const batchResults = await Promise.all( + Array.from({ length: batchesPerSecond }, () => { + const batch = Array.from({ length: SQS_MAX_BATCH_SIZE }, () => + createEvent(selectWeighted(eventMix)), + ); + return sendSqsBatch(client, queueUrl, batch).then(() => batch.length); + }), + ); + + sent += batchResults.reduce((sum, count) => sum + count, 0); + + const remaining = 1000 - (Date.now() - waveStart); + if (remaining > 0 && second < phase.durationSecs - 1) { + await new Promise((resolve) => { + setTimeout(resolve, remaining); + }); + } + } + + const durationMs = Date.now() - start; + + return { + targetEps: phase.targetEps, + achievedEps: Math.round(sent / (durationMs / 1000)), + sent, + durationMs, + }; +} diff --git a/lambdas/perf-runner-lambda/src/types.ts b/lambdas/perf-runner-lambda/src/types.ts new file mode 100644 index 00000000..3634a36e --- /dev/null +++ b/lambdas/perf-runner-lambda/src/types.ts @@ -0,0 +1,151 @@ +import type { SQSClient } from "@aws-sdk/client-sqs"; +import type { CloudWatchLogsClient } from "@aws-sdk/client-cloudwatch-logs"; +import type { + ChannelStatus, + MessageStatus, +} from "@nhs-notify-client-callbacks/models"; + +export type MessageStatusMixEntry = { + weight: number; + factory: "messageStatus"; + clientId: string; + messageStatus: MessageStatus; + forcedStatusCode?: number; + forcedStatusCodeUntilMs?: number; +}; + +export type ChannelStatusMixEntry = { + weight: number; + factory: "channelStatus"; + clientId: string; + channelStatus: ChannelStatus; + forcedStatusCode?: number; + forcedStatusCodeUntilMs?: number; +}; + +export type EventMixEntry = MessageStatusMixEntry | ChannelStatusMixEntry; + +export type Phase = { + durationSecs: number; + targetEps: number; + eventMix?: EventMixEntry[]; +}; + +export type Scenario = { + phases: Phase[]; + eventMix: EventMixEntry[]; + metricsIntervalSecs: number; +}; + +export type PhaseResult = { + targetEps: number; + achievedEps: number; + sent: number; + durationMs: number; +}; + +export type MetricsSnapshot = { + snapshotAt: number; + p50Ms: number; + p95Ms: number; + p99Ms: number; + count: number; +}; + +export type DeliveryMetricsSnapshot = { + snapshotAt: number; + deliveryCount: number; + p50Ms: number; + p95Ms: number; + p99Ms: number; +}; + +export type CircuitBreakerSnapshot = { + snapshotAt: number; + intervalStartSec: number; + intervalEndSec: number; + circuitOpenEvents: number; + circuitCloseEvents: number; + admissionDeniedCircuitOpen: number; + admissionDeniedRateLimited: number; + deliveryAttempts: number; + deliverySuccesses: number; + deliveryFailures: number; + deliveryRateLimited: number; +}; + +export type PerClientRateEntry = { + timestampSec: number; + deliveryAttempts: number; +}; + +export type PerClientRateTimeline = { + clientId: string; + entries: PerClientRateEntry[]; +}; + +export type EndpointRateLimitState = { + key: string; + isOpen: string | null; + switchedAt: string | null; + bucketTokens: string | null; + bucketRefilledAt: string | null; + curAttempts: string | null; + prevAttempts: string | null; + curFailures: string | null; + prevFailures: string | null; + sampleTill: string | null; +}; + +export type WebhookVerificationResult = { + receivedCallbacks: number; + verified: boolean; +}; + +export type QueueDepthSample = { + timestampMs: number; + queues: { + queueUrl: string; + visible: number; + notVisible: number; + }[]; +}; + +export type PerformanceResult = { + testId: string; + scenario: Scenario; + startedAt: string; + completedAt: string; + phases: PhaseResult[]; + metrics: MetricsSnapshot[]; + deliveryMetrics: DeliveryMetricsSnapshot[]; + circuitBreakerMetrics: CircuitBreakerSnapshot[]; + perClientRateTimelines?: PerClientRateTimeline[]; + webhookVerification?: WebhookVerificationResult; + rateLimitStateBefore?: EndpointRateLimitState[]; + rateLimitStateAfter?: EndpointRateLimitState[]; +}; + +export type PerfRunnerPayload = { + testId: string; + scenario: Scenario; + cloudWatchSettlingMs?: number; + skipPurge?: boolean; +}; + +export type RunnerDeps = { + sqsClient: SQSClient; + cloudWatchClient: CloudWatchLogsClient; + queueUrl: string; + deliveryQueueUrlPrefix?: string; + logGroupName: string; + deliveryLogGroupPrefix?: string; + mockWebhookLogGroup?: string; +}; + +export type ElastiCacheDeps = { + endpoint: string; + cacheName: string; + iamUsername: string; + region: string; +}; diff --git a/lambdas/perf-runner-lambda/src/webhook-verify.ts b/lambdas/perf-runner-lambda/src/webhook-verify.ts new file mode 100644 index 00000000..77c1fa6d --- /dev/null +++ b/lambdas/perf-runner-lambda/src/webhook-verify.ts @@ -0,0 +1,59 @@ +import { + type CloudWatchLogsClient, + GetQueryResultsCommand, + StartQueryCommand, +} from "@aws-sdk/client-cloudwatch-logs"; +import type { WebhookVerificationResult } from "types"; + +const INSIGHTS_POLL_INTERVAL_MS = 2000; +const INSIGHTS_TIMEOUT_MS = 30_000; + +export async function verifyMockWebhook( + client: CloudWatchLogsClient, + logGroupName: string, + startTimeSec: number, + endTimeSec: number, +): Promise { + const { queryId } = await client.send( + new StartQueryCommand({ + logGroupName, + startTime: startTimeSec, + endTime: endTimeSec, + queryString: [ + 'filter msg = "Callback received"', + "| stats count(*) as callbackCount", + ].join("\n"), + }), + ); + + if (!queryId) { + return { receivedCallbacks: 0, verified: false }; + } + + const deadline = Date.now() + INSIGHTS_TIMEOUT_MS; + + while (Date.now() < deadline) { + await new Promise((resolve) => { + setTimeout(resolve, INSIGHTS_POLL_INTERVAL_MS); + }); + + const response = await client.send(new GetQueryResultsCommand({ queryId })); + + if (response.status === "Failed" || response.status === "Cancelled") { + return { receivedCallbacks: 0, verified: false }; + } + + if (response.status === "Complete") { + const rows = + (response.results as { field?: string; value?: string }[][]) ?? []; + const row = rows[0] ?? []; + const count = Number( + row.find((f) => f.field === "callbackCount")?.value ?? 0, + ); + + return { receivedCallbacks: count, verified: count > 0 }; + } + } + + return { receivedCallbacks: 0, verified: false }; +} diff --git a/lambdas/perf-runner-lambda/tsconfig.json b/lambdas/perf-runner-lambda/tsconfig.json new file mode 100644 index 00000000..a50e6fc0 --- /dev/null +++ b/lambdas/perf-runner-lambda/tsconfig.json @@ -0,0 +1,14 @@ +{ + "compilerOptions": { + "isolatedModules": true, + "paths": { + "*": [ + "./src/*" + ] + } + }, + "extends": "../../tsconfig.base.json", + "include": [ + "src/**/*" + ] +} diff --git a/package.json b/package.json index fa6f2223..aeddf03e 100644 --- a/package.json +++ b/package.json @@ -32,15 +32,6 @@ "typescript-eslint": "catalog:lint" }, "name": "nhs-notify-client-callbacks", - "pnpm": { - "overrides": { - "collect-v8-coverage": "^1.0.3", - "pretty-format>react-is": "19.0.0", - "flatted": "^3.4.0", - "fast-xml-parser": "^5.5.6", - "ts-jest>handlebars": "^4.7.9" - } - }, "scripts": { "generate-dependencies": "pnpm -r run --if-present generate-dependencies || true", "lint": "pnpm -r run lint", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 7ebf8557..de4d084e 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -6,6 +6,9 @@ settings: catalogs: app: + '@redis/client': + specifier: ^1.5.14 + version: 1.6.1 async-wait-until: specifier: ^2.0.31 version: 2.0.31 @@ -15,9 +18,15 @@ catalogs: cloudevents: specifier: ^10.0.0 version: 10.0.0 + node-forge: + specifier: ^1.3.1 + version: 1.4.0 p-map: specifier: ^4.0.0 version: 4.0.0 + picocolors: + specifier: ^1.1.1 + version: 1.1.1 pino: specifier: ^10.3.1 version: 10.3.1 @@ -31,6 +40,9 @@ catalogs: specifier: ^4.3.6 version: 4.3.6 aws: + '@aws-crypto/sha256-js': + specifier: ^5.2.0 + version: 5.2.0 '@aws-sdk/client-cloudwatch': specifier: ^3.1025.0 version: 3.1029.0 @@ -43,15 +55,18 @@ catalogs: '@aws-sdk/client-sqs': specifier: ^3.1023.0 version: 3.1026.0 - '@aws-sdk/client-ssm': - specifier: ^3.1025.0 - version: 3.1029.0 '@aws-sdk/client-sts': specifier: ^3.1023.0 version: 3.1026.0 '@aws-sdk/credential-providers': specifier: ^3.1023.0 version: 3.1026.0 + '@smithy/signature-v4': + specifier: ^5.0.0 + version: 5.3.13 + '@smithy/types': + specifier: ^4.3.1 + version: 4.14.0 lint: '@eslint/js': specifier: ^9.39.4 @@ -134,8 +149,11 @@ catalogs: specifier: ^8.10.161 version: 8.10.161 '@types/node': - specifier: ^24.12.0 - version: 24.12.0 + specifier: ^25.5.0 + version: 25.6.0 + '@types/node-forge': + specifier: ^1.3.11 + version: 1.3.14 '@types/yargs': specifier: ^17.0.24 version: 17.0.35 @@ -204,7 +222,7 @@ importers: version: 4.16.2(@typescript-eslint/utils@8.58.0(eslint@9.39.4(jiti@2.6.1))(typescript@5.9.3))(eslint-import-resolver-node@0.3.9)(eslint@9.39.4(jiti@2.6.1)) eslint-plugin-jest: specifier: catalog:lint - version: 29.15.2(@typescript-eslint/eslint-plugin@8.58.0(@typescript-eslint/parser@8.58.0(eslint@9.39.4(jiti@2.6.1))(typescript@5.9.3))(eslint@9.39.4(jiti@2.6.1))(typescript@5.9.3))(eslint@9.39.4(jiti@2.6.1))(jest@30.3.0(@types/node@25.5.0)(ts-node@10.9.2(@types/node@25.5.0)(typescript@5.9.3)))(typescript@5.9.3) + version: 29.15.2(@typescript-eslint/eslint-plugin@8.58.0(@typescript-eslint/parser@8.58.0(eslint@9.39.4(jiti@2.6.1))(typescript@5.9.3))(eslint@9.39.4(jiti@2.6.1))(typescript@5.9.3))(eslint@9.39.4(jiti@2.6.1))(jest@30.3.0(@types/node@25.6.0)(ts-node@10.9.2(@types/node@25.6.0)(typescript@5.9.3)))(typescript@5.9.3) eslint-plugin-json: specifier: catalog:lint version: 4.0.1 @@ -231,10 +249,10 @@ importers: version: 63.0.0(eslint@9.39.4(jiti@2.6.1)) jest: specifier: catalog:test - version: 30.3.0(@types/node@25.5.0)(ts-node@10.9.2(@types/node@25.5.0)(typescript@5.9.3)) + version: 30.3.0(@types/node@25.6.0)(ts-node@10.9.2(@types/node@25.6.0)(typescript@5.9.3)) jest-html-reporter: specifier: catalog:test - version: 4.4.0(jest@30.3.0(@types/node@25.5.0)(ts-node@10.9.2(@types/node@25.5.0)(typescript@5.9.3))) + version: 4.4.0(jest@30.3.0(@types/node@25.6.0)(ts-node@10.9.2(@types/node@25.6.0)(typescript@5.9.3))) knip: specifier: catalog:tools version: 6.4.1(@emnapi/core@1.9.1)(@emnapi/runtime@1.9.1) @@ -243,10 +261,10 @@ importers: version: 5.0.1 ts-jest: specifier: catalog:test - version: 29.4.9(@babel/core@7.29.0)(@jest/transform@30.3.0)(@jest/types@30.3.0)(babel-jest@30.3.0(@babel/core@7.29.0))(esbuild@0.28.0)(jest-util@30.3.0)(jest@30.3.0(@types/node@25.5.0)(ts-node@10.9.2(@types/node@25.5.0)(typescript@5.9.3)))(typescript@5.9.3) + version: 29.4.9(@babel/core@7.29.0)(@jest/transform@30.3.0)(@jest/types@30.3.0)(babel-jest@30.3.0(@babel/core@7.29.0))(esbuild@0.28.0)(jest-util@30.3.0)(jest@30.3.0(@types/node@25.6.0)(ts-node@10.9.2(@types/node@25.6.0)(typescript@5.9.3)))(typescript@5.9.3) ts-node: specifier: catalog:tools - version: 10.9.2(@types/node@25.5.0)(typescript@5.9.3) + version: 10.9.2(@types/node@25.6.0)(typescript@5.9.3) tsx: specifier: catalog:tools version: 4.21.0 @@ -264,9 +282,9 @@ importers: '@aws-sdk/client-s3': specifier: catalog:aws version: 3.1029.0 - '@aws-sdk/client-ssm': - specifier: catalog:aws - version: 3.1029.0 + '@nhs-notify-client-callbacks/config-subscription-cache': + specifier: workspace:* + version: link:../../src/config-subscription-cache '@nhs-notify-client-callbacks/logger': specifier: workspace:* version: link:../../src/logger @@ -297,7 +315,7 @@ importers: version: 30.0.0 '@types/node': specifier: catalog:tools - version: 24.12.0 + version: 25.6.0 esbuild: specifier: catalog:tools version: 0.28.0 @@ -306,7 +324,80 @@ importers: version: 9.39.4(jiti@2.6.1) jest: specifier: catalog:test - version: 30.3.0(@types/node@24.12.0)(ts-node@10.9.2(@types/node@24.12.0)(typescript@5.9.3)) + version: 30.3.0(@types/node@25.6.0)(ts-node@10.9.2(@types/node@25.6.0)(typescript@5.9.3)) + typescript: + specifier: catalog:tools + version: 5.9.3 + + lambdas/https-client-lambda: + dependencies: + '@aws-crypto/sha256-js': + specifier: catalog:aws + version: 5.2.0 + '@aws-sdk/client-s3': + specifier: catalog:aws + version: 3.1029.0 + '@aws-sdk/client-sqs': + specifier: catalog:aws + version: 3.1026.0 + '@aws-sdk/credential-providers': + specifier: catalog:aws + version: 3.1026.0 + '@nhs-notify-client-callbacks/config-subscription-cache': + specifier: workspace:* + version: link:../../src/config-subscription-cache + '@nhs-notify-client-callbacks/logger': + specifier: workspace:* + version: link:../../src/logger + '@nhs-notify-client-callbacks/models': + specifier: workspace:* + version: link:../../src/models + '@redis/client': + specifier: catalog:app + version: 1.6.1 + '@smithy/signature-v4': + specifier: catalog:aws + version: 5.3.13 + aws-embedded-metrics: + specifier: catalog:app + version: 4.2.1 + esbuild: + specifier: catalog:tools + version: 0.28.0 + node-forge: + specifier: catalog:app + version: 1.4.0 + p-map: + specifier: catalog:app + version: 4.0.0 + devDependencies: + '@smithy/types': + specifier: catalog:aws + version: 4.14.0 + '@tsconfig/node22': + specifier: catalog:tools + version: 22.0.5 + '@types/aws-lambda': + specifier: catalog:tools + version: 8.10.161 + '@types/jest': + specifier: catalog:test + version: 30.0.0 + '@types/node': + specifier: catalog:tools + version: 25.6.0 + '@types/node-forge': + specifier: catalog:tools + version: 1.3.14 + eslint: + specifier: catalog:lint + version: 9.39.4(jiti@2.6.1) + fengari: + specifier: ^0.1.5 + version: 0.1.5 + jest: + specifier: catalog:test + version: 30.3.0(@types/node@25.6.0)(ts-node@10.9.2(@types/node@25.6.0)(typescript@5.9.3)) typescript: specifier: catalog:tools version: 5.9.3 @@ -331,7 +422,7 @@ importers: version: 30.0.0 '@types/node': specifier: catalog:tools - version: 24.12.0 + version: 25.6.0 esbuild: specifier: catalog:tools version: 0.28.0 @@ -340,50 +431,111 @@ importers: version: 9.39.4(jiti@2.6.1) jest: specifier: catalog:test - version: 30.3.0(@types/node@24.12.0)(ts-node@10.9.2(@types/node@24.12.0)(typescript@5.9.3)) + version: 30.3.0(@types/node@25.6.0)(ts-node@10.9.2(@types/node@25.6.0)(typescript@5.9.3)) jest-html-reporter: specifier: catalog:test - version: 4.4.0(jest@30.3.0(@types/node@24.12.0)(ts-node@10.9.2(@types/node@24.12.0)(typescript@5.9.3))) + version: 4.4.0(jest@30.3.0(@types/node@25.6.0)(ts-node@10.9.2(@types/node@25.6.0)(typescript@5.9.3))) ts-jest: specifier: catalog:test - version: 29.4.9(@babel/core@7.29.0)(@jest/transform@30.3.0)(@jest/types@30.3.0)(babel-jest@30.3.0(@babel/core@7.29.0))(esbuild@0.28.0)(jest-util@30.3.0)(jest@30.3.0(@types/node@24.12.0)(ts-node@10.9.2(@types/node@24.12.0)(typescript@5.9.3)))(typescript@5.9.3) + version: 29.4.9(@babel/core@7.29.0)(@jest/transform@30.3.0)(@jest/types@30.3.0)(babel-jest@30.3.0(@babel/core@7.29.0))(esbuild@0.28.0)(jest-util@30.3.0)(jest@30.3.0(@types/node@25.6.0)(ts-node@10.9.2(@types/node@25.6.0)(typescript@5.9.3)))(typescript@5.9.3) typescript: specifier: catalog:tools version: 5.9.3 - src/logger: + lambdas/perf-runner-lambda: dependencies: - pino: + '@aws-crypto/sha256-js': + specifier: catalog:aws + version: 5.2.0 + '@aws-sdk/client-cloudwatch-logs': + specifier: catalog:aws + version: 3.1026.0 + '@aws-sdk/client-sqs': + specifier: catalog:aws + version: 3.1026.0 + '@aws-sdk/credential-providers': + specifier: catalog:aws + version: 3.1026.0 + '@nhs-notify-client-callbacks/logger': + specifier: workspace:* + version: link:../../src/logger + '@nhs-notify-client-callbacks/models': + specifier: workspace:* + version: link:../../src/models + '@redis/client': specifier: catalog:app - version: 10.3.1 + version: 1.6.1 + '@smithy/signature-v4': + specifier: catalog:aws + version: 5.3.13 devDependencies: '@tsconfig/node22': specifier: catalog:tools version: 22.0.5 + '@types/aws-lambda': + specifier: catalog:tools + version: 8.10.161 '@types/jest': specifier: catalog:test version: 30.0.0 '@types/node': specifier: catalog:tools - version: 24.12.0 + version: 25.6.0 + esbuild: + specifier: catalog:tools + version: 0.28.0 eslint: specifier: catalog:lint version: 9.39.4(jiti@2.6.1) jest: specifier: catalog:test - version: 30.3.0(@types/node@24.12.0)(ts-node@10.9.2(@types/node@24.12.0)(typescript@5.9.3)) + version: 30.3.0(@types/node@25.6.0)(ts-node@10.9.2(@types/node@25.6.0)(typescript@5.9.3)) ts-jest: specifier: catalog:test - version: 29.4.9(@babel/core@7.29.0)(@jest/transform@30.3.0)(@jest/types@30.3.0)(babel-jest@30.3.0(@babel/core@7.29.0))(esbuild@0.28.0)(jest-util@30.3.0)(jest@30.3.0(@types/node@24.12.0)(ts-node@10.9.2(@types/node@24.12.0)(typescript@5.9.3)))(typescript@5.9.3) + version: 29.4.9(@babel/core@7.29.0)(@jest/transform@30.3.0)(@jest/types@30.3.0)(babel-jest@30.3.0(@babel/core@7.29.0))(esbuild@0.28.0)(jest-util@30.3.0)(jest@30.3.0(@types/node@25.6.0)(ts-node@10.9.2(@types/node@25.6.0)(typescript@5.9.3)))(typescript@5.9.3) typescript: specifier: catalog:tools version: 5.9.3 - src/models: + src/config-subscription-cache: dependencies: - zod: + '@aws-sdk/client-s3': + specifier: catalog:aws + version: 3.1029.0 + '@nhs-notify-client-callbacks/logger': + specifier: workspace:* + version: link:../logger + '@nhs-notify-client-callbacks/models': + specifier: workspace:* + version: link:../models + devDependencies: + '@tsconfig/node22': + specifier: catalog:tools + version: 22.0.5 + '@types/jest': + specifier: catalog:test + version: 30.0.0 + '@types/node': + specifier: catalog:tools + version: 25.6.0 + eslint: + specifier: catalog:lint + version: 9.39.4(jiti@2.6.1) + jest: + specifier: catalog:test + version: 30.3.0(@types/node@25.6.0)(ts-node@10.9.2(@types/node@25.6.0)(typescript@5.9.3)) + ts-jest: + specifier: catalog:test + version: 29.4.9(@babel/core@7.29.0)(@jest/transform@30.3.0)(@jest/types@30.3.0)(babel-jest@30.3.0(@babel/core@7.29.0))(esbuild@0.28.0)(jest-util@30.3.0)(jest@30.3.0(@types/node@25.6.0)(ts-node@10.9.2(@types/node@25.6.0)(typescript@5.9.3)))(typescript@5.9.3) + typescript: + specifier: catalog:tools + version: 5.9.3 + + src/logger: + dependencies: + pino: specifier: catalog:app - version: 4.3.6 + version: 10.3.1 devDependencies: '@tsconfig/node22': specifier: catalog:tools @@ -393,43 +545,25 @@ importers: version: 30.0.0 '@types/node': specifier: catalog:tools - version: 24.12.0 + version: 25.6.0 eslint: specifier: catalog:lint version: 9.39.4(jiti@2.6.1) jest: specifier: catalog:test - version: 30.3.0(@types/node@24.12.0)(ts-node@10.9.2(@types/node@24.12.0)(typescript@5.9.3)) + version: 30.3.0(@types/node@25.6.0)(ts-node@10.9.2(@types/node@25.6.0)(typescript@5.9.3)) ts-jest: specifier: catalog:test - version: 29.4.9(@babel/core@7.29.0)(@jest/transform@30.3.0)(@jest/types@30.3.0)(babel-jest@30.3.0(@babel/core@7.29.0))(esbuild@0.28.0)(jest-util@30.3.0)(jest@30.3.0(@types/node@24.12.0)(ts-node@10.9.2(@types/node@24.12.0)(typescript@5.9.3)))(typescript@5.9.3) + version: 29.4.9(@babel/core@7.29.0)(@jest/transform@30.3.0)(@jest/types@30.3.0)(babel-jest@30.3.0(@babel/core@7.29.0))(esbuild@0.28.0)(jest-util@30.3.0)(jest@30.3.0(@types/node@25.6.0)(ts-node@10.9.2(@types/node@25.6.0)(typescript@5.9.3)))(typescript@5.9.3) typescript: specifier: catalog:tools version: 5.9.3 - tests/integration: + src/models: dependencies: - '@aws-sdk/client-cloudwatch': - specifier: catalog:aws - version: 3.1029.0 - '@aws-sdk/client-cloudwatch-logs': - specifier: catalog:aws - version: 3.1026.0 - '@aws-sdk/client-sqs': - specifier: catalog:aws - version: 3.1026.0 - '@nhs-notify-client-callbacks/logger': - specifier: workspace:* - version: link:../../src/logger - '@nhs-notify-client-callbacks/models': - specifier: workspace:* - version: link:../../src/models - '@nhs-notify-client-callbacks/test-support': - specifier: workspace:* - version: link:../test-support - async-wait-until: + zod: specifier: catalog:app - version: 2.0.31 + version: 4.3.6 devDependencies: '@tsconfig/node22': specifier: catalog:tools @@ -439,25 +573,34 @@ importers: version: 30.0.0 '@types/node': specifier: catalog:tools - version: 24.12.0 + version: 25.6.0 eslint: specifier: catalog:lint version: 9.39.4(jiti@2.6.1) jest: specifier: catalog:test - version: 30.3.0(@types/node@24.12.0)(ts-node@10.9.2(@types/node@24.12.0)(typescript@5.9.3)) + version: 30.3.0(@types/node@25.6.0)(ts-node@10.9.2(@types/node@25.6.0)(typescript@5.9.3)) + ts-jest: + specifier: catalog:test + version: 29.4.9(@babel/core@7.29.0)(@jest/transform@30.3.0)(@jest/types@30.3.0)(babel-jest@30.3.0(@babel/core@7.29.0))(esbuild@0.28.0)(jest-util@30.3.0)(jest@30.3.0(@types/node@25.6.0)(ts-node@10.9.2(@types/node@25.6.0)(typescript@5.9.3)))(typescript@5.9.3) typescript: specifier: catalog:tools version: 5.9.3 - tests/performance: + tests/integration: dependencies: + '@aws-sdk/client-cloudwatch': + specifier: catalog:aws + version: 3.1029.0 '@aws-sdk/client-cloudwatch-logs': specifier: catalog:aws version: 3.1026.0 '@aws-sdk/client-sqs': specifier: catalog:aws version: 3.1026.0 + '@nhs-notify-client-callbacks/logger': + specifier: workspace:* + version: link:../../src/logger '@nhs-notify-client-callbacks/models': specifier: workspace:* version: link:../../src/models @@ -476,13 +619,13 @@ importers: version: 30.0.0 '@types/node': specifier: catalog:tools - version: 24.12.0 + version: 25.6.0 eslint: specifier: catalog:lint version: 9.39.4(jiti@2.6.1) jest: specifier: catalog:test - version: 30.3.0(@types/node@24.12.0)(ts-node@10.9.2(@types/node@24.12.0)(typescript@5.9.3)) + version: 30.3.0(@types/node@25.6.0)(ts-node@10.9.2(@types/node@25.6.0)(typescript@5.9.3)) typescript: specifier: catalog:tools version: 5.9.3 @@ -504,7 +647,7 @@ importers: version: 22.0.5 '@types/node': specifier: catalog:tools - version: 24.12.0 + version: 25.6.0 eslint: specifier: catalog:lint version: 9.39.4(jiti@2.6.1) @@ -517,9 +660,6 @@ importers: '@aws-sdk/client-s3': specifier: catalog:aws version: 3.1029.0 - '@aws-sdk/client-ssm': - specifier: catalog:aws - version: 3.1029.0 '@aws-sdk/client-sts': specifier: catalog:aws version: 3.1026.0 @@ -529,6 +669,9 @@ importers: '@nhs-notify-client-callbacks/models': specifier: workspace:* version: link:../../src/models + picocolors: + specifier: catalog:app + version: 1.1.1 table: specifier: catalog:app version: 6.9.0 @@ -539,12 +682,15 @@ importers: specifier: catalog:app version: 4.3.6 devDependencies: + '@smithy/types': + specifier: catalog:aws + version: 4.14.0 '@types/jest': specifier: catalog:test version: 30.0.0 '@types/node': specifier: catalog:tools - version: 24.12.0 + version: 25.6.0 '@types/yargs': specifier: catalog:tools version: 17.0.35 @@ -553,10 +699,10 @@ importers: version: 9.39.4(jiti@2.6.1) jest: specifier: catalog:test - version: 30.3.0(@types/node@24.12.0)(ts-node@10.9.2(@types/node@24.12.0)(typescript@5.9.3)) + version: 30.3.0(@types/node@25.6.0)(ts-node@10.9.2(@types/node@25.6.0)(typescript@5.9.3)) ts-jest: specifier: catalog:test - version: 29.4.9(@babel/core@7.29.0)(@jest/transform@30.3.0)(@jest/types@30.3.0)(babel-jest@30.3.0(@babel/core@7.29.0))(esbuild@0.28.0)(jest-util@30.3.0)(jest@30.3.0(@types/node@24.12.0)(ts-node@10.9.2(@types/node@24.12.0)(typescript@5.9.3)))(typescript@5.9.3) + version: 29.4.9(@babel/core@7.29.0)(@jest/transform@30.3.0)(@jest/types@30.3.0)(babel-jest@30.3.0(@babel/core@7.29.0))(esbuild@0.28.0)(jest-util@30.3.0)(jest@30.3.0(@types/node@25.6.0)(ts-node@10.9.2(@types/node@25.6.0)(typescript@5.9.3)))(typescript@5.9.3) tsx: specifier: catalog:tools version: 4.21.0 @@ -609,10 +755,6 @@ packages: resolution: {integrity: sha512-b7z2WI1tqObk4U7vUbmBfXIeFhxKbFr7xQ4rWi879iFl5aSPvpd1WAmLi6z1boVKTEwEqHALuE5MyGBHhOCy5A==} engines: {node: '>=20.0.0'} - '@aws-sdk/client-ssm@3.1029.0': - resolution: {integrity: sha512-LthC1Dkh7r4ihZ7EI+6Sms9Ml0XQXoBZbw5LmtT1EJElriMugAfMnG5pKzDAcWpLiZgVBSZVai7moQR/QM/cCw==} - engines: {node: '>=20.0.0'} - '@aws-sdk/client-sts@3.1026.0': resolution: {integrity: sha512-kyqU8QMroxh6vc22cLWRT/wk5I142PiwGpGosnqJ36mLmiLtn84HuDYyivaNRAjKWIUQNlWeB0HHSoeqbn2O6Q==} engines: {node: '>=20.0.0'} @@ -1697,6 +1839,10 @@ packages: resolution: {integrity: sha512-QNqXyfVS2wm9hweSYD2O7F0G06uurj9kZ96TRQE5Y9hU7+tgdZwIkbAKc5Ocy1HxEY2kuDQa6cQ1WRs/O5LFKA==} engines: {node: ^12.20.0 || ^14.18.0 || >=16.0.0} + '@redis/client@1.6.1': + resolution: {integrity: sha512-/KCsg3xSlR+nCK8/8ZYSknYxvXHwubJrU82F3Lm1Fp6789VQ0/3RJKfsmRXjqfaTA++23CvC3hqmqe/2GEt6Kw==} + engines: {node: '>=14'} + '@rtsao/scc@1.1.0': resolution: {integrity: sha512-zt6OdqaDoOnJ1ZYsCYGt9YmWzDXl4vQdKTyJev62gFhRGKdx7mcT54V9KIjg+d2wi9EXsPvAPKe7i7WjfVWB8g==} @@ -1985,11 +2131,11 @@ packages: '@types/json5@0.0.29': resolution: {integrity: sha512-dRLjCWHYg4oaA77cxO64oO+7JwCwnIzkZPdrrC71jQmQtlhM556pwKo5bUzqvZndkVbeFLIIi+9TC40JNF5hNQ==} - '@types/node@24.12.0': - resolution: {integrity: sha512-GYDxsZi3ChgmckRT9HPU0WEhKLP08ev/Yfcq2AstjrDASOYCSXeyjDsHg4v5t4jOj7cyDX3vmprafKlWIG9MXQ==} + '@types/node-forge@1.3.14': + resolution: {integrity: sha512-mhVF2BnD4BO+jtOp7z1CdzaK4mbuK0LLQYAvdOLqHTavxFNq4zA1EmYkpnFjP8HOUzedfQkRnp0E2ulSAYSzAw==} - '@types/node@25.5.0': - resolution: {integrity: sha512-jp2P3tQMSxWugkCUKLRPVUpGaL5MVFwF8RDuSRztfwgN1wmqJeMSbKlnEtQqU8UrhTmzEmZdu2I6v2dpp7XIxw==} + '@types/node@25.6.0': + resolution: {integrity: sha512-+qIYRKdNYJwY3vRCZMdJbPLJAtGjQBudzZzdzwQYkEPQd+PJGixUL5QfvCLDaULoLv+RhT3LDkwEfKaAkgSmNQ==} '@types/stack-utils@2.0.3': resolution: {integrity: sha512-9aEbYZ3TbYMznPdcdr3SmIrLXwC/AKZXQeCf9Pgao5CKb8CyHuEX5jzWPTkvregvhRJHcpRO6BFoGW9ycaOkYw==} @@ -2061,6 +2207,7 @@ packages: '@ungap/structured-clone@1.3.0': resolution: {integrity: sha512-WmoN8qaIAo7WTYWbAZuG8PYEhn5fkz7dZrqTBZ7dtt//lL2Gwms1IcnQ5yHqjDfX8Ft5j4YzDM23f87zBfDe9g==} + deprecated: Potential CWE-502 - Update to 1.3.1 or higher '@unrs/resolver-binding-android-arm-eabi@1.11.1': resolution: {integrity: sha512-ppLRUgHVaGRWUx0R0Ut06Mjo9gBaBkg3v/8AxusGLhsIotbBLuRk51rAzqLC8gq6NyyAojEXglNjzf6R948DNw==} @@ -2453,6 +2600,10 @@ packages: resolution: {integrity: sha512-uyzC+PpMMRawbouHO+3mlisr3QfEDObmo2pN4oTTF6dZncZgpIzdasZx0tRBFI1dMsqCLZZXMtz8cUuvYqHdbw==} engines: {node: '>=20 <=24'} + cluster-key-slot@1.1.2: + resolution: {integrity: sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==} + engines: {node: '>=0.10.0'} + co@4.6.0: resolution: {integrity: sha512-QVb0dM5HvG+uaxitm8wONl7jltx8dqhfU33DcqtOZcLSVIKSDDLDi7+0LbAKiyI8hD9u42m2YxXSkMGWThaecQ==} engines: {iojs: '>= 1.0.0', node: '>= 0.12.0'} @@ -2956,6 +3107,9 @@ packages: picomatch: optional: true + fengari@0.1.5: + resolution: {integrity: sha512-0DS4Nn4rV8qyFlQCpKK8brT61EUtswynrpfFTcgLErcilBIBskSMQ86fO2WVuybr14ywyKdRjv91FiRZwnEuvQ==} + fflate@0.8.1: resolution: {integrity: sha512-/exOvEuc+/iaUm105QIiOt4LpBdMTWsXxqR0HDF35vx3fmaKzw7354gTilCh5rkzEt8WYyG//ku3h3nRmd7CHQ==} @@ -3024,6 +3178,10 @@ packages: resolution: {integrity: sha512-SFdFmIJi+ybC0vjlHN0ZGVGHc3lgE0DxPAT0djjVg+kjOnSqclqmj0KQ7ykTOLP6YxoqOvuAODGdcHJn+43q3g==} engines: {node: '>= 0.4'} + generic-pool@3.9.0: + resolution: {integrity: sha512-hymDOu5B53XvN4QT9dBmZxPX4CWhBPPLguTZ9MMFeFa/Kg0xWVfylOVNlJji/E7yTZWFd/q9GO5TxDLq156D7g==} + engines: {node: '>= 4'} + gensync@1.0.0-beta.2: resolution: {integrity: sha512-3hN7NaskYvMDLQY55gnW3NQ+mesEAepTqlg+VEbj7zzqEMBVNhzcGYYeqFo/TlYz6eQiFcp1HcsCZO+nGgS8zg==} engines: {node: '>=6.9.0'} @@ -3675,6 +3833,10 @@ packages: resolution: {integrity: sha512-pyFS63ptit/P5WqUkt+UUfe+4oevH+bFeIiPPdfb0pFeYEu/1ELnJu5l+5EcTKYL5M7zaAa7S8ddywgXypqKCw==} engines: {node: '>= 0.4'} + node-forge@1.4.0: + resolution: {integrity: sha512-LarFH0+6VfriEhqMMcLX2F7SwSXeWwnEAJEsYm5QKWchiVYVvJyV9v7UDvUv+w5HO23ZpQTXDv/GxdDdMyOuoQ==} + engines: {node: '>= 6.13.0'} + node-int64@0.4.0: resolution: {integrity: sha512-O5lz91xSOeoXP6DulyHfllpq+Eg00MWitZIbtPfoSEvqIHdl5gfcY6hYzDWnj0qD5tz52PI08u9qUvSVeUBeHw==} @@ -3888,6 +4050,10 @@ packages: react-is@19.0.0: resolution: {integrity: sha512-H91OHcwjZsbq3ClIDHMzBShc1rotbfACdWENsmEf0IFvZ3FgGPtdHMcsv45bQ1hAbgdfiA8SnxTKfDS+x/8m2g==} + readline-sync@1.4.10: + resolution: {integrity: sha512-gNva8/6UAe8QYepIQH/jQ2qn91Qj0B9sYjMBBs3QOB8F2CXcKgLxQaJRP76sWVRQt+QU+8fAkCbCvjjMFu7Ycw==} + engines: {node: '>= 0.8.0'} + real-require@0.2.0: resolution: {integrity: sha512-57frrGM/OCTLqLOAh0mhVA9VBMHd+9U7Zb2THMGdBUoZVOtGbJzjxsYGDJ3A9AYYCP4hn6y1TVbaOfzWtm5GFg==} engines: {node: '>= 12.13.0'} @@ -4060,6 +4226,9 @@ packages: sprintf-js@1.0.3: resolution: {integrity: sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==} + sprintf-js@1.1.3: + resolution: {integrity: sha512-Oo+0REFV59/rz3gfJNKQiBlwfHaSESl1pcGyABQsnnIfWOFt6JNj5gCog2U6MLZ//IGYD+nA8nI+mTShREReaA==} + stable-hash-x@0.2.0: resolution: {integrity: sha512-o3yWv49B/o4QZk5ZcsALc6t0+eCelPc44zZsLtCQnZPDwFpDYSWcDnrv2TtMmMbQ7uKo3J0HTURCqckw23czNQ==} engines: {node: '>=12.0.0'} @@ -4178,6 +4347,10 @@ packages: resolution: {integrity: sha512-j2Zq4NyQYG5XMST4cbs02Ak8iJUdxRM0XI5QyxXuZOzKOINmWurp3smXu3y5wDcJrptwpSjgXHzIQxR0omXljQ==} engines: {node: '>=12.0.0'} + tmp@0.2.5: + resolution: {integrity: sha512-voyz6MApa1rQGUxT3E+BK7/ROe8itEx7vD8/HEvt4xwXucvQ5G5oeEiHkmHZJuBO21RpOf+YYm9MOivj709jow==} + engines: {node: '>=14.14'} + tmpl@1.0.5: resolution: {integrity: sha512-3f0uOEAQwIqGuWW2MVzYg8fV/QNnc/IpuJNG837rLuczAaLVHslWHZQj4IGiEl5Hs3kkbhwL9Ab7Hrsmuj+Smw==} @@ -4310,11 +4483,8 @@ packages: resolution: {integrity: sha512-nWJ91DjeOkej/TA8pXQ3myruKpKEYgqvpw9lz4OPHj/NWFNluYrjbz9j01CJ8yKQd2g4jFoOkINCTW2I5LEEyw==} engines: {node: '>= 0.4'} - undici-types@7.16.0: - resolution: {integrity: sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw==} - - undici-types@7.18.2: - resolution: {integrity: sha512-AsuCzffGHJybSaRrmr5eHr81mwJU3kjw6M+uprWvCXiNeN9SOGwQ3Jn8jb8m3Z6izVgknn1R0FTCEAP2QrLY/w==} + undici-types@7.19.2: + resolution: {integrity: sha512-qYVnV5OEm2AW8cJMCpdV20CDyaN3g0AjDlOGf1OW4iaDEx8MwdtChUp4zu4H0VP3nDRF/8RKWH+IPp9uW0YGZg==} unrs-resolver@1.11.1: resolution: {integrity: sha512-bSjt9pjaEBnNiGgc9rUiHGKv5l4/TGzDmYw3RhnkJGtLhbnnA/5qJj7x3dNDCRx/PJxu774LlH8lCOlB4hEfKg==} @@ -4333,6 +4503,7 @@ packages: uuid@8.3.2: resolution: {integrity: sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==} + deprecated: uuid@10 and below is no longer supported. For ESM codebases, update to uuid@latest. For CommonJS codebases, use uuid@11 (but be aware this version will likely be deprecated in 2028). hasBin: true v8-compile-cache-lib@3.0.1: @@ -4418,6 +4589,9 @@ packages: yallist@3.1.1: resolution: {integrity: sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==} + yallist@4.0.0: + resolution: {integrity: sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==} + yaml@2.8.3: resolution: {integrity: sha512-AvbaCLOO2Otw/lW5bmh9d/WEdcDFdQp2Z2ZUH3pX9U2ihyUY0nvLv7J6TrWowklRGPYbB/IuIMfYgxaCPg5Bpg==} engines: {node: '>= 14.6'} @@ -4748,51 +4922,6 @@ snapshots: transitivePeerDependencies: - aws-crt - '@aws-sdk/client-ssm@3.1029.0': - dependencies: - '@aws-crypto/sha256-browser': 5.2.0 - '@aws-crypto/sha256-js': 5.2.0 - '@aws-sdk/core': 3.973.27 - '@aws-sdk/credential-provider-node': 3.972.30 - '@aws-sdk/middleware-host-header': 3.972.9 - '@aws-sdk/middleware-logger': 3.972.9 - '@aws-sdk/middleware-recursion-detection': 3.972.10 - '@aws-sdk/middleware-user-agent': 3.972.29 - '@aws-sdk/region-config-resolver': 3.972.11 - '@aws-sdk/types': 3.973.7 - '@aws-sdk/util-endpoints': 3.996.6 - '@aws-sdk/util-user-agent-browser': 3.972.9 - '@aws-sdk/util-user-agent-node': 3.973.15 - '@smithy/config-resolver': 4.4.14 - '@smithy/core': 3.23.14 - '@smithy/fetch-http-handler': 5.3.16 - '@smithy/hash-node': 4.2.13 - '@smithy/invalid-dependency': 4.2.13 - '@smithy/middleware-content-length': 4.2.13 - '@smithy/middleware-endpoint': 4.4.29 - '@smithy/middleware-retry': 4.5.0 - '@smithy/middleware-serde': 4.2.17 - '@smithy/middleware-stack': 4.2.13 - '@smithy/node-config-provider': 4.3.13 - '@smithy/node-http-handler': 4.5.2 - '@smithy/protocol-http': 5.3.13 - '@smithy/smithy-client': 4.12.9 - '@smithy/types': 4.14.0 - '@smithy/url-parser': 4.2.13 - '@smithy/util-base64': 4.3.2 - '@smithy/util-body-length-browser': 4.2.2 - '@smithy/util-body-length-node': 4.2.3 - '@smithy/util-defaults-mode-browser': 4.3.45 - '@smithy/util-defaults-mode-node': 4.2.49 - '@smithy/util-endpoints': 3.3.4 - '@smithy/util-middleware': 4.2.13 - '@smithy/util-retry': 4.3.0 - '@smithy/util-utf8': 4.2.2 - '@smithy/util-waiter': 4.2.15 - tslib: 2.8.1 - transitivePeerDependencies: - - aws-crt - '@aws-sdk/client-sts@3.1026.0': dependencies: '@aws-crypto/sha256-browser': 5.2.0 @@ -5664,13 +5793,13 @@ snapshots: '@jest/console@30.3.0': dependencies: '@jest/types': 30.3.0 - '@types/node': 25.5.0 + '@types/node': 25.6.0 chalk: 4.1.2 jest-message-util: 30.3.0 jest-util: 30.3.0 slash: 3.0.0 - '@jest/core@30.3.0(ts-node@10.9.2(@types/node@24.12.0)(typescript@5.9.3))': + '@jest/core@30.3.0(ts-node@10.9.2(@types/node@25.6.0)(typescript@5.9.3))': dependencies: '@jest/console': 30.3.0 '@jest/pattern': 30.0.1 @@ -5678,49 +5807,14 @@ snapshots: '@jest/test-result': 30.3.0 '@jest/transform': 30.3.0 '@jest/types': 30.3.0 - '@types/node': 25.5.0 + '@types/node': 25.6.0 ansi-escapes: 4.3.2 chalk: 4.1.2 ci-info: 4.4.0 exit-x: 0.2.2 graceful-fs: 4.2.11 jest-changed-files: 30.3.0 - jest-config: 30.3.0(@types/node@25.5.0)(ts-node@10.9.2(@types/node@24.12.0)(typescript@5.9.3)) - jest-haste-map: 30.3.0 - jest-message-util: 30.3.0 - jest-regex-util: 30.0.1 - jest-resolve: 30.3.0 - jest-resolve-dependencies: 30.3.0 - jest-runner: 30.3.0 - jest-runtime: 30.3.0 - jest-snapshot: 30.3.0 - jest-util: 30.3.0 - jest-validate: 30.3.0 - jest-watcher: 30.3.0 - pretty-format: 30.3.0 - slash: 3.0.0 - transitivePeerDependencies: - - babel-plugin-macros - - esbuild-register - - supports-color - - ts-node - - '@jest/core@30.3.0(ts-node@10.9.2(@types/node@25.5.0)(typescript@5.9.3))': - dependencies: - '@jest/console': 30.3.0 - '@jest/pattern': 30.0.1 - '@jest/reporters': 30.3.0 - '@jest/test-result': 30.3.0 - '@jest/transform': 30.3.0 - '@jest/types': 30.3.0 - '@types/node': 25.5.0 - ansi-escapes: 4.3.2 - chalk: 4.1.2 - ci-info: 4.4.0 - exit-x: 0.2.2 - graceful-fs: 4.2.11 - jest-changed-files: 30.3.0 - jest-config: 30.3.0(@types/node@25.5.0)(ts-node@10.9.2(@types/node@25.5.0)(typescript@5.9.3)) + jest-config: 30.3.0(@types/node@25.6.0)(ts-node@10.9.2(@types/node@25.6.0)(typescript@5.9.3)) jest-haste-map: 30.3.0 jest-message-util: 30.3.0 jest-regex-util: 30.0.1 @@ -5746,7 +5840,7 @@ snapshots: dependencies: '@jest/fake-timers': 30.3.0 '@jest/types': 30.3.0 - '@types/node': 25.5.0 + '@types/node': 25.6.0 jest-mock: 30.3.0 '@jest/expect-utils@30.3.0': @@ -5764,7 +5858,7 @@ snapshots: dependencies: '@jest/types': 30.3.0 '@sinonjs/fake-timers': 15.3.2 - '@types/node': 25.5.0 + '@types/node': 25.6.0 jest-message-util: 30.3.0 jest-mock: 30.3.0 jest-util: 30.3.0 @@ -5782,7 +5876,7 @@ snapshots: '@jest/pattern@30.0.1': dependencies: - '@types/node': 25.5.0 + '@types/node': 25.6.0 jest-regex-util: 30.0.1 '@jest/reporters@30.3.0': @@ -5793,7 +5887,7 @@ snapshots: '@jest/transform': 30.3.0 '@jest/types': 30.3.0 '@jridgewell/trace-mapping': 0.3.31 - '@types/node': 25.5.0 + '@types/node': 25.6.0 chalk: 4.1.2 collect-v8-coverage: 1.0.3 exit-x: 0.2.2 @@ -5869,7 +5963,7 @@ snapshots: '@jest/schemas': 30.0.5 '@types/istanbul-lib-coverage': 2.0.6 '@types/istanbul-reports': 3.0.4 - '@types/node': 25.5.0 + '@types/node': 25.6.0 '@types/yargs': 17.0.35 chalk: 4.1.2 @@ -6068,6 +6162,12 @@ snapshots: '@pkgr/core@0.2.9': {} + '@redis/client@1.6.1': + dependencies: + cluster-key-slot: 1.1.2 + generic-pool: 3.9.0 + yallist: 4.0.0 + '@rtsao/scc@1.1.0': {} '@sinclair/typebox@0.34.49': {} @@ -6495,13 +6595,13 @@ snapshots: '@types/json5@0.0.29': {} - '@types/node@24.12.0': + '@types/node-forge@1.3.14': dependencies: - undici-types: 7.16.0 + '@types/node': 25.6.0 - '@types/node@25.5.0': + '@types/node@25.6.0': dependencies: - undici-types: 7.18.2 + undici-types: 7.19.2 '@types/stack-utils@2.0.3': {} @@ -6984,6 +7084,8 @@ snapshots: util: 0.12.5 uuid: 8.3.2 + cluster-key-slot@1.1.2: {} + co@4.6.0: {} collect-v8-coverage@1.0.3: {} @@ -7418,13 +7520,13 @@ snapshots: - eslint-import-resolver-webpack - supports-color - eslint-plugin-jest@29.15.2(@typescript-eslint/eslint-plugin@8.58.0(@typescript-eslint/parser@8.58.0(eslint@9.39.4(jiti@2.6.1))(typescript@5.9.3))(eslint@9.39.4(jiti@2.6.1))(typescript@5.9.3))(eslint@9.39.4(jiti@2.6.1))(jest@30.3.0(@types/node@25.5.0)(ts-node@10.9.2(@types/node@25.5.0)(typescript@5.9.3)))(typescript@5.9.3): + eslint-plugin-jest@29.15.2(@typescript-eslint/eslint-plugin@8.58.0(@typescript-eslint/parser@8.58.0(eslint@9.39.4(jiti@2.6.1))(typescript@5.9.3))(eslint@9.39.4(jiti@2.6.1))(typescript@5.9.3))(eslint@9.39.4(jiti@2.6.1))(jest@30.3.0(@types/node@25.6.0)(ts-node@10.9.2(@types/node@25.6.0)(typescript@5.9.3)))(typescript@5.9.3): dependencies: '@typescript-eslint/utils': 8.58.0(eslint@9.39.4(jiti@2.6.1))(typescript@5.9.3) eslint: 9.39.4(jiti@2.6.1) optionalDependencies: '@typescript-eslint/eslint-plugin': 8.58.0(@typescript-eslint/parser@8.58.0(eslint@9.39.4(jiti@2.6.1))(typescript@5.9.3))(eslint@9.39.4(jiti@2.6.1))(typescript@5.9.3) - jest: 30.3.0(@types/node@25.5.0)(ts-node@10.9.2(@types/node@25.5.0)(typescript@5.9.3)) + jest: 30.3.0(@types/node@25.6.0)(ts-node@10.9.2(@types/node@25.6.0)(typescript@5.9.3)) typescript: 5.9.3 transitivePeerDependencies: - supports-color @@ -7704,6 +7806,12 @@ snapshots: optionalDependencies: picomatch: 4.0.4 + fengari@0.1.5: + dependencies: + readline-sync: 1.4.10 + sprintf-js: 1.1.3 + tmp: 0.2.5 + fflate@0.8.1: {} file-entry-cache@8.0.0: @@ -7768,6 +7876,8 @@ snapshots: generator-function@2.0.1: {} + generic-pool@3.9.0: {} + gensync@1.0.0-beta.2: {} get-caller-file@2.0.5: {} @@ -8121,7 +8231,7 @@ snapshots: '@jest/expect': 30.3.0 '@jest/test-result': 30.3.0 '@jest/types': 30.3.0 - '@types/node': 25.5.0 + '@types/node': 25.6.0 chalk: 4.1.2 co: 4.6.0 dedent: 1.7.2 @@ -8141,34 +8251,15 @@ snapshots: - babel-plugin-macros - supports-color - jest-cli@30.3.0(@types/node@24.12.0)(ts-node@10.9.2(@types/node@24.12.0)(typescript@5.9.3)): - dependencies: - '@jest/core': 30.3.0(ts-node@10.9.2(@types/node@24.12.0)(typescript@5.9.3)) - '@jest/test-result': 30.3.0 - '@jest/types': 30.3.0 - chalk: 4.1.2 - exit-x: 0.2.2 - import-local: 3.2.0 - jest-config: 30.3.0(@types/node@24.12.0)(ts-node@10.9.2(@types/node@24.12.0)(typescript@5.9.3)) - jest-util: 30.3.0 - jest-validate: 30.3.0 - yargs: 17.7.2 - transitivePeerDependencies: - - '@types/node' - - babel-plugin-macros - - esbuild-register - - supports-color - - ts-node - - jest-cli@30.3.0(@types/node@25.5.0)(ts-node@10.9.2(@types/node@25.5.0)(typescript@5.9.3)): + jest-cli@30.3.0(@types/node@25.6.0)(ts-node@10.9.2(@types/node@25.6.0)(typescript@5.9.3)): dependencies: - '@jest/core': 30.3.0(ts-node@10.9.2(@types/node@25.5.0)(typescript@5.9.3)) + '@jest/core': 30.3.0(ts-node@10.9.2(@types/node@25.6.0)(typescript@5.9.3)) '@jest/test-result': 30.3.0 '@jest/types': 30.3.0 chalk: 4.1.2 exit-x: 0.2.2 import-local: 3.2.0 - jest-config: 30.3.0(@types/node@25.5.0)(ts-node@10.9.2(@types/node@25.5.0)(typescript@5.9.3)) + jest-config: 30.3.0(@types/node@25.6.0)(ts-node@10.9.2(@types/node@25.6.0)(typescript@5.9.3)) jest-util: 30.3.0 jest-validate: 30.3.0 yargs: 17.7.2 @@ -8179,7 +8270,7 @@ snapshots: - supports-color - ts-node - jest-config@30.3.0(@types/node@24.12.0)(ts-node@10.9.2(@types/node@24.12.0)(typescript@5.9.3)): + jest-config@30.3.0(@types/node@25.6.0)(ts-node@10.9.2(@types/node@25.6.0)(typescript@5.9.3)): dependencies: '@babel/core': 7.29.0 '@jest/get-type': 30.1.0 @@ -8205,72 +8296,8 @@ snapshots: slash: 3.0.0 strip-json-comments: 3.1.1 optionalDependencies: - '@types/node': 24.12.0 - ts-node: 10.9.2(@types/node@24.12.0)(typescript@5.9.3) - transitivePeerDependencies: - - babel-plugin-macros - - supports-color - - jest-config@30.3.0(@types/node@25.5.0)(ts-node@10.9.2(@types/node@24.12.0)(typescript@5.9.3)): - dependencies: - '@babel/core': 7.29.0 - '@jest/get-type': 30.1.0 - '@jest/pattern': 30.0.1 - '@jest/test-sequencer': 30.3.0 - '@jest/types': 30.3.0 - babel-jest: 30.3.0(@babel/core@7.29.0) - chalk: 4.1.2 - ci-info: 4.4.0 - deepmerge: 4.3.1 - glob: 10.5.0 - graceful-fs: 4.2.11 - jest-circus: 30.3.0 - jest-docblock: 30.2.0 - jest-environment-node: 30.3.0 - jest-regex-util: 30.0.1 - jest-resolve: 30.3.0 - jest-runner: 30.3.0 - jest-util: 30.3.0 - jest-validate: 30.3.0 - parse-json: 5.2.0 - pretty-format: 30.3.0 - slash: 3.0.0 - strip-json-comments: 3.1.1 - optionalDependencies: - '@types/node': 25.5.0 - ts-node: 10.9.2(@types/node@24.12.0)(typescript@5.9.3) - transitivePeerDependencies: - - babel-plugin-macros - - supports-color - - jest-config@30.3.0(@types/node@25.5.0)(ts-node@10.9.2(@types/node@25.5.0)(typescript@5.9.3)): - dependencies: - '@babel/core': 7.29.0 - '@jest/get-type': 30.1.0 - '@jest/pattern': 30.0.1 - '@jest/test-sequencer': 30.3.0 - '@jest/types': 30.3.0 - babel-jest: 30.3.0(@babel/core@7.29.0) - chalk: 4.1.2 - ci-info: 4.4.0 - deepmerge: 4.3.1 - glob: 10.5.0 - graceful-fs: 4.2.11 - jest-circus: 30.3.0 - jest-docblock: 30.2.0 - jest-environment-node: 30.3.0 - jest-regex-util: 30.0.1 - jest-resolve: 30.3.0 - jest-runner: 30.3.0 - jest-util: 30.3.0 - jest-validate: 30.3.0 - parse-json: 5.2.0 - pretty-format: 30.3.0 - slash: 3.0.0 - strip-json-comments: 3.1.1 - optionalDependencies: - '@types/node': 25.5.0 - ts-node: 10.9.2(@types/node@25.5.0)(typescript@5.9.3) + '@types/node': 25.6.0 + ts-node: 10.9.2(@types/node@25.6.0)(typescript@5.9.3) transitivePeerDependencies: - babel-plugin-macros - supports-color @@ -8299,7 +8326,7 @@ snapshots: '@jest/environment': 30.3.0 '@jest/fake-timers': 30.3.0 '@jest/types': 30.3.0 - '@types/node': 25.5.0 + '@types/node': 25.6.0 jest-mock: 30.3.0 jest-util: 30.3.0 jest-validate: 30.3.0 @@ -8307,7 +8334,7 @@ snapshots: jest-haste-map@30.3.0: dependencies: '@jest/types': 30.3.0 - '@types/node': 25.5.0 + '@types/node': 25.6.0 anymatch: 3.1.3 fb-watchman: 2.0.2 graceful-fs: 4.2.11 @@ -8319,27 +8346,13 @@ snapshots: optionalDependencies: fsevents: 2.3.3 - jest-html-reporter@4.4.0(jest@30.3.0(@types/node@24.12.0)(ts-node@10.9.2(@types/node@24.12.0)(typescript@5.9.3))): - dependencies: - '@jest/reporters': 30.3.0 - '@jest/test-result': 30.3.0 - '@jest/types': 30.3.0 - dateformat: 3.0.2 - jest: 30.3.0(@types/node@24.12.0)(ts-node@10.9.2(@types/node@24.12.0)(typescript@5.9.3)) - mkdirp: 1.0.4 - strip-ansi: 6.0.1 - xmlbuilder: 15.0.0 - transitivePeerDependencies: - - node-notifier - - supports-color - - jest-html-reporter@4.4.0(jest@30.3.0(@types/node@25.5.0)(ts-node@10.9.2(@types/node@25.5.0)(typescript@5.9.3))): + jest-html-reporter@4.4.0(jest@30.3.0(@types/node@25.6.0)(ts-node@10.9.2(@types/node@25.6.0)(typescript@5.9.3))): dependencies: '@jest/reporters': 30.3.0 '@jest/test-result': 30.3.0 '@jest/types': 30.3.0 dateformat: 3.0.2 - jest: 30.3.0(@types/node@25.5.0)(ts-node@10.9.2(@types/node@25.5.0)(typescript@5.9.3)) + jest: 30.3.0(@types/node@25.6.0)(ts-node@10.9.2(@types/node@25.6.0)(typescript@5.9.3)) mkdirp: 1.0.4 strip-ansi: 6.0.1 xmlbuilder: 15.0.0 @@ -8374,7 +8387,7 @@ snapshots: jest-mock@30.3.0: dependencies: '@jest/types': 30.3.0 - '@types/node': 25.5.0 + '@types/node': 25.6.0 jest-util: 30.3.0 jest-pnp-resolver@1.2.3(jest-resolve@30.3.0): @@ -8408,7 +8421,7 @@ snapshots: '@jest/test-result': 30.3.0 '@jest/transform': 30.3.0 '@jest/types': 30.3.0 - '@types/node': 25.5.0 + '@types/node': 25.6.0 chalk: 4.1.2 emittery: 0.13.1 exit-x: 0.2.2 @@ -8437,7 +8450,7 @@ snapshots: '@jest/test-result': 30.3.0 '@jest/transform': 30.3.0 '@jest/types': 30.3.0 - '@types/node': 25.5.0 + '@types/node': 25.6.0 chalk: 4.1.2 cjs-module-lexer: 2.2.0 collect-v8-coverage: 1.0.3 @@ -8484,7 +8497,7 @@ snapshots: jest-util@30.3.0: dependencies: '@jest/types': 30.3.0 - '@types/node': 25.5.0 + '@types/node': 25.6.0 chalk: 4.1.2 ci-info: 4.4.0 graceful-fs: 4.2.11 @@ -8503,7 +8516,7 @@ snapshots: dependencies: '@jest/test-result': 30.3.0 '@jest/types': 30.3.0 - '@types/node': 25.5.0 + '@types/node': 25.6.0 ansi-escapes: 4.3.2 chalk: 4.1.2 emittery: 0.13.1 @@ -8512,31 +8525,18 @@ snapshots: jest-worker@30.3.0: dependencies: - '@types/node': 25.5.0 + '@types/node': 25.6.0 '@ungap/structured-clone': 1.3.0 jest-util: 30.3.0 merge-stream: 2.0.0 supports-color: 8.1.1 - jest@30.3.0(@types/node@24.12.0)(ts-node@10.9.2(@types/node@24.12.0)(typescript@5.9.3)): - dependencies: - '@jest/core': 30.3.0(ts-node@10.9.2(@types/node@24.12.0)(typescript@5.9.3)) - '@jest/types': 30.3.0 - import-local: 3.2.0 - jest-cli: 30.3.0(@types/node@24.12.0)(ts-node@10.9.2(@types/node@24.12.0)(typescript@5.9.3)) - transitivePeerDependencies: - - '@types/node' - - babel-plugin-macros - - esbuild-register - - supports-color - - ts-node - - jest@30.3.0(@types/node@25.5.0)(ts-node@10.9.2(@types/node@25.5.0)(typescript@5.9.3)): + jest@30.3.0(@types/node@25.6.0)(ts-node@10.9.2(@types/node@25.6.0)(typescript@5.9.3)): dependencies: - '@jest/core': 30.3.0(ts-node@10.9.2(@types/node@25.5.0)(typescript@5.9.3)) + '@jest/core': 30.3.0(ts-node@10.9.2(@types/node@25.6.0)(typescript@5.9.3)) '@jest/types': 30.3.0 import-local: 3.2.0 - jest-cli: 30.3.0(@types/node@25.5.0)(ts-node@10.9.2(@types/node@25.5.0)(typescript@5.9.3)) + jest-cli: 30.3.0(@types/node@25.6.0)(ts-node@10.9.2(@types/node@25.6.0)(typescript@5.9.3)) transitivePeerDependencies: - '@types/node' - babel-plugin-macros @@ -8719,6 +8719,8 @@ snapshots: object.entries: 1.1.9 semver: 6.3.1 + node-forge@1.4.0: {} + node-int64@0.4.0: {} node-releases@2.0.36: {} @@ -8972,6 +8974,8 @@ snapshots: react-is@19.0.0: {} + readline-sync@1.4.10: {} + real-require@0.2.0: {} refa@0.12.1: @@ -9164,6 +9168,8 @@ snapshots: sprintf-js@1.0.3: {} + sprintf-js@1.1.3: {} + stable-hash-x@0.2.0: {} stack-utils@2.0.6: @@ -9303,6 +9309,8 @@ snapshots: fdir: 6.5.0(picomatch@4.0.4) picomatch: 4.0.4 + tmp@0.2.5: {} + tmpl@1.0.5: {} to-regex-range@5.0.1: @@ -9322,12 +9330,12 @@ snapshots: picomatch: 4.0.4 typescript: 5.9.3 - ts-jest@29.4.9(@babel/core@7.29.0)(@jest/transform@30.3.0)(@jest/types@30.3.0)(babel-jest@30.3.0(@babel/core@7.29.0))(esbuild@0.28.0)(jest-util@30.3.0)(jest@30.3.0(@types/node@24.12.0)(ts-node@10.9.2(@types/node@24.12.0)(typescript@5.9.3)))(typescript@5.9.3): + ts-jest@29.4.9(@babel/core@7.29.0)(@jest/transform@30.3.0)(@jest/types@30.3.0)(babel-jest@30.3.0(@babel/core@7.29.0))(esbuild@0.28.0)(jest-util@30.3.0)(jest@30.3.0(@types/node@25.6.0)(ts-node@10.9.2(@types/node@25.6.0)(typescript@5.9.3)))(typescript@5.9.3): dependencies: bs-logger: 0.2.6 fast-json-stable-stringify: 2.1.0 handlebars: 4.7.9 - jest: 30.3.0(@types/node@24.12.0)(ts-node@10.9.2(@types/node@24.12.0)(typescript@5.9.3)) + jest: 30.3.0(@types/node@25.6.0)(ts-node@10.9.2(@types/node@25.6.0)(typescript@5.9.3)) json5: 2.2.3 lodash.memoize: 4.1.2 make-error: 1.3.6 @@ -9343,54 +9351,14 @@ snapshots: esbuild: 0.28.0 jest-util: 30.3.0 - ts-jest@29.4.9(@babel/core@7.29.0)(@jest/transform@30.3.0)(@jest/types@30.3.0)(babel-jest@30.3.0(@babel/core@7.29.0))(esbuild@0.28.0)(jest-util@30.3.0)(jest@30.3.0(@types/node@25.5.0)(ts-node@10.9.2(@types/node@25.5.0)(typescript@5.9.3)))(typescript@5.9.3): - dependencies: - bs-logger: 0.2.6 - fast-json-stable-stringify: 2.1.0 - handlebars: 4.7.9 - jest: 30.3.0(@types/node@25.5.0)(ts-node@10.9.2(@types/node@25.5.0)(typescript@5.9.3)) - json5: 2.2.3 - lodash.memoize: 4.1.2 - make-error: 1.3.6 - semver: 7.7.4 - type-fest: 4.41.0 - typescript: 5.9.3 - yargs-parser: 21.1.1 - optionalDependencies: - '@babel/core': 7.29.0 - '@jest/transform': 30.3.0 - '@jest/types': 30.3.0 - babel-jest: 30.3.0(@babel/core@7.29.0) - esbuild: 0.28.0 - jest-util: 30.3.0 - - ts-node@10.9.2(@types/node@24.12.0)(typescript@5.9.3): - dependencies: - '@cspotcode/source-map-support': 0.8.1 - '@tsconfig/node10': 1.0.12 - '@tsconfig/node12': 1.0.11 - '@tsconfig/node14': 1.0.3 - '@tsconfig/node16': 1.0.4 - '@types/node': 24.12.0 - acorn: 8.16.0 - acorn-walk: 8.3.5 - arg: 4.1.3 - create-require: 1.1.1 - diff: 4.0.4 - make-error: 1.3.6 - typescript: 5.9.3 - v8-compile-cache-lib: 3.0.1 - yn: 3.1.1 - optional: true - - ts-node@10.9.2(@types/node@25.5.0)(typescript@5.9.3): + ts-node@10.9.2(@types/node@25.6.0)(typescript@5.9.3): dependencies: '@cspotcode/source-map-support': 0.8.1 '@tsconfig/node10': 1.0.12 '@tsconfig/node12': 1.0.11 '@tsconfig/node14': 1.0.3 '@tsconfig/node16': 1.0.4 - '@types/node': 25.5.0 + '@types/node': 25.6.0 acorn: 8.16.0 acorn-walk: 8.3.5 arg: 4.1.3 @@ -9487,9 +9455,7 @@ snapshots: has-symbols: 1.1.0 which-boxed-primitive: 1.1.1 - undici-types@7.16.0: {} - - undici-types@7.18.2: {} + undici-types@7.19.2: {} unrs-resolver@1.11.1: dependencies: @@ -9639,6 +9605,8 @@ snapshots: yallist@3.1.1: {} + yallist@4.0.0: {} + yaml@2.8.3: {} yargs-parser@20.2.9: {} diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index cc1bdeb4..f138cdcc 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -6,12 +6,22 @@ packages: - "tools/*" blockExoticSubdeps: true +overrides: + collect-v8-coverage: "^1.0.3" + "pretty-format>react-is": "19.0.0" + flatted: "^3.4.0" + fast-xml-parser: "^5.5.6" + "ts-jest>handlebars": "^4.7.9" + catalogs: app: + "@redis/client": "^1.5.14" async-wait-until: "^2.0.31" aws-embedded-metrics: "^4.2.1" cloudevents: "^10.0.0" + node-forge: "^1.3.1" p-map: "^4.0.0" + picocolors: "^1.1.1" pino: "^10.3.1" table: "^6.9.0" yargs: "^17.7.2" @@ -21,9 +31,11 @@ catalogs: "@aws-sdk/client-cloudwatch-logs": "^3.1023.0" "@aws-sdk/client-s3": "^3.1024.0" "@aws-sdk/client-sqs": "^3.1023.0" - "@aws-sdk/client-ssm": "^3.1025.0" + "@aws-crypto/sha256-js": "^5.2.0" "@aws-sdk/client-sts": "^3.1023.0" "@aws-sdk/credential-providers": "^3.1023.0" + "@smithy/signature-v4": "^5.0.0" + "@smithy/types": "^4.3.1" lint: "@eslint/js": "^9.39.4" "@stylistic/eslint-plugin": "^5.10.0" @@ -53,7 +65,8 @@ catalogs: tools: "@tsconfig/node22": "^22.0.5" "@types/aws-lambda": "^8.10.161" - "@types/node": "^24.12.0" + "@types/node": "^25.5.0" + "@types/node-forge": "^1.3.11" "@types/yargs": "^17.0.24" esbuild: "^0.28.0" knip: "^6.3.1" diff --git a/scripts/config/pre-commit.yaml b/scripts/config/pre-commit.yaml index 10bf21d0..cce00bde 100644 --- a/scripts/config/pre-commit.yaml +++ b/scripts/config/pre-commit.yaml @@ -8,6 +8,7 @@ repos: - id: check-added-large-files - id: check-symlinks - id: detect-private-key + exclude: 'lambdas/https-client-lambda/src/__tests__/tls-agent-factory\.test\.ts' - id: end-of-file-fixer - id: forbid-new-submodules - id: mixed-line-ending @@ -39,3 +40,10 @@ repos: entry: pnpm exec knip --no-progress language: system pass_filenames: false + - repo: local + hooks: + - id: check-lua-format + name: Check Lua format + entry: /usr/bin/env check=branch ./scripts/githooks/check-lua-format.sh + language: script + pass_filenames: false diff --git a/scripts/config/sonar-scanner.properties b/scripts/config/sonar-scanner.properties index 52da5eeb..534653ca 100644 --- a/scripts/config/sonar-scanner.properties +++ b/scripts/config/sonar-scanner.properties @@ -7,3 +7,9 @@ sonar.terraform.provider.aws.version=5.54.1 sonar.cpd.exclusions=**.test.*, src/models/** sonar.coverage.exclusions=.github/**, docs/**, infrastructure/terraform/**, scripts/**, tests/test-support/**, tests/**, lambdas/**/src/__tests__/**, src/**/src/__tests__/**, src/models/**, tools/**/src/__tests__/**, **/jest.config.*, **/knip.ts, eslint.config.mjs sonar.javascript.lcov.reportPaths=lcov.info + +# typescript:S4325 flags non-null/type assertions as "unnecessary" based on a different +# interpretation than our own strict typechecking enforces, causing conflicting lint loops. +sonar.issue.ignore.multicriteria=e1 +sonar.issue.ignore.multicriteria.e1.ruleKey=typescript:S4325 +sonar.issue.ignore.multicriteria.e1.resourceKey=** diff --git a/scripts/config/vale/styles/config/vocabularies/words/accept.txt b/scripts/config/vale/styles/config/vocabularies/words/accept.txt index ee5b597c..8f60e06c 100644 --- a/scripts/config/vale/styles/config/vocabularies/words/accept.txt +++ b/scripts/config/vale/styles/config/vocabularies/words/accept.txt @@ -1,6 +1,8 @@ ajv +APIs asdf auditability +[Bb]ackoff Bitwarden bot [Cc]onfigs? @@ -22,20 +24,27 @@ Grype idempotence Jira namespace +NFRs npm OAuth Octokit onboarding +pnpm Podman Python queryable rawContent read_file -repo +[rR][eE][pP][oO] [Rr]unbook sed +Serverless Syft teardown Terraform toolchain Trufflehog +typecheck +validators +Valkey +[Zz]od diff --git a/scripts/githooks/check-lua-format.sh b/scripts/githooks/check-lua-format.sh new file mode 100755 index 00000000..fe84f9a9 --- /dev/null +++ b/scripts/githooks/check-lua-format.sh @@ -0,0 +1,104 @@ +#!/bin/bash + +set -euo pipefail + +# Pre-commit git hook to lint Lua files using luacheck. Runs luacheck natively +# if installed, otherwise falls back to Docker. +# +# Usage: +# $ [options] ./check-lua-format.sh +# +# Options: +# check={all,staged-changes,working-tree-changes,branch} # Check mode, default is 'working-tree-changes' +# BRANCH_NAME=other-branch-than-main # Branch to compare with, default is `origin/main` +# FORCE_USE_DOCKER=true # If set to true the command is run in a Docker container, default is 'false' +# VERBOSE=true # Show all the executed commands, default is `false` + +# ============================================================================== + +function main() { + + cd "$(git rev-parse --show-toplevel)" || return 1 + + check=${check:-working-tree-changes} + case $check in + "all") + files="$(git ls-files "*.lua")" + ;; + "staged-changes") + files="$(git diff --diff-filter=ACMRT --name-only --cached "*.lua")" + ;; + "working-tree-changes") + files="$(git diff --diff-filter=ACMRT --name-only "*.lua")" + ;; + "branch") + files="$( (git diff --diff-filter=ACMRT --name-only "${BRANCH_NAME:-origin/main}" "*.lua"; git diff --name-only "*.lua") | sort | uniq )" + ;; + *) + echo "Unrecognised check mode: $check" >&2 && exit 1 + ;; + esac + + if [[ -n "$files" ]]; then + # shellcheck disable=SC2155 + local globals=$(jq -r '.diagnostics.globals[]' .luarc.json | tr '\n' ' ') + if command -v luacheck > /dev/null 2>&1 && ! is-arg-true "${FORCE_USE_DOCKER:-false}"; then + files="$files" globals="$globals" run-luacheck-natively + else + files="$files" globals="$globals" run-luacheck-in-docker + fi + fi + return 0 +} + +# Run luacheck natively. +# Arguments (provided as environment variables): +# files=[files to check] +# globals=[space-separated list of global names] +function run-luacheck-natively() { + + # shellcheck disable=SC2086 + luacheck $files --globals $globals + return 0 +} + +# Run luacheck in a Docker container. +# Arguments (provided as environment variables): +# files=[files to check] +# globals=[space-separated list of global names] +function run-luacheck-in-docker() { + + # shellcheck disable=SC1091 + source ./scripts/docker/docker.lib.sh + + # shellcheck disable=SC2155 + local image=$(name=pipelinecomponents/luacheck docker-get-image-version-and-pull) + # shellcheck disable=SC2086 + docker run --rm --platform linux/amd64 \ + --volume "$PWD":/data \ + --workdir /data \ + --entrypoint luacheck \ + "$image" \ + $files --globals $globals + return 0 +} + +# ============================================================================== + +function is-arg-true() { + local value="$1" + + if [[ "$value" =~ ^(true|yes|y|on|1|TRUE|YES|Y|ON)$ ]]; then + return 0 + else + return 1 + fi +} + +# ============================================================================== + +is-arg-true "${VERBOSE:-false}" && set -x + +main "$@" + +exit 0 diff --git a/scripts/tests/integration-debug.sh b/scripts/tests/integration-debug.sh index 08dc0e90..6b34e462 100755 --- a/scripts/tests/integration-debug.sh +++ b/scripts/tests/integration-debug.sh @@ -8,22 +8,29 @@ set -euo pipefail # ENVIRONMENT= AWS_PROFILE= make test-integration-debug ACTION=queue-status # ENVIRONMENT= AWS_PROFILE= make test-integration-debug ACTION=tail-transform # ENVIRONMENT= AWS_PROFILE= make test-integration-debug ACTION=tail-transform LOG_FILTER= +# ENVIRONMENT= AWS_PROFILE= FOLLOW=false make test-integration-debug ACTION=tail-transform > out.log # # Actions: # queue-status Show SQS queue message counts # queue-peek Peek one message from each SQS queue -# tail-transform Tail client-transform-filter lambda logs -# tail-webhook Tail mock-webhook lambda logs -# tail-pipe Tail EventBridge pipe log group -# pipe-state Show EventBridge pipe state and recent metrics +# tail-transform Tail client-transform-filter lambda logs +# tail-https-client Tail https-client lambda logs (requires CLIENT_ID) +# tail-webhook Tail mock-webhook lambda logs +# tail-pipe Tail EventBridge pipe log group +# pipe-state Show EventBridge pipe state and recent metrics # # Required: # ENVIRONMENT # AWS_PROFILE # ACTION # +# Required for queue-status, queue-peek: +# CLIENT_ID Client ID (e.g. mock-client-1) +# # Optional: # LOG_FILTER CloudWatch Logs filter pattern / text +# LOG_SINCE How far back to start tailing logs (default: 30m, e.g. 1h, 2h, 30m) +# FOLLOW Follow logs continuously (default: true, set false to dump and exit) # AWS_REGION (default: eu-west-2) if [[ -z "${ENVIRONMENT:-}" ]]; then @@ -45,7 +52,9 @@ fi REGION="${AWS_REGION:-eu-west-2}" LOG_FILTER="${LOG_FILTER:-}" -SUBSCRIPTION_FIXTURE_PATH="${SUBSCRIPTION_FIXTURE_PATH:-tests/integration/fixtures/subscriptions/mock-client-1.json}" +LOG_SINCE="${LOG_SINCE:-30m}" +CLIENT_ID="${CLIENT_ID:-}" +FOLLOW="${FOLLOW:-true}" if ! aws sts get-caller-identity --profile "$AWS_PROFILE" >/dev/null 2>&1; then echo "No active AWS SSO session for profile '$AWS_PROFILE'. Running aws sso login..." @@ -54,15 +63,13 @@ fi ACCOUNT_ID="$(aws sts get-caller-identity --profile "$AWS_PROFILE" --query Account --output text)" -PREFIX="nhs-${ENVIRONMENT}-callbacks" +PREFIX="nhs-${ENVIRONMENT}-cb" +CLIENT_PREFIX="nhs-${ENVIRONMENT}-cbc" PIPE_NAME="${PREFIX}-main" print_section() { local title="$1" - echo "" - echo "========================================" - echo "$title" - echo "========================================" + printf '\n========================================\n%s\n========================================\n' "$title" return 0 } @@ -72,21 +79,12 @@ queue_url() { return 0 } -target_dlq_queue_name() { - local target_id - - if [[ ! -f "$SUBSCRIPTION_FIXTURE_PATH" ]]; then - echo "Error: subscription fixture not found: $SUBSCRIPTION_FIXTURE_PATH" >&2 - exit 1 - fi - - target_id="$(jq -r '.targets[0].targetId // empty' "$SUBSCRIPTION_FIXTURE_PATH")" - if [[ -z "$target_id" ]]; then - echo "Error: unable to read targets[0].targetId from $SUBSCRIPTION_FIXTURE_PATH" >&2 +require_client_id() { + if [[ -z "$CLIENT_ID" ]]; then + echo "Error: CLIENT_ID must be set for this action." >&2 + echo "Example: CLIENT_ID=mock-client-1 ENVIRONMENT= AWS_PROFILE= make test-integration-debug ACTION=queue-status" >&2 exit 1 fi - - echo "${PREFIX}-${target_id}-dlq-queue" return 0 } @@ -106,9 +104,12 @@ show_queue_counts() { } action_queue_status() { - show_queue_counts "Mock Target DLQ - Queue Message Counts" "$(target_dlq_queue_name)" - show_queue_counts "Inbound Event Queue - Queue Message Counts" "${PREFIX}-inbound-event-queue" - show_queue_counts "Inbound Event DLQ - Queue Message Counts" "${PREFIX}-inbound-event-dlq" + require_client_id + + show_queue_counts "Client Delivery Queue - Message Counts" "${CLIENT_PREFIX}-${CLIENT_ID}-delivery-queue" + show_queue_counts "Client Delivery DLQ - Message Counts" "${CLIENT_PREFIX}-${CLIENT_ID}-delivery-dlq-queue" + show_queue_counts "Inbound Event Queue - Message Counts" "${PREFIX}-inbound-event-queue" + show_queue_counts "Inbound Event DLQ - Message Counts" "${PREFIX}-inbound-event-dlq" return 0 } @@ -135,53 +136,82 @@ peek_queue_message() { } action_queue_peek() { - peek_queue_message "Mock Target DLQ - Message Peek" "$(target_dlq_queue_name)" + require_client_id + peek_queue_message "Client Delivery Queue - Message Peek" "${CLIENT_PREFIX}-${CLIENT_ID}-delivery-queue" + peek_queue_message "Client Delivery DLQ - Message Peek" "${CLIENT_PREFIX}-${CLIENT_ID}-delivery-dlq-queue" peek_queue_message "Inbound Event Queue - Message Peek" "${PREFIX}-inbound-event-queue" peek_queue_message "Inbound Event DLQ - Message Peek" "${PREFIX}-inbound-event-dlq" return 0 } log_filter_args() { - local -a args=() - local escaped_log_filter if [[ -n "$LOG_FILTER" ]]; then - escaped_log_filter="${LOG_FILTER//\"/\\\"}" + local escaped_log_filter="${LOG_FILTER//\"/\\\"}" # CloudWatch filter patterns treat quoted strings as exact phrases. - args+=(--filter-pattern "\"$escaped_log_filter\"") + printf '%s\n' --filter-pattern "\"$escaped_log_filter\"" fi + return 0 +} - printf '%s\n' "${args[@]}" +follow_args() { + if [[ "$FOLLOW" == "true" ]]; then + printf '%s\n' --follow + fi return 0 } action_tail_transform() { local -a filter_args=() + local -a follow_arg=() mapfile -t filter_args < <(log_filter_args) + mapfile -t follow_arg < <(follow_args) print_section "Transform/Filter Lambda Logs" aws logs tail \ "/aws/lambda/${PREFIX}-client-transform-filter" \ --region "$REGION" \ --profile "$AWS_PROFILE" \ - --since 30m \ - --follow \ + --since "$LOG_SINCE" \ + --format short \ + "${follow_arg[@]}" \ + "${filter_args[@]}" + return 0 +} + +action_tail_https_client() { + require_client_id + + local -a filter_args=() + local -a follow_arg=() + mapfile -t filter_args < <(log_filter_args) + mapfile -t follow_arg < <(follow_args) + + print_section "HTTPS Client Lambda Logs" + aws logs tail \ + "/aws/lambda/${CLIENT_PREFIX}-https-client-${CLIENT_ID}" \ + --region "$REGION" \ + --profile "$AWS_PROFILE" \ + --since "$LOG_SINCE" \ --format short \ + "${follow_arg[@]}" \ "${filter_args[@]}" return 0 } action_tail_webhook() { local -a filter_args=() + local -a follow_arg=() mapfile -t filter_args < <(log_filter_args) + mapfile -t follow_arg < <(follow_args) print_section "Mock Webhook Lambda Logs" aws logs tail \ "/aws/lambda/${PREFIX}-mock-webhook" \ --region "$REGION" \ --profile "$AWS_PROFILE" \ - --since 30m \ - --follow \ + --since "$LOG_SINCE" \ --format short \ + "${follow_arg[@]}" \ "${filter_args[@]}" return 0 } @@ -208,14 +238,17 @@ action_tail_pipe() { pipe_log_group_name="${pipe_log_group_arn#*:log-group:}" + local -a follow_arg=() + mapfile -t follow_arg < <(follow_args) + print_section "EventBridge Pipe Logs" aws logs tail \ "$pipe_log_group_name" \ --region "$REGION" \ --profile "$AWS_PROFILE" \ - --since 30m \ - --follow \ + --since "$LOG_SINCE" \ --format short \ + "${follow_arg[@]}" \ "${filter_args[@]}" return 0 } @@ -280,6 +313,9 @@ case "$ACTION" in tail-transform) action_tail_transform ;; + tail-https-client) + action_tail_https_client + ;; tail-webhook) action_tail_webhook ;; @@ -291,7 +327,7 @@ case "$ACTION" in ;; *) echo "Unknown action: $ACTION" >&2 - echo "Actions: queue-status, queue-peek, tail-transform, tail-webhook, tail-pipe, pipe-state" >&2 + echo "Actions: queue-status, queue-peek, tail-transform, tail-https-client, tail-webhook, tail-pipe, pipe-state" >&2 exit 1 ;; esac diff --git a/scripts/tests/integration-env.sh b/scripts/tests/integration-env.sh index cd5ff1a8..9a889902 100644 --- a/scripts/tests/integration-env.sh +++ b/scripts/tests/integration-env.sh @@ -7,8 +7,12 @@ set -euo pipefail # Add new clients here: "fixture-filename.json:ENV_VAR_PREFIX" CLIENTS=( - "mock-client-1.json:MOCK_CLIENT" - "mock-client-2.json:MOCK_CLIENT_2" + "mock-client-single-target.json:MOCK_CLIENT" + "mock-client-fan-out.json:MOCK_CLIENT_FAN_OUT" + "mock-client-mtls.json:MOCK_CLIENT_MTLS" + "mock-client-rate-limit.json:MOCK_CLIENT_RATE_LIMIT" + "mock-client-circuit-breaker.json:MOCK_CLIENT_CIRCUIT_BREAKER" + "mock-client-short-retry.json:MOCK_CLIENT_SHORT_RETRY" ) for CLIENT_ENTRY in "${CLIENTS[@]}"; do diff --git a/scripts/tests/integration-local.sh b/scripts/tests/integration-local.sh index 55605b2d..95280dcf 100755 --- a/scripts/tests/integration-local.sh +++ b/scripts/tests/integration-local.sh @@ -30,7 +30,7 @@ fi AWS_REGION="${AWS_REGION:-eu-west-2}" LOG_LEVEL="${LOG_LEVEL:-debug}" NODE_OPTIONS="${NODE_OPTIONS:---experimental-vm-modules}" -COMPONENT="callbacks" +COMPONENT="cb" PROJECT="nhs" if ! aws sts get-caller-identity --profile "$AWS_PROFILE" >/dev/null 2>&1; then diff --git a/scripts/tests/lua-lint.sh b/scripts/tests/lua-lint.sh new file mode 100755 index 00000000..ae271da4 --- /dev/null +++ b/scripts/tests/lua-lint.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +set -euo pipefail + +cd "$(git rev-parse --show-toplevel)" + +check=all ./scripts/githooks/check-lua-format.sh diff --git a/scripts/tests/test.mk b/scripts/tests/test.mk index d9303d92..2bb70740 100644 --- a/scripts/tests/test.mk +++ b/scripts/tests/test.mk @@ -14,6 +14,9 @@ test-unit: # Run your unit tests from scripts/test/unit @Testing test-lint: # Lint your code from scripts/test/lint @Testing make _test name="lint" +test-lua-lint: # Lint Lua scripts @Testing + make _test name="lua-lint" + test-typecheck: # Typecheck your code from scripts/test/typecheck @Testing make _test name="typecheck" @@ -35,7 +38,7 @@ test-integration-local: # Run integration tests locally against a remoptely depl test-integration-debug: # Debug a live environment - inspect queues, tail logs, check pipe state (requires ENVIRONMENT, AWS_PROFILE, ACTION) @Testing make _test name="integration-debug" ACTION="$(or $(ACTION),$(word 2,$(MAKECMDGOALS)))" -queue-status queue-peek tail-transform tail-webhook tail-pipe pipe-state: +queue-status queue-peek tail-transform tail-https-client tail-webhook tail-pipe pipe-state: @: test-load: # Run all your load tests @Testing diff --git a/src/config-subscription-cache/jest.config.ts b/src/config-subscription-cache/jest.config.ts new file mode 100644 index 00000000..6ecf333b --- /dev/null +++ b/src/config-subscription-cache/jest.config.ts @@ -0,0 +1,14 @@ +import { nodeJestConfig } from "../../jest.config.base.ts"; + +export default { + ...nodeJestConfig, + coverageThreshold: { + global: { + ...nodeJestConfig.coverageThreshold?.global, + branches: 100, + functions: 100, + lines: 100, + statements: 100, + }, + }, +}; diff --git a/src/config-subscription-cache/package.json b/src/config-subscription-cache/package.json new file mode 100644 index 00000000..c7bd0be5 --- /dev/null +++ b/src/config-subscription-cache/package.json @@ -0,0 +1,34 @@ +{ + "exports": { + ".": { + "types": "./src/index.ts", + "default": "./src/index.ts" + } + }, + "dependencies": { + "@aws-sdk/client-s3": "catalog:aws", + "@nhs-notify-client-callbacks/logger": "workspace:*", + "@nhs-notify-client-callbacks/models": "workspace:*" + }, + "devDependencies": { + "@tsconfig/node22": "catalog:tools", + "@types/jest": "catalog:test", + "@types/node": "catalog:tools", + "eslint": "catalog:lint", + "jest": "catalog:test", + "ts-jest": "catalog:test", + "typescript": "catalog:tools" + }, + "engines": { + "node": ">=24.14.1" + }, + "name": "@nhs-notify-client-callbacks/config-subscription-cache", + "private": true, + "scripts": { + "lint": "eslint .", + "lint:fix": "eslint . --fix", + "test:unit": "jest", + "typecheck": "tsc --noEmit" + }, + "version": "0.0.1" +} diff --git a/src/config-subscription-cache/src/__tests__/config-subscription-cache.test.ts b/src/config-subscription-cache/src/__tests__/config-subscription-cache.test.ts new file mode 100644 index 00000000..053b2398 --- /dev/null +++ b/src/config-subscription-cache/src/__tests__/config-subscription-cache.test.ts @@ -0,0 +1,157 @@ +import { GetObjectCommand, NoSuchKey } from "@aws-sdk/client-s3"; +import { ConfigSubscriptionCache } from "config-subscription-cache"; + +const mockS3Send = jest.fn(); +jest.mock("@aws-sdk/client-s3", () => { + const actual = jest.requireActual("@aws-sdk/client-s3"); + return { + ...actual, + S3Client: jest.fn().mockImplementation(() => ({ + send: (...args: unknown[]) => mockS3Send(...args), + })), + }; +}); + +jest.mock("@nhs-notify-client-callbacks/logger", () => ({ + logger: { + info: jest.fn(), + warn: jest.fn(), + error: jest.fn(), + debug: jest.fn(), + }, +})); + +const VALID_CONFIG = { + clientId: "client-1", + subscriptions: [], + targets: [ + { + targetId: "target-1", + type: "API", + invocationEndpoint: "https://webhook.example.invalid", + invocationMethod: "POST", + invocationRateLimit: 10, + apiKey: { headerName: "x-api-key", headerValue: "secret" }, + }, + ], +}; + +const makeS3Response = (body: unknown) => ({ + Body: { + transformToString: jest.fn().mockResolvedValue(JSON.stringify(body)), + }, +}); + +const createCache = (ttlMs = 1000) => { + const { S3Client } = jest.requireMock("@aws-sdk/client-s3"); + return new ConfigSubscriptionCache({ + s3Client: new S3Client(), + bucketName: "test-bucket", + keyPrefix: "client_subscriptions/", + ttlMs, + }); +}; + +describe("ConfigSubscriptionCache", () => { + beforeEach(() => { + mockS3Send.mockReset(); + }); + + it("loads and parses valid config from S3", async () => { + mockS3Send.mockResolvedValue(makeS3Response(VALID_CONFIG)); + const cache = createCache(); + + const result = await cache.loadClientConfig("client-1"); + + expect(result).toEqual(VALID_CONFIG); + expect(mockS3Send).toHaveBeenCalledTimes(1); + expect(mockS3Send.mock.calls[0][0]).toBeInstanceOf(GetObjectCommand); + }); + + it("uses the configured key prefix for S3 requests", async () => { + mockS3Send.mockResolvedValue(makeS3Response(VALID_CONFIG)); + const cache = createCache(); + + await cache.loadClientConfig("client-1"); + + const command: GetObjectCommand = mockS3Send.mock.calls[0][0]; + expect(command.input.Key).toBe("client_subscriptions/client-1.json"); + expect(command.input.Bucket).toBe("test-bucket"); + }); + + it("returns cached config on subsequent calls", async () => { + mockS3Send.mockResolvedValue(makeS3Response(VALID_CONFIG)); + const cache = createCache(); + + await cache.loadClientConfig("client-1"); + await cache.loadClientConfig("client-1"); + + expect(mockS3Send).toHaveBeenCalledTimes(1); + }); + + it("re-fetches from S3 after TTL expiry", async () => { + jest.useFakeTimers(); + jest.setSystemTime(new Date("2026-01-01T10:00:00Z")); + + mockS3Send.mockResolvedValue(makeS3Response(VALID_CONFIG)); + const cache = createCache(1000); + + await cache.loadClientConfig("client-1"); + + jest.advanceTimersByTime(1001); + + await cache.loadClientConfig("client-1"); + + expect(mockS3Send).toHaveBeenCalledTimes(2); + + jest.useRealTimers(); + }); + + it("returns undefined when S3 key does not exist", async () => { + mockS3Send.mockRejectedValue(new NoSuchKey({ $metadata: {}, message: "" })); + const cache = createCache(); + + const result = await cache.loadClientConfig("missing-client"); + + expect(result).toBeUndefined(); + }); + + it("throws when config fails validation", async () => { + const invalidConfig = { ...VALID_CONFIG, targets: [{ invalid: true }] }; + mockS3Send.mockResolvedValue(makeS3Response(invalidConfig)); + const cache = createCache(); + + await expect(cache.loadClientConfig("client-1")).rejects.toThrow( + "Invalid client config for 'client-1'", + ); + }); + + it("throws when S3 body is empty", async () => { + mockS3Send.mockResolvedValue({ Body: undefined }); + const cache = createCache(); + + await expect(cache.loadClientConfig("client-1")).rejects.toThrow( + "S3 response body was empty for client 'client-1'", + ); + }); + + it("propagates non-NoSuchKey S3 errors", async () => { + mockS3Send.mockRejectedValue(new Error("S3 access denied")); + const cache = createCache(); + + await expect(cache.loadClientConfig("client-1")).rejects.toThrow( + "S3 access denied", + ); + }); + + it("clears cache on reset", async () => { + mockS3Send.mockResolvedValue(makeS3Response(VALID_CONFIG)); + const cache = createCache(); + + await cache.loadClientConfig("client-1"); + cache.reset(); + await cache.loadClientConfig("client-1"); + + expect(mockS3Send).toHaveBeenCalledTimes(2); + }); +}); diff --git a/src/config-subscription-cache/src/config-subscription-cache.ts b/src/config-subscription-cache/src/config-subscription-cache.ts new file mode 100644 index 00000000..0ce3547c --- /dev/null +++ b/src/config-subscription-cache/src/config-subscription-cache.ts @@ -0,0 +1,110 @@ +import { GetObjectCommand, NoSuchKey, S3Client } from "@aws-sdk/client-s3"; +import type { ClientSubscriptionConfiguration } from "@nhs-notify-client-callbacks/models"; +import { parseClientSubscriptionConfiguration } from "@nhs-notify-client-callbacks/models"; +import { logger } from "@nhs-notify-client-callbacks/logger"; + +type CacheEntry = { + value: ClientSubscriptionConfiguration; + expiresAt: number; +}; + +export type ConfigSubscriptionCacheOptions = { + s3Client: S3Client; + bucketName: string; + keyPrefix: string; + ttlMs: number; +}; + +export class ConfigSubscriptionCache { + private readonly cache = new Map(); + + private readonly s3Client: S3Client; + + private readonly bucketName: string; + + private readonly keyPrefix: string; + + private readonly ttlMs: number; + + constructor(options: ConfigSubscriptionCacheOptions) { + this.s3Client = options.s3Client; + this.bucketName = options.bucketName; + this.keyPrefix = options.keyPrefix; + this.ttlMs = options.ttlMs; + } + + async loadClientConfig( + clientId: string, + ): Promise { + const cached = this.getCached(clientId); + if (cached) { + return cached; + } + + const raw = await this.fetchFromS3(clientId); + if (raw === undefined) { + return undefined; + } + + const parsed = JSON.parse(raw) as unknown; + const result = parseClientSubscriptionConfiguration(parsed); + + if (!result.success) { + throw new Error( + `Invalid client config for '${clientId}': ${result.error.message}`, + ); + } + + this.cache.set(clientId, { + value: result.data, + expiresAt: Date.now() + this.ttlMs, + }); + + logger.info("Client config loaded from S3", { clientId }); + return result.data; + } + + reset(): void { + this.cache.clear(); + } + + // eslint-disable-next-line sonarjs/function-return-type -- cache lookup returns T | undefined + private getCached( + clientId: string, + ): ClientSubscriptionConfiguration | undefined { + const entry = this.cache.get(clientId); + + if (entry && entry.expiresAt <= Date.now()) { + this.cache.delete(clientId); + return undefined; + } + + return entry?.value; + } + + private async fetchFromS3(clientId: string): Promise { + try { + const response = await this.s3Client.send( + new GetObjectCommand({ + Bucket: this.bucketName, + Key: `${this.keyPrefix}${clientId}.json`, + }), + ); + + if (!response.Body) { + throw new Error(`S3 response body was empty for client '${clientId}'`); + } + + return await response.Body.transformToString(); + } catch (error) { + if (error instanceof NoSuchKey) { + logger.info( + "No config found in S3 for client — events will be filtered out", + { clientId }, + ); + return undefined; + } + throw error; + } + } +} diff --git a/src/config-subscription-cache/src/index.ts b/src/config-subscription-cache/src/index.ts new file mode 100644 index 00000000..39a4501b --- /dev/null +++ b/src/config-subscription-cache/src/index.ts @@ -0,0 +1,2 @@ +export { ConfigSubscriptionCache } from "./config-subscription-cache"; +export type { ConfigSubscriptionCacheOptions } from "./config-subscription-cache"; diff --git a/src/config-subscription-cache/tsconfig.json b/src/config-subscription-cache/tsconfig.json new file mode 100644 index 00000000..a50e6fc0 --- /dev/null +++ b/src/config-subscription-cache/tsconfig.json @@ -0,0 +1,14 @@ +{ + "compilerOptions": { + "isolatedModules": true, + "paths": { + "*": [ + "./src/*" + ] + } + }, + "extends": "../../tsconfig.base.json", + "include": [ + "src/**/*" + ] +} diff --git a/src/models/src/__tests__/client-config-schema.test.ts b/src/models/src/__tests__/client-config-schema.test.ts index da1e5429..f1ff0702 100644 --- a/src/models/src/__tests__/client-config-schema.test.ts +++ b/src/models/src/__tests__/client-config-schema.test.ts @@ -19,6 +19,8 @@ const expectFailedParse = ( return result; }; +const VALID_SPKI_HASH = "KL/yFsVH+gnkkzdQ+DSlV8xMQOMehksgT6aOqQviOu8="; + const createValidConfig = (): ClientSubscriptionConfiguration => ({ clientId: "client-1", subscriptions: [ @@ -45,6 +47,12 @@ const createValidConfig = (): ClientSubscriptionConfiguration => ({ invocationMethod: "POST", invocationRateLimit: 10, apiKey: { headerName: "x-api-key", headerValue: "secret" }, + delivery: { + mtls: { + enabled: true, + certPinning: { enabled: true, spkiHash: VALID_SPKI_HASH }, + }, + }, }, ], }); @@ -147,4 +155,121 @@ describe("parseClientSubscriptionConfiguration", () => { }), ]); }); + + it("parses a valid config with mtls, certPinning, and delivery fields", () => { + const config = createValidConfig(); + config.targets[0].delivery = { + ...config.targets[0].delivery, + maxRetryDurationSeconds: 7200, + circuitBreaker: { enabled: true }, + }; + + expect(parseClientSubscriptionConfiguration(config)).toEqual({ + success: true, + data: config, + }); + }); + + it("returns a failed parse result when delivery.mtls has invalid shape", () => { + const config = createValidConfig(); + (config.targets[0] as Record).delivery = { + mtls: { enabled: "not-a-boolean" }, + }; + + const result = expectFailedParse( + parseClientSubscriptionConfiguration(config), + ); + + expect(result.error.issues).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + path: expect.arrayContaining(["targets", 0, "delivery"]), + }), + ]), + ); + }); + + it("returns a failed parse result when spkiHash has an invalid pattern", () => { + const config = createValidConfig(); + config.targets[0].delivery!.mtls!.certPinning = { + enabled: true, + spkiHash: "not-a-valid-hash", + }; + + const result = expectFailedParse( + parseClientSubscriptionConfiguration(config), + ); + + expect(result.error.issues).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + message: "Invalid SPKI hash", + }), + ]), + ); + }); + + it("returns a failed parse result when certPinning.enabled is true without spkiHash", () => { + const config = createValidConfig(); + config.targets[0].delivery!.mtls!.certPinning = { enabled: true }; + + const result = expectFailedParse( + parseClientSubscriptionConfiguration(config), + ); + + expect(result.error.issues).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + message: "spkiHash is required when certPinning is enabled", + }), + ]), + ); + }); + + it("returns a failed parse result when maxRetryDurationSeconds is zero", () => { + const config = createValidConfig(); + config.targets[0].delivery = { maxRetryDurationSeconds: 0 }; + + const result = expectFailedParse( + parseClientSubscriptionConfiguration(config), + ); + + expect(result.success).toBe(false); + }); + + it("returns a failed parse result when maxRetryDurationSeconds is negative", () => { + const config = createValidConfig(); + config.targets[0].delivery = { maxRetryDurationSeconds: -1 }; + + const result = expectFailedParse( + parseClientSubscriptionConfiguration(config), + ); + + expect(result.success).toBe(false); + }); + + it("returns a failed parse result when maxRetryDurationSeconds is above 43200", () => { + const config = createValidConfig(); + config.targets[0].delivery = { maxRetryDurationSeconds: 43_201 }; + + const result = expectFailedParse( + parseClientSubscriptionConfiguration(config), + ); + + expect(result.success).toBe(false); + }); + + it("accepts maxRetryDurationSeconds at boundary value 1", () => { + const config = createValidConfig(); + config.targets[0].delivery = { maxRetryDurationSeconds: 1 }; + + expect(parseClientSubscriptionConfiguration(config).success).toBe(true); + }); + + it("accepts maxRetryDurationSeconds at boundary value 43200", () => { + const config = createValidConfig(); + config.targets[0].delivery = { maxRetryDurationSeconds: 43_200 }; + + expect(parseClientSubscriptionConfiguration(config).success).toBe(true); + }); }); diff --git a/src/models/src/client-config-schema.ts b/src/models/src/client-config-schema.ts index b56a9439..cae4587a 100644 --- a/src/models/src/client-config-schema.ts +++ b/src/models/src/client-config-schema.ts @@ -22,6 +22,20 @@ const httpsUrlSchema = z.string().refine( }, ); +const SPKI_HASH_PATTERN = /^[A-Za-z0-9+/]{43}=$/; + +const certPinningSchema = z + .object({ + enabled: z.boolean(), + spkiHash: z + .string() + .regex(SPKI_HASH_PATTERN, "Invalid SPKI hash") + .optional(), + }) + .refine((val) => !val.enabled || val.spkiHash !== undefined, { + message: "spkiHash is required when certPinning is enabled", + }); + const targetSchema = z.object({ targetId: z.string(), type: z.literal("API"), @@ -32,6 +46,22 @@ const targetSchema = z.object({ headerName: z.string(), headerValue: z.string(), }), + delivery: z + .object({ + maxRetryDurationSeconds: z.number().positive().max(43_200).optional(), + circuitBreaker: z + .object({ + enabled: z.boolean(), + }) + .optional(), + mtls: z + .object({ + enabled: z.boolean(), + certPinning: certPinningSchema.optional(), + }) + .optional(), + }) + .optional(), }); const baseSubscriptionSchema = z.object({ diff --git a/src/models/src/client-config.ts b/src/models/src/client-config.ts index 84116353..4d1796d1 100644 --- a/src/models/src/client-config.ts +++ b/src/models/src/client-config.ts @@ -15,6 +15,19 @@ export type CallbackTarget = { headerName: string; headerValue: string; }; + delivery?: { + maxRetryDurationSeconds?: number; + circuitBreaker?: { + enabled: boolean; + }; + mtls?: { + enabled: boolean; + certPinning?: { + enabled: boolean; + spkiHash?: string; + }; + }; + }; }; type SubscriptionConfigurationBase = { diff --git a/tests/integration/README.md b/tests/integration/README.md index a58531b8..0a76bf74 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -50,30 +50,33 @@ All are run via `make test-integration-debug ACTION=`. - [`queue-status`](#queue-status) – SQS queue message counts - [`queue-peek`](#queue-peek) – Peek at one message from each SQS queue - [`tail-transform`](#tail-transform) – Tail the transform/filter Lambda logs +- [`tail-https-client`](#tail-https-client) – Tail the https-client Lambda logs - [`tail-webhook`](#tail-webhook) – Tail the mock-webhook Lambda logs - [`tail-pipe`](#tail-pipe) – Tail the EventBridge pipe logs - [`pipe-state`](#pipe-state) – Show EventBridge pipe state and recent metrics -All log-tailing actions (`tail-transform`, `tail-webhook`, `tail-pipe`) accept an optional `LOG_FILTER` to narrow output to a specific message ID or pattern. +Some actions require `CLIENT_ID` (e.g. `mock-client-single-target`) — see individual actions below. + +All log-tailing actions (`tail-transform`, `tail-https-client`, `tail-webhook`, `tail-pipe`) accept an optional `LOG_FILTER` to narrow output to a specific message ID or pattern. --- ### `queue-status` -Shows approximate message counts for the inbound event queue, inbound event DLQ, and mock target DLQ. +Shows approximate message counts for the inbound event queue, inbound event DLQ, client delivery queue, and client delivery DLQ. Requires `CLIENT_ID`. ```sh -ENVIRONMENT= AWS_PROFILE= make test-integration-debug ACTION=queue-status +CLIENT_ID= ENVIRONMENT= AWS_PROFILE= make test-integration-debug ACTION=queue-status ``` --- ### `queue-peek` -Reads one message (without deleting it) from each of the same three queues, printing body, attributes, and message attributes. +Reads one message (without deleting it) from each of the same four queues, printing body, attributes, and message attributes. Requires `CLIENT_ID`. ```sh -ENVIRONMENT= AWS_PROFILE= make test-integration-debug ACTION=queue-peek +CLIENT_ID= ENVIRONMENT= AWS_PROFILE= make test-integration-debug ACTION=queue-peek ``` --- @@ -94,6 +97,22 @@ ENVIRONMENT= AWS_PROFILE= LOG_FILTER=SOME-MESSAGE-ID make test-int --- +### `tail-https-client` + +Tails CloudWatch logs for the `https-client` Lambda for the given client, following from the last 30 minutes. Requires `CLIENT_ID`. + +```sh +CLIENT_ID= ENVIRONMENT= AWS_PROFILE= make test-integration-debug ACTION=tail-https-client +``` + +Filter to a specific message ID: + +```sh +CLIENT_ID= ENVIRONMENT= AWS_PROFILE= LOG_FILTER=SOME-MESSAGE-ID make test-integration-debug ACTION=tail-https-client +``` + +--- + ### `tail-webhook` Tails CloudWatch logs for the `mock-webhook` Lambda, following from the last 30 minutes. diff --git a/tests/integration/delivery-resilience.test.ts b/tests/integration/delivery-resilience.test.ts new file mode 100644 index 00000000..6420d0b8 --- /dev/null +++ b/tests/integration/delivery-resilience.test.ts @@ -0,0 +1,273 @@ +import type { + MessageStatusData, + StatusPublishEvent, +} from "@nhs-notify-client-callbacks/models"; +import { + awaitCallback, + awaitCallbacks, + countLogEntries, +} from "./helpers/cloudwatch"; +import { createMessageStatusPublishEvent } from "./helpers/event-factories"; +import { + buildMockWebhookTargetPath, + getClientConfig, +} from "./helpers/mock-client-config"; +import { assertCallbackHeaders } from "./helpers/signature"; +import { + awaitQueueMessage, + deleteMessage, + getQueueDepth, + purgeQueues, + sendSqsEvent, +} from "./helpers/sqs"; +import { + type TestContext, + createTestContext, + destroyTestContext, +} from "./helpers/test-context"; + +describe("Delivery Resilience", () => { + let ctx: TestContext; + + beforeAll(() => { + ctx = createTestContext(); + }); + + afterAll(() => { + destroyTestContext(ctx); + }); + + describe("Retry & Window Exhaustion", () => { + let dlqUrl: string; + let deliveryUrl: string; + + beforeAll(async () => { + const { clientId } = getClientConfig("clientShortRetry"); + dlqUrl = ctx.clientDlqUrl(clientId); + deliveryUrl = ctx.clientDeliveryUrl(clientId); + await purgeQueues(ctx.sqs, [dlqUrl, deliveryUrl]); + }); + + afterAll(async () => { + await purgeQueues(ctx.sqs, [dlqUrl, deliveryUrl]); + }); + + it("should exhaust the retry window on persistent 5xx and route to DLQ", async () => { + const { clientId } = getClientConfig("clientShortRetry"); + const messageId = `force-500-${crypto.randomUUID()}`; + + const event: StatusPublishEvent = + createMessageStatusPublishEvent({ data: { clientId, messageId } }); + + await sendSqsEvent(ctx.sqs, ctx.inboundQueueUrl, event); + + const dlqMessage = await awaitQueueMessage(ctx.sqs, dlqUrl, 90_000); + + expect(dlqMessage.Body).toBeDefined(); + const dlqPayload = JSON.parse(dlqMessage.Body as string); + expect(dlqPayload.payload.data[0].attributes.messageId).toBe(messageId); + + const attemptCount = await countLogEntries( + ctx.cwLogs, + ctx.webhookLogGroup, + `{ $.msg = "Forced status code response" && $.messageId = "${messageId}" }`, + ctx.startTime, + 2, + ); + expect(attemptCount).toBeGreaterThan(1); + + await deleteMessage(ctx.sqs, dlqUrl, dlqMessage); + }, 180_000); + + it("should exhaust the retry window on persistent 429 and route to DLQ", async () => { + const { clientId } = getClientConfig("clientShortRetry"); + const messageId = `force-429-${crypto.randomUUID()}`; + + const event: StatusPublishEvent = + createMessageStatusPublishEvent({ data: { clientId, messageId } }); + + await sendSqsEvent(ctx.sqs, ctx.inboundQueueUrl, event); + + const dlqMessage = await awaitQueueMessage(ctx.sqs, dlqUrl, 90_000); + + expect(dlqMessage.Body).toBeDefined(); + const dlqPayload = JSON.parse(dlqMessage.Body as string); + expect(dlqPayload.payload.data[0].attributes.messageId).toBe(messageId); + + const attemptCount = await countLogEntries( + ctx.cwLogs, + ctx.webhookLogGroup, + `{ $.msg = "Forced status code response" && $.messageId = "${messageId}" }`, + ctx.startTime, + 2, + ); + expect(attemptCount).toBeGreaterThan(1); + + await deleteMessage(ctx.sqs, dlqUrl, dlqMessage); + }, 180_000); + }); + + describe("Rate Limiting", () => { + const BURST_SIZE = 15; + let dlqUrl: string; + let deliveryUrl: string; + let httpsClientLogGroup: string; + + beforeAll(async () => { + const { clientId } = getClientConfig("clientRateLimit"); + dlqUrl = ctx.clientDlqUrl(clientId); + deliveryUrl = ctx.clientDeliveryUrl(clientId); + httpsClientLogGroup = ctx.clientLogGroup(`https-client-${clientId}`); + await purgeQueues(ctx.sqs, [dlqUrl, deliveryUrl]); + }); + + afterAll(async () => { + await purgeQueues(ctx.sqs, [dlqUrl, deliveryUrl]); + }); + + it("should eventually deliver all events in a burst without dropping any to the DLQ", async () => { + const rateLimitConfig = getClientConfig("clientRateLimit"); + const targetPath = buildMockWebhookTargetPath("clientRateLimit"); + + const events = Array.from({ length: BURST_SIZE }, () => + createMessageStatusPublishEvent({ + data: { + clientId: rateLimitConfig.clientId, + messageId: `rate-limit-burst-${crypto.randomUUID()}`, + }, + }), + ); + + await Promise.all( + events.map((event) => + sendSqsEvent(ctx.sqs, ctx.inboundQueueUrl, event), + ), + ); + + const callbackMap = await awaitCallbacks( + ctx.cwLogs, + ctx.webhookLogGroup, + events.map((e) => e.data.messageId), + "MessageStatus", + 1, + ctx.startTime, + ); + + const deliveredIds = [...callbackMap.keys()]; + const expectedIds = events.map((e) => e.data.messageId); + expect(deliveredIds).toHaveLength(expectedIds.length); + expect(deliveredIds).toEqual(expect.arrayContaining(expectedIds)); + + for (const [, [callback]] of callbackMap) { + expect(callback.path).toBe(targetPath); + assertCallbackHeaders( + callback, + rateLimitConfig.apiKeyVar, + rateLimitConfig.applicationIdVar, + ); + } + + expect(await getQueueDepth(ctx.sqs, dlqUrl)).toBe(0); + + const rateLimitedCount = await countLogEntries( + ctx.cwLogs, + httpsClientLogGroup, + `{ $.msg = "Client rate limited" }`, + ctx.startTime, + 1, + ); + expect(rateLimitedCount).toBeGreaterThanOrEqual(1); + }, 180_000); + }); + + describe("Circuit Breaker", () => { + const CB_BURST_SIZE = 15; + let dlqUrl: string; + let deliveryUrl: string; + let httpsClientLogGroup: string; + + beforeAll(async () => { + const { clientId } = getClientConfig("clientCircuitBreaker"); + dlqUrl = ctx.clientDlqUrl(clientId); + deliveryUrl = ctx.clientDeliveryUrl(clientId); + httpsClientLogGroup = ctx.clientLogGroup(`https-client-${clientId}`); + await purgeQueues(ctx.sqs, [dlqUrl, deliveryUrl]); + }); + + afterAll(async () => { + await purgeQueues(ctx.sqs, [dlqUrl, deliveryUrl]); + }); + + it("should open the circuit breaker after repeated failures and not affect other clients", async () => { + const cbConfig = getClientConfig("clientCircuitBreaker"); + const cbTargetPath = buildMockWebhookTargetPath("clientCircuitBreaker"); + const singleTargetConfig = getClientConfig("clientSingleTarget"); + const singleTargetPath = buildMockWebhookTargetPath("clientSingleTarget"); + + // Send a successful message first so the circuit is confirmed closed (it starts half-open) + const warmupEvent = createMessageStatusPublishEvent({ + data: { + clientId: cbConfig.clientId, + messageId: `cb-warmup-${crypto.randomUUID()}`, + }, + }); + await sendSqsEvent(ctx.sqs, ctx.inboundQueueUrl, warmupEvent); + const warmupCallback = await awaitCallback( + ctx.cwLogs, + ctx.webhookLogGroup, + warmupEvent.data.messageId, + "MessageStatus", + ctx.startTime, + ); + expect(warmupCallback.path).toBe(cbTargetPath); + + const forceUntil = Date.now() + 60_000; + const cbEvents = Array.from({ length: CB_BURST_SIZE }, () => + createMessageStatusPublishEvent({ + data: { + clientId: cbConfig.clientId, + messageId: `force-500-until-${forceUntil}-cb-${crypto.randomUUID()}`, + }, + }), + ); + + const normalEvent = createMessageStatusPublishEvent({ + data: { + clientId: singleTargetConfig.clientId, + messageId: `cb-isolation-${crypto.randomUUID()}`, + }, + }); + + await Promise.all([ + ...cbEvents.map((event) => + sendSqsEvent(ctx.sqs, ctx.inboundQueueUrl, event), + ), + sendSqsEvent(ctx.sqs, ctx.inboundQueueUrl, normalEvent), + ]); + + const normalCallback = await awaitCallback( + ctx.cwLogs, + ctx.webhookLogGroup, + normalEvent.data.messageId, + "MessageStatus", + ctx.startTime, + ); + + expect(normalCallback.path).toBe(singleTargetPath); + assertCallbackHeaders( + normalCallback, + singleTargetConfig.apiKeyVar, + singleTargetConfig.applicationIdVar, + ); + + const circuitOpenCount = await countLogEntries( + ctx.cwLogs, + httpsClientLogGroup, + `{ $.msg = "Circuit blocked" }`, + ctx.startTime, + 1, + ); + expect(circuitOpenCount).toBeGreaterThanOrEqual(1); + }, 180_000); + }); +}); diff --git a/tests/integration/dlq-alarms.test.ts b/tests/integration/dlq-alarms.test.ts index 1cf3a578..ae1e1bff 100644 --- a/tests/integration/dlq-alarms.test.ts +++ b/tests/integration/dlq-alarms.test.ts @@ -5,14 +5,18 @@ import { } from "@aws-sdk/client-cloudwatch"; import type { DeploymentDetails } from "@nhs-notify-client-callbacks/test-support/helpers"; import { getDeploymentDetails } from "@nhs-notify-client-callbacks/test-support/helpers"; -import { getAllSubscriptionTargetIds } from "./helpers/mock-client-config"; +import { + CLIENT_FIXTURES, + type ClientFixtureKey, + getClientConfig, +} from "./helpers/mock-client-config"; import { buildMockClientDlqQueueUrl } from "./helpers/sqs"; function buildDlqDepthAlarmName( - { component, environment, project }: DeploymentDetails, - targetId: string, + { clientComponent, environment, project }: DeploymentDetails, + clientId: string, ): string { - return `${project}-${environment}-${component}-${targetId}-dlq-depth`; + return `${project}-${environment}-${clientComponent}-${clientId}-dlq-depth`; } function getQueueNameFromUrl(queueUrl: string): string { @@ -27,7 +31,7 @@ function getQueueNameFromUrl(queueUrl: string): string { describe("DLQ alarms", () => { let cloudWatchClient: CloudWatchClient; let deploymentDetails: DeploymentDetails; - let targetIds: string[]; + let clientIds: string[]; beforeAll(() => { deploymentDetails = getDeploymentDetails(); @@ -35,22 +39,25 @@ describe("DLQ alarms", () => { region: deploymentDetails.region, }); - targetIds = getAllSubscriptionTargetIds(); + clientIds = (Object.keys(CLIENT_FIXTURES) as ClientFixtureKey[]).map( + (key) => getClientConfig(key).clientId, + ); }); afterAll(() => { cloudWatchClient.destroy(); }); - it("should create a DLQ depth alarm for every target DLQ", async () => { - expect(targetIds.length).toBeGreaterThan(0); + it("should create a DLQ depth alarm for every client DLQ", async () => { + expect(clientIds.length).toBeGreaterThan(0); - for (const targetId of targetIds) { - const alarmName = buildDlqDepthAlarmName(deploymentDetails, targetId); - const targetDlqQueueUrl = buildMockClientDlqQueueUrl(deploymentDetails, [ - { targetId }, - ]); - const targetDlqQueueName = getQueueNameFromUrl(targetDlqQueueUrl); + for (const clientId of clientIds) { + const alarmName = buildDlqDepthAlarmName(deploymentDetails, clientId); + const clientDlqQueueUrl = buildMockClientDlqQueueUrl( + deploymentDetails, + clientId, + ); + const clientDlqQueueName = getQueueNameFromUrl(clientDlqQueueUrl); const response = await cloudWatchClient.send( new DescribeAlarmsCommand({ AlarmNames: [alarmName], @@ -67,7 +74,7 @@ describe("DLQ alarms", () => { expect.arrayContaining([ expect.objectContaining({ Name: "QueueName", - Value: targetDlqQueueName, + Value: clientDlqQueueName, }), ]), ); diff --git a/tests/integration/dlq-redrive.test.ts b/tests/integration/dlq-redrive.test.ts index e88e4920..325c82ed 100644 --- a/tests/integration/dlq-redrive.test.ts +++ b/tests/integration/dlq-redrive.test.ts @@ -1,98 +1,76 @@ -import { GetQueueAttributesCommand, SQSClient } from "@aws-sdk/client-sqs"; -import { CloudWatchLogsClient } from "@aws-sdk/client-cloudwatch-logs"; import type { MessageStatusData, StatusPublishEvent, } from "@nhs-notify-client-callbacks/models"; +import { awaitCallback, awaitCallbacks } from "./helpers/cloudwatch"; +import { createMessageStatusPublishEvent } from "./helpers/event-factories"; import { - buildInboundEventQueueUrl, - buildLambdaLogGroupName, - createCloudWatchLogsClient, - createSqsClient, - getDeploymentDetails, -} from "@nhs-notify-client-callbacks/test-support/helpers"; + CLIENT_FIXTURES, + type ClientFixtureKey, + getClientConfig, +} from "./helpers/mock-client-config"; +import sendEventToDlqAndRedrive from "./helpers/redrive"; import { assertCallbackHeaders } from "./helpers/signature"; import { - buildMockClientDlqQueueUrl, + awaitQueueMessage, + deleteMessage, ensureInboundQueueIsEmpty, + getQueueDepth, purgeQueues, sendSqsEvent, } from "./helpers/sqs"; import { - buildMockWebhookTargetPath, - getAllSubscriptionTargetIds, - getMockItClientConfig, -} from "./helpers/mock-client-config"; -import { awaitSignedCallbacksFromWebhookLogGroup } from "./helpers/cloudwatch"; -import { createMessageStatusPublishEvent } from "./helpers/event-factories"; -import sendEventToDlqAndRedrive from "./helpers/redrive"; + type TestContext, + createTestContext, + destroyTestContext, +} from "./helpers/test-context"; describe("DLQ Redrive", () => { - let sqsClient: SQSClient; - let cloudWatchClient: CloudWatchLogsClient; - let dlqQueueUrl!: string; - let allTargetDlqQueueUrls: string[]; - let inboundQueueUrl: string; - let webhookLogGroupName: string; + let ctx: TestContext; + let dlqUrl: string; + let deliveryUrl: string; + let allDlqUrls: string[]; beforeAll(async () => { - const deploymentDetails = getDeploymentDetails(); - const mockClient1 = getMockItClientConfig(); - - const allSubscriptionTargetIds = getAllSubscriptionTargetIds(); - - sqsClient = createSqsClient(deploymentDetails); - cloudWatchClient = createCloudWatchLogsClient(deploymentDetails); + ctx = createTestContext(); + const { clientId } = getClientConfig("clientSingleTarget"); - inboundQueueUrl = buildInboundEventQueueUrl(deploymentDetails); - dlqQueueUrl = buildMockClientDlqQueueUrl( - deploymentDetails, - mockClient1.targets, - ); - allTargetDlqQueueUrls = allSubscriptionTargetIds.map((targetId) => - buildMockClientDlqQueueUrl(deploymentDetails, [{ targetId }]), - ); - webhookLogGroupName = buildLambdaLogGroupName( - deploymentDetails, - "mock-webhook", + dlqUrl = ctx.clientDlqUrl(clientId); + deliveryUrl = ctx.clientDeliveryUrl(clientId); + allDlqUrls = (Object.keys(CLIENT_FIXTURES) as ClientFixtureKey[]).map( + (key) => ctx.clientDlqUrl(getClientConfig(key).clientId), ); - await purgeQueues(sqsClient, [inboundQueueUrl, ...allTargetDlqQueueUrls]); + await purgeQueues(ctx.sqs, [ + ctx.inboundQueueUrl, + deliveryUrl, + ...allDlqUrls, + ]); }); afterAll(async () => { - await purgeQueues(sqsClient, [inboundQueueUrl, ...allTargetDlqQueueUrls]); - sqsClient.destroy(); - cloudWatchClient.destroy(); + await purgeQueues(ctx.sqs, [ + ctx.inboundQueueUrl, + deliveryUrl, + ...allDlqUrls, + ]); + destroyTestContext(ctx); }); describe("Infrastructure validation", () => { - it("should confirm a target DLQ is accessible for all configured subscription targets", async () => { - const responses = await Promise.all( - allTargetDlqQueueUrls.map((queueUrl) => - sqsClient.send( - new GetQueueAttributesCommand({ - QueueUrl: queueUrl, - AttributeNames: ["QueueArn", "ApproximateNumberOfMessages"], - }), - ), - ), + it("should confirm a DLQ is accessible for all configured clients", async () => { + const depths = await Promise.all( + allDlqUrls.map((url) => getQueueDepth(ctx.sqs, url)), ); - for (const response of responses) { - expect(response.Attributes?.QueueArn).toBeDefined(); + for (const depth of depths) { + expect(depth).toBeGreaterThanOrEqual(0); } }); it("should confirm the inbound event queue exists and is accessible", async () => { - const response = await sqsClient.send( - new GetQueueAttributesCommand({ - QueueUrl: inboundQueueUrl, - AttributeNames: ["QueueArn", "ApproximateNumberOfMessages"], - }), - ); - - expect(response.Attributes?.QueueArn).toBeDefined(); + const depth = await getQueueDepth(ctx.sqs, ctx.inboundQueueUrl); + expect(depth).toBeGreaterThanOrEqual(0); }); }); @@ -101,33 +79,32 @@ describe("DLQ Redrive", () => { const startTime = Date.now(); const event: StatusPublishEvent = createMessageStatusPublishEvent(); + const { payload: redrivePayload } = await sendEventToDlqAndRedrive( - sqsClient, - dlqQueueUrl, - inboundQueueUrl, + ctx.sqs, + dlqUrl, + ctx.inboundQueueUrl, event, ); expect(redrivePayload.id).toBe(event.id); - await ensureInboundQueueIsEmpty(sqsClient, inboundQueueUrl); + await ensureInboundQueueIsEmpty(ctx.sqs, ctx.inboundQueueUrl); - const callbacks = await awaitSignedCallbacksFromWebhookLogGroup( - cloudWatchClient, - webhookLogGroupName, + const callback = await awaitCallback( + ctx.cwLogs, + ctx.webhookLogGroup, event.data.messageId, "MessageStatus", startTime, - buildMockWebhookTargetPath(), ); - expect(callbacks.length).toBeGreaterThan(0); - expect(callbacks[0].payload).toMatchObject({ + expect(callback.payload).toMatchObject({ type: "MessageStatus", attributes: expect.objectContaining({ messageStatus: "delivered", }), }); - assertCallbackHeaders(callbacks[0]); + assertCallbackHeaders(callback); }, 120_000); it("should apply the same transformation logic to redriven events as original deliveries", async () => { @@ -153,47 +130,90 @@ describe("DLQ Redrive", () => { }, }); - await sendSqsEvent(sqsClient, inboundQueueUrl, directEvent); + await sendSqsEvent(ctx.sqs, ctx.inboundQueueUrl, directEvent); const { payload: dlqPayload } = await sendEventToDlqAndRedrive( - sqsClient, - dlqQueueUrl, - inboundQueueUrl, + ctx.sqs, + dlqUrl, + ctx.inboundQueueUrl, redriveEvent, ); expect(dlqPayload.data.messageId).toBe(redriveEvent.data.messageId); - const [directCallbacks, redriveCallbacks] = await Promise.all([ - awaitSignedCallbacksFromWebhookLogGroup( - cloudWatchClient, - webhookLogGroupName, - directEvent.data.messageId, - "MessageStatus", - startTime, - buildMockWebhookTargetPath(), - ), - awaitSignedCallbacksFromWebhookLogGroup( - cloudWatchClient, - webhookLogGroupName, - redriveEvent.data.messageId, - "MessageStatus", - startTime, - buildMockWebhookTargetPath(), - ), - ]); - - await ensureInboundQueueIsEmpty(sqsClient, inboundQueueUrl); - - expect(redriveCallbacks[0].payload).toMatchObject({ - type: directCallbacks[0].payload.type, + const callbackMap = await awaitCallbacks( + ctx.cwLogs, + ctx.webhookLogGroup, + [directEvent.data.messageId, redriveEvent.data.messageId], + "MessageStatus", + 1, + startTime, + ); + + const directCallback = callbackMap.get(directEvent.data.messageId)![0]; + const redriveCallback = callbackMap.get(redriveEvent.data.messageId)![0]; + + await ensureInboundQueueIsEmpty(ctx.sqs, ctx.inboundQueueUrl); + + expect(redriveCallback.payload).toMatchObject({ + type: directCallback.payload.type, attributes: expect.objectContaining({ messageStatus: ( - directCallbacks[0].payload.attributes as { messageStatus?: string } + directCallback.payload.attributes as { messageStatus?: string } ).messageStatus, }), }); - assertCallbackHeaders(redriveCallbacks[0]); + assertCallbackHeaders(redriveCallback); }, 120_000); }); + + describe("Delivery DLQ redrive", () => { + it("should redrive a 4xx-failed message from the delivery DLQ back through the delivery queue", async () => { + const redriveStartTime = Date.now(); + const forceMessageId = `force-400-redrive-${crypto.randomUUID()}`; + + const failingEvent: StatusPublishEvent = + createMessageStatusPublishEvent({ + data: { messageId: forceMessageId }, + }); + + await sendSqsEvent(ctx.sqs, ctx.inboundQueueUrl, failingEvent); + + const dlqMessage = await awaitQueueMessage(ctx.sqs, dlqUrl, 90_000); + + expect(dlqMessage.Body).toBeDefined(); + expect(dlqMessage.MessageAttributes?.ERROR_CODE?.StringValue).toBe( + "HTTP_CLIENT_ERROR", + ); + + const dlqBody = JSON.parse(dlqMessage.Body as string) as { + payload: { data: { attributes: { messageId: string } }[] }; + subscriptionId: string; + targetId: string; + }; + + const redriveMessageId = `redriven-dlq-${crypto.randomUUID()}`; + dlqBody.payload.data[0].attributes.messageId = redriveMessageId; + + await sendSqsEvent(ctx.sqs, deliveryUrl, dlqBody); + await deleteMessage(ctx.sqs, dlqUrl, dlqMessage); + + const callback = await awaitCallback( + ctx.cwLogs, + ctx.webhookLogGroup, + redriveMessageId, + "MessageStatus", + redriveStartTime, + ); + + expect(callback.payload).toMatchObject({ + type: "MessageStatus", + attributes: expect.objectContaining({ + messageId: redriveMessageId, + messageStatus: "delivered", + }), + }); + assertCallbackHeaders(callback); + }, 180_000); + }); }); diff --git a/tests/integration/fixtures/subscriptions/mock-client-circuit-breaker.json b/tests/integration/fixtures/subscriptions/mock-client-circuit-breaker.json new file mode 100644 index 00000000..58243d3d --- /dev/null +++ b/tests/integration/fixtures/subscriptions/mock-client-circuit-breaker.json @@ -0,0 +1,40 @@ +{ + "clientId": "mock-client-circuit-breaker", + "subscriptions": [ + { + "messageStatuses": [ + "DELIVERED", + "FAILED" + ], + "subscriptionId": "sub-cb-msg-001", + "subscriptionType": "MessageStatus", + "targetIds": [ + "target-cb-001" + ] + } + ], + "targets": [ + { + "apiKey": { + "headerName": "x-api-key", + "headerValue": "REPLACED_BY_TERRAFORM" + }, + "delivery": { + "circuitBreaker": { + "enabled": true + }, + "mtls": { + "certPinning": { + "enabled": false + }, + "enabled": false + } + }, + "invocationEndpoint": "https://REPLACED_BY_TERRAFORM", + "invocationMethod": "POST", + "invocationRateLimit": 10, + "targetId": "target-cb-001", + "type": "API" + } + ] +} diff --git a/tests/integration/fixtures/subscriptions/mock-client-2.json b/tests/integration/fixtures/subscriptions/mock-client-fan-out.json similarity index 68% rename from tests/integration/fixtures/subscriptions/mock-client-2.json rename to tests/integration/fixtures/subscriptions/mock-client-fan-out.json index ee7091cd..0a606537 100644 --- a/tests/integration/fixtures/subscriptions/mock-client-2.json +++ b/tests/integration/fixtures/subscriptions/mock-client-fan-out.json @@ -1,5 +1,5 @@ { - "clientId": "mock-client-2", + "clientId": "mock-client-fan-out", "subscriptions": [ { "messageStatuses": [ @@ -20,6 +20,17 @@ "headerName": "x-api-key", "headerValue": "REPLACED_BY_TERRAFORM" }, + "delivery": { + "circuitBreaker": { + "enabled": false + }, + "mtls": { + "certPinning": { + "enabled": false + }, + "enabled": false + } + }, "invocationEndpoint": "https://REPLACED_BY_TERRAFORM", "invocationMethod": "POST", "invocationRateLimit": 10, @@ -31,6 +42,17 @@ "headerName": "x-api-key", "headerValue": "REPLACED_BY_TERRAFORM" }, + "delivery": { + "circuitBreaker": { + "enabled": false + }, + "mtls": { + "certPinning": { + "enabled": false + }, + "enabled": false + } + }, "invocationEndpoint": "https://REPLACED_BY_TERRAFORM", "invocationMethod": "POST", "invocationRateLimit": 10, diff --git a/tests/integration/fixtures/subscriptions/mock-client-mtls.json b/tests/integration/fixtures/subscriptions/mock-client-mtls.json new file mode 100644 index 00000000..3e78197a --- /dev/null +++ b/tests/integration/fixtures/subscriptions/mock-client-mtls.json @@ -0,0 +1,41 @@ +{ + "clientId": "mock-client-mtls", + "subscriptions": [ + { + "messageStatuses": [ + "DELIVERED", + "FAILED" + ], + "subscriptionId": "sub-mtls-msg-001", + "subscriptionType": "MessageStatus", + "targetIds": [ + "target-mtls-001" + ] + } + ], + "targets": [ + { + "apiKey": { + "headerName": "x-api-key", + "headerValue": "REPLACED_BY_TERRAFORM" + }, + "delivery": { + "circuitBreaker": { + "enabled": false + }, + "mtls": { + "certPinning": { + "enabled": true, + "spkiHash": "REPLACED_BY_TERRAFORM" + }, + "enabled": true + } + }, + "invocationEndpoint": "https://REPLACED_BY_TERRAFORM", + "invocationMethod": "POST", + "invocationRateLimit": 10, + "targetId": "target-mtls-001", + "type": "API" + } + ] +} diff --git a/tests/integration/fixtures/subscriptions/mock-client-rate-limit.json b/tests/integration/fixtures/subscriptions/mock-client-rate-limit.json new file mode 100644 index 00000000..0c798742 --- /dev/null +++ b/tests/integration/fixtures/subscriptions/mock-client-rate-limit.json @@ -0,0 +1,40 @@ +{ + "clientId": "mock-client-rate-limit", + "subscriptions": [ + { + "messageStatuses": [ + "DELIVERED", + "FAILED" + ], + "subscriptionId": "sub-rl-msg-001", + "subscriptionType": "MessageStatus", + "targetIds": [ + "target-rl-001" + ] + } + ], + "targets": [ + { + "apiKey": { + "headerName": "x-api-key", + "headerValue": "REPLACED_BY_TERRAFORM" + }, + "delivery": { + "circuitBreaker": { + "enabled": false + }, + "mtls": { + "certPinning": { + "enabled": false + }, + "enabled": false + } + }, + "invocationEndpoint": "https://REPLACED_BY_TERRAFORM", + "invocationMethod": "POST", + "invocationRateLimit": 1, + "targetId": "target-rl-001", + "type": "API" + } + ] +} diff --git a/tests/integration/fixtures/subscriptions/mock-client-short-retry.json b/tests/integration/fixtures/subscriptions/mock-client-short-retry.json new file mode 100644 index 00000000..2cc672d4 --- /dev/null +++ b/tests/integration/fixtures/subscriptions/mock-client-short-retry.json @@ -0,0 +1,41 @@ +{ + "clientId": "mock-client-short-retry", + "subscriptions": [ + { + "messageStatuses": [ + "DELIVERED", + "FAILED" + ], + "subscriptionId": "sub-sr-msg-001", + "subscriptionType": "MessageStatus", + "targetIds": [ + "target-sr-001" + ] + } + ], + "targets": [ + { + "apiKey": { + "headerName": "x-api-key", + "headerValue": "REPLACED_BY_TERRAFORM" + }, + "delivery": { + "circuitBreaker": { + "enabled": false + }, + "maxRetryDurationSeconds": 10, + "mtls": { + "certPinning": { + "enabled": false + }, + "enabled": false + } + }, + "invocationEndpoint": "https://REPLACED_BY_TERRAFORM", + "invocationMethod": "POST", + "invocationRateLimit": 10, + "targetId": "target-sr-001", + "type": "API" + } + ] +} diff --git a/tests/integration/fixtures/subscriptions/mock-client-1.json b/tests/integration/fixtures/subscriptions/mock-client-single-target.json similarity index 80% rename from tests/integration/fixtures/subscriptions/mock-client-1.json rename to tests/integration/fixtures/subscriptions/mock-client-single-target.json index 1e76ad65..ce8f9d11 100644 --- a/tests/integration/fixtures/subscriptions/mock-client-1.json +++ b/tests/integration/fixtures/subscriptions/mock-client-single-target.json @@ -1,5 +1,5 @@ { - "clientId": "mock-client-1", + "clientId": "mock-client-single-target", "subscriptions": [ { "messageStatuses": [ @@ -35,6 +35,17 @@ "headerName": "x-api-key", "headerValue": "REPLACED_BY_TERRAFORM" }, + "delivery": { + "circuitBreaker": { + "enabled": false + }, + "mtls": { + "certPinning": { + "enabled": false + }, + "enabled": false + } + }, "invocationEndpoint": "https://REPLACED_BY_TERRAFORM", "invocationMethod": "POST", "invocationRateLimit": 10, diff --git a/tests/integration/helpers/cloudwatch.ts b/tests/integration/helpers/cloudwatch.ts index 9ee13739..d66b18ce 100644 --- a/tests/integration/helpers/cloudwatch.ts +++ b/tests/integration/helpers/cloudwatch.ts @@ -6,266 +6,255 @@ import { logger } from "@nhs-notify-client-callbacks/logger"; import type { CallbackItem } from "@nhs-notify-client-callbacks/models"; import { TimeoutError, waitUntil } from "async-wait-until"; -const CALLBACK_WAIT_TIMEOUT_MS = 60_000; -const METRICS_WAIT_TIMEOUT_MS = 60_000; +const WAIT_TIMEOUT_MS = 60_000; const POLL_INTERVAL_MS = 2000; -const CLOUDWATCH_QUERY_LOOKBACK_MS = Number( - process.env.CLOUDWATCH_QUERY_LOOKBACK_MS ?? 5000, -); +const LOOKBACK_MS = Number(process.env.CLOUDWATCH_QUERY_LOOKBACK_MS ?? 5000); type LogEntry = { msg: string; - correlationId?: string; + messageId?: string; callbackType?: string; - clientId?: string; apiKey?: string; signature?: string; payload?: string; path?: string; + isMtls?: boolean; }; export type SignedCallback = { payload: CallbackItem; path: string; + isMtls: boolean; headers: { "x-api-key": string; "x-hmac-sha256-signature": string; }; }; -async function querySignedCallbacksFromWebhookLogGroup( - client: CloudWatchLogsClient, - logGroupName: string, - messageId: string, - callbackType: CallbackItem["type"], - startTime: number, -): Promise { - const filterPattern = `{ $.msg = "Callback received" && $.messageId = "${messageId}" && $.callbackType = "${callbackType}" }`; - const queryStartTime = Math.max(0, startTime - CLOUDWATCH_QUERY_LOOKBACK_MS); - - const response = await client.send( - new FilterLogEventsCommand({ - logGroupName, - startTime: queryStartTime, - filterPattern, - }), - ); - - const events = response.events ?? []; - const callbacks: SignedCallback[] = []; - - for (const event of events) { - if (event.message) { - try { - const entry = JSON.parse(event.message) as LogEntry; - if (entry.signature !== undefined && entry.payload) { - callbacks.push({ - payload: JSON.parse(entry.payload) as CallbackItem, - path: entry.path ?? "", - headers: { - "x-api-key": entry.apiKey ?? "", - "x-hmac-sha256-signature": entry.signature, - }, - }); - } - } catch { - // skip unparseable entries - } - } - } - - return callbacks; -} - -async function pollUntilFound( - poll: () => Promise, - timeoutMs: number, - timeoutMessage: string, -): Promise { - let results: T[] = []; - +// eslint-disable-next-line sonarjs/function-return-type -- returns SignedCallback | undefined consistently +function parseCallback( + message: string, + messageIdSet: Set, +): SignedCallback | undefined { try { - await waitUntil( - async () => { - results = await poll(); - return results.length > 0; + const entry = JSON.parse(message) as LogEntry; + if ( + !entry.messageId || + !messageIdSet.has(entry.messageId) || + entry.signature === undefined || + !entry.payload + ) + return undefined; + + return { + payload: JSON.parse(entry.payload) as CallbackItem, + path: entry.path ?? "", + isMtls: entry.isMtls ?? false, + headers: { + "x-api-key": entry.apiKey ?? "", + "x-hmac-sha256-signature": entry.signature, }, - { timeout: timeoutMs, intervalBetweenAttempts: POLL_INTERVAL_MS }, - ); - } catch (error) { - if (error instanceof TimeoutError) { - logger.warn(timeoutMessage); - } else { - throw error; - } + }; + } catch { + return undefined; } - - return results; } -export async function awaitSignedCallbacksFromWebhookLogGroup( +async function pollCallbacks( client: CloudWatchLogsClient, logGroupName: string, - messageId: string, + messageIds: string[], callbackType: CallbackItem["type"], + expectedPerMessage: number, startTime: number, - path: string, ): Promise { - const queryStartTime = Math.max(0, startTime - CLOUDWATCH_QUERY_LOOKBACK_MS); - logger.debug( - `Waiting for callback in webhook CloudWatch log group (messageId=${messageId}, path=${path}, logGroup=${logGroupName}, startTimeIso=${new Date(startTime).toISOString()}, queryStartTimeIso=${new Date(queryStartTime).toISOString()}, lookbackMs=${CLOUDWATCH_QUERY_LOOKBACK_MS})`, - ); - - const callbacks = await pollUntilFound( - () => - querySignedCallbacksFromWebhookLogGroup( - client, - logGroupName, - messageId, - callbackType, - startTime, - ), - CALLBACK_WAIT_TIMEOUT_MS, - `Timed out waiting for callback in webhook CloudWatch log group (messageId=${messageId}, callbackType=${callbackType}, path=${path}, timeoutMs=${CALLBACK_WAIT_TIMEOUT_MS})`, - ); - - if (callbacks.length !== 1) { - throw new Error( - `Expected exactly 1 callback for messageId="${messageId}" callbackType="${callbackType}", but found ${callbacks.length}`, - ); - } - - if (callbacks[0].path !== path) { - throw new Error( - `Expected callback path "${path}" for messageId="${messageId}", but got "${callbacks[0].path}"`, - ); - } - - return callbacks; -} + const messageIdSet = new Set(messageIds); + const expectedTotal = messageIds.length * expectedPerMessage; + const queryStartTime = Math.max(0, startTime - LOOKBACK_MS); + const filterPattern = `{ $.msg = "Callback received" && $.callbackType = "${callbackType}" }`; -export async function awaitSignedCallbacksByCountFromWebhookLogGroup( - client: CloudWatchLogsClient, - logGroupName: string, - messageId: string, - callbackType: CallbackItem["type"], - expectedCount: number, - startTime: number, -): Promise { logger.debug( - `Waiting for callbacks in webhook CloudWatch log group (messageId=${messageId}, callbackType=${callbackType}, expectedCount=${expectedCount}, logGroup=${logGroupName})`, + `Waiting for ${expectedTotal} callback(s) (type=${callbackType}, messages=${messageIds.length}, logGroup=${logGroupName})`, ); - let callbacks: SignedCallback[] = []; + let matched: SignedCallback[] = []; try { await waitUntil( async () => { - callbacks = await querySignedCallbacksFromWebhookLogGroup( - client, - logGroupName, - messageId, - callbackType, - startTime, + const response = await client.send( + new FilterLogEventsCommand({ + logGroupName, + startTime: queryStartTime, + filterPattern, + }), ); - return callbacks.length === expectedCount; - }, - { - timeout: CALLBACK_WAIT_TIMEOUT_MS, - intervalBetweenAttempts: POLL_INTERVAL_MS, + + matched = (response.events ?? []) + .filter((event): event is typeof event & { message: string } => + Boolean(event.message), + ) + .map((event) => parseCallback(event.message, messageIdSet)) + .filter((cb): cb is SignedCallback => cb !== undefined); + + return matched.length >= expectedTotal; }, + { timeout: WAIT_TIMEOUT_MS, intervalBetweenAttempts: POLL_INTERVAL_MS }, ); } catch (error) { if (error instanceof TimeoutError) { logger.warn( - `Timed out waiting for callbacks in webhook CloudWatch log group (messageId=${messageId}, callbackType=${callbackType}, expectedCount=${expectedCount}, timeoutMs=${CALLBACK_WAIT_TIMEOUT_MS})`, + `Timed out waiting for callbacks (expected=${expectedTotal}, found=${matched.length})`, ); } else { throw error; } } - if (callbacks.length !== expectedCount) { + if (matched.length !== expectedTotal) { + const foundIds = new Set( + matched.map( + (cb) => + (cb.payload.attributes as { messageId?: string }).messageId ?? "", + ), + ); + const missingIds = messageIds.filter((id) => !foundIds.has(id)); + logger.warn("Missing callbacks", { + callbackType, + expectedTotal, + foundCount: matched.length, + missingIds, + }); throw new Error( - `Expected exactly ${expectedCount} callbacks for messageId="${messageId}" callbackType="${callbackType}", but found ${callbacks.length}`, + `Expected ${expectedTotal} callback(s) for type="${callbackType}", found ${matched.length}`, ); } - return callbacks; + return matched; } -type EmfEntry = Record; - -function collectMetricNamesFromEvent( - message: string, - metricNames: string[], - found: Set, -): void { - try { - const entry = JSON.parse(message) as EmfEntry; - if (entry._aws) { - for (const name of metricNames) { - if (name in entry) found.add(name); - } - } - } catch { - // skip unparseable entries - } +export async function awaitCallback( + client: CloudWatchLogsClient, + logGroupName: string, + messageId: string, + callbackType: CallbackItem["type"], + startTime: number, +): Promise { + const [callback] = await pollCallbacks( + client, + logGroupName, + [messageId], + callbackType, + 1, + startTime, + ); + return callback; } -async function queryEmfMetricsFromLogGroup( +export async function awaitCallbacks( client: CloudWatchLogsClient, logGroupName: string, - metricNames: string[], + messageIds: string[], + callbackType: CallbackItem["type"], + expectedPerMessage: number, startTime: number, -): Promise> { - const queryStartTime = Math.max(0, startTime - CLOUDWATCH_QUERY_LOOKBACK_MS); - const conditions = metricNames.map((name) => `$.${name} > 0`).join(" || "); - const filterPattern = `{ ${conditions} }`; - - const response = await client.send( - new FilterLogEventsCommand({ - logGroupName, - startTime: queryStartTime, - filterPattern, - }), +): Promise> { + const results = await pollCallbacks( + client, + logGroupName, + messageIds, + callbackType, + expectedPerMessage, + startTime, ); - const found = new Set(); - for (const event of response.events ?? []) { - if (event.message) { - collectMetricNamesFromEvent(event.message, metricNames, found); - } + const map = new Map(); + for (const cb of results) { + const id = + (cb.payload.attributes as { messageId?: string }).messageId ?? ""; + const existing = map.get(id) ?? []; + existing.push(cb); + map.set(id, existing); } - return found; + return map; } -export async function awaitAllEmfMetricsInLogGroup( +export async function awaitEmfMetrics( client: CloudWatchLogsClient, logGroupName: string, metricNames: string[], startTime: number, ): Promise { - const queryStartTime = Math.max(0, startTime - CLOUDWATCH_QUERY_LOOKBACK_MS); - const queryStartTimeIso = new Date(queryStartTime).toISOString(); - const startTimeIso = new Date(startTime).toISOString(); + const queryStartTime = Math.max(0, startTime - LOOKBACK_MS); + const conditions = metricNames.map((name) => `$.${name} > 0`).join(" || "); + const filterPattern = `{ ${conditions} }`; + logger.debug( - `Waiting for EMF metrics in CloudWatch log group (metrics=${metricNames.join(",")}, logGroup=${logGroupName}, startTimeIso=${startTimeIso}, queryStartTimeIso=${queryStartTimeIso}, lookbackMs=${CLOUDWATCH_QUERY_LOOKBACK_MS})`, + `Waiting for EMF metrics [${metricNames.join(", ")}] in ${logGroupName}`, ); await waitUntil( async () => { - const found = await queryEmfMetricsFromLogGroup( - client, - logGroupName, - metricNames, - startTime, + const response = await client.send( + new FilterLogEventsCommand({ + logGroupName, + startTime: queryStartTime, + filterPattern, + }), ); + + const found = new Set(); + for (const event of response.events ?? []) { + try { + const entry = JSON.parse(event.message ?? "") as Record< + string, + unknown + >; + if (entry._aws) { + for (const name of metricNames) { + if (name in entry) found.add(name); + } + } + } catch { + // skip unparseable entries + } + } return metricNames.every((name) => found.has(name)); }, - { - timeout: METRICS_WAIT_TIMEOUT_MS, - intervalBetweenAttempts: POLL_INTERVAL_MS, - }, + { timeout: WAIT_TIMEOUT_MS, intervalBetweenAttempts: POLL_INTERVAL_MS }, ); } + +export async function countLogEntries( + client: CloudWatchLogsClient, + logGroupName: string, + filterPattern: string, + startTime: number, + minCount: number, +): Promise { + const queryStartTime = Math.max(0, startTime - LOOKBACK_MS); + + let count = 0; + try { + await waitUntil( + async () => { + const response = await client.send( + new FilterLogEventsCommand({ + logGroupName, + startTime: queryStartTime, + filterPattern, + }), + ); + count = (response.events ?? []).length; + return count >= minCount; + }, + { timeout: WAIT_TIMEOUT_MS, intervalBetweenAttempts: POLL_INTERVAL_MS }, + ); + } catch (error) { + if (!(error instanceof TimeoutError)) { + throw error; + } + } + + return count; +} diff --git a/tests/integration/helpers/mock-client-config.ts b/tests/integration/helpers/mock-client-config.ts index eb94974c..52bb3570 100644 --- a/tests/integration/helpers/mock-client-config.ts +++ b/tests/integration/helpers/mock-client-config.ts @@ -1,6 +1,6 @@ import { readFileSync } from "node:fs"; import path from "node:path"; -import type seedConfigJson from "../fixtures/subscriptions/mock-client-1.json"; +import type seedConfigJson from "../fixtures/subscriptions/mock-client-single-target.json"; type ClientFixtureShape = typeof seedConfigJson; @@ -10,28 +10,40 @@ export type MockItClientConfig = ClientFixtureShape & { }; export const CLIENT_FIXTURES = { - client1: { - fixture: "mock-client-1.json", + clientSingleTarget: { + fixture: "mock-client-single-target.json", apiKeyVar: "MOCK_CLIENT_API_KEY", applicationIdVar: "MOCK_CLIENT_APPLICATION_ID", }, - client2: { - fixture: "mock-client-2.json", - apiKeyVar: "MOCK_CLIENT_2_API_KEY", - applicationIdVar: "MOCK_CLIENT_2_APPLICATION_ID", + clientFanOut: { + fixture: "mock-client-fan-out.json", + apiKeyVar: "MOCK_CLIENT_FAN_OUT_API_KEY", + applicationIdVar: "MOCK_CLIENT_FAN_OUT_APPLICATION_ID", + }, + clientMtls: { + fixture: "mock-client-mtls.json", + apiKeyVar: "MOCK_CLIENT_MTLS_API_KEY", + applicationIdVar: "MOCK_CLIENT_MTLS_APPLICATION_ID", + }, + clientRateLimit: { + fixture: "mock-client-rate-limit.json", + apiKeyVar: "MOCK_CLIENT_RATE_LIMIT_API_KEY", + applicationIdVar: "MOCK_CLIENT_RATE_LIMIT_APPLICATION_ID", + }, + clientCircuitBreaker: { + fixture: "mock-client-circuit-breaker.json", + apiKeyVar: "MOCK_CLIENT_CIRCUIT_BREAKER_API_KEY", + applicationIdVar: "MOCK_CLIENT_CIRCUIT_BREAKER_APPLICATION_ID", + }, + clientShortRetry: { + fixture: "mock-client-short-retry.json", + apiKeyVar: "MOCK_CLIENT_SHORT_RETRY_API_KEY", + applicationIdVar: "MOCK_CLIENT_SHORT_RETRY_APPLICATION_ID", }, } as const; export type ClientFixtureKey = keyof typeof CLIENT_FIXTURES; -const ALL_CLIENT_FIXTURE_KEYS = Object.keys( - CLIENT_FIXTURES, -) as ClientFixtureKey[]; - -function dedupe(values: string[]): string[] { - return [...new Set(values)]; -} - export function getClientConfig(key: ClientFixtureKey): MockItClientConfig { // eslint-disable-next-line security/detect-object-injection -- key is constrained to ClientFixtureKey, a keyof the hardcoded as-const CLIENT_FIXTURES object const { apiKeyVar, applicationIdVar, fixture } = CLIENT_FIXTURES[key]; @@ -48,11 +60,7 @@ export function getClientConfig(key: ClientFixtureKey): MockItClientConfig { } export function getMockItClientConfig(): MockItClientConfig { - return getClientConfig("client1"); -} - -export function getMockItClient2Config(): MockItClientConfig { - return getClientConfig("client2"); + return getClientConfig("clientSingleTarget"); } function buildWebhookTargetPaths(key: ClientFixtureKey): string[] { @@ -61,7 +69,7 @@ function buildWebhookTargetPaths(key: ClientFixtureKey): string[] { } export function buildMockWebhookTargetPath( - key: ClientFixtureKey = "client1", + key: ClientFixtureKey = "clientSingleTarget", ): string { const paths = buildWebhookTargetPaths(key); @@ -73,22 +81,7 @@ export function buildMockWebhookTargetPath( } export function buildMockWebhookTargetPaths( - key: ClientFixtureKey = "client1", + key: ClientFixtureKey = "clientSingleTarget", ): string[] { return buildWebhookTargetPaths(key); } - -export function getSubscriptionTargetIds( - key: ClientFixtureKey = "client1", -): string[] { - const config = getClientConfig(key); - return dedupe( - config.subscriptions.flatMap((subscription) => subscription.targetIds), - ); -} - -export function getAllSubscriptionTargetIds( - keys: ClientFixtureKey[] = ALL_CLIENT_FIXTURE_KEYS, -): string[] { - return dedupe(keys.flatMap((key) => getSubscriptionTargetIds(key))); -} diff --git a/tests/integration/helpers/sqs.ts b/tests/integration/helpers/sqs.ts index 857fd3a7..1420b19d 100644 --- a/tests/integration/helpers/sqs.ts +++ b/tests/integration/helpers/sqs.ts @@ -1,5 +1,6 @@ import { ChangeMessageVisibilityCommand, + DeleteMessageCommand, GetQueueAttributesCommand, type Message, PurgeQueueCommand, @@ -33,7 +34,8 @@ function buildReceiveMessageInput( } function buildQueueUrl( - { accountId, component, environment, project, region }: DeploymentDetails, + { accountId, environment, project, region }: DeploymentDetails, + component: string, name: string, options?: { appendQueueSuffix?: boolean }, ): string { @@ -46,13 +48,24 @@ function buildQueueUrl( export function buildMockClientDlqQueueUrl( deploymentDetails: DeploymentDetails, - targets: { targetId: string }[], + clientId: string, ): string { - const [firstTarget] = targets; - if (!firstTarget) { - throw new Error("At least one target is required to build DLQ URL"); - } - return buildQueueUrl(deploymentDetails, `${firstTarget.targetId}-dlq`); + return buildQueueUrl( + deploymentDetails, + deploymentDetails.clientComponent, + `${clientId}-delivery-dlq`, + ); +} + +export function buildMockClientDeliveryQueueUrl( + deploymentDetails: DeploymentDetails, + clientId: string, +): string { + return buildQueueUrl( + deploymentDetails, + deploymentDetails.clientComponent, + `${clientId}-delivery`, + ); } export async function sendSqsEvent( @@ -140,9 +153,11 @@ export async function purgeQueue( }), ); } catch (error) { - if (!(error instanceof Error) || error.name !== "PurgeQueueInProgress") { - throw error; + if (error instanceof Error && error.name === "PurgeQueueInProgress") { + logger.warn(`Purge already in progress, skipping (${queueUrl})`); + return; } + throw error; } } @@ -165,6 +180,7 @@ async function receiveOneMessage(client: SQSClient, queueUrl: string) { export async function awaitQueueMessage( client: SQSClient, queueUrl: string, + timeoutMs: number = QUEUE_WAIT_TIMEOUT_MS, ): Promise { let message: Message | undefined; @@ -176,13 +192,13 @@ export async function awaitQueueMessage( }, { intervalBetweenAttempts: POLL_INTERVAL_MS, - timeout: QUEUE_WAIT_TIMEOUT_MS, + timeout: timeoutMs, }, ); if (!message) { throw new Error( - `Timed out after ${QUEUE_WAIT_TIMEOUT_MS}ms waiting for a message to appear in ${queueUrl}`, + `Timed out after ${timeoutMs}ms waiting for a message to appear in ${queueUrl}`, ); } @@ -246,3 +262,26 @@ export async function awaitQueueMessageByMessageId( return matchedMessage; } + +export async function deleteMessage( + client: SQSClient, + queueUrl: string, + message: Message, +): Promise { + await client.send( + new DeleteMessageCommand({ + QueueUrl: queueUrl, + ReceiptHandle: message.ReceiptHandle!, + }), + ); +} + +export async function getQueueDepth( + client: SQSClient, + queueUrl: string, +): Promise { + return getQueueMessageCount(client, queueUrl, [ + "ApproximateNumberOfMessages", + "ApproximateNumberOfMessagesNotVisible", + ]); +} diff --git a/tests/integration/helpers/status-events.ts b/tests/integration/helpers/status-events.ts deleted file mode 100644 index 1bccf0bb..00000000 --- a/tests/integration/helpers/status-events.ts +++ /dev/null @@ -1,89 +0,0 @@ -import { CloudWatchLogsClient } from "@aws-sdk/client-cloudwatch-logs"; -import { SQSClient } from "@aws-sdk/client-sqs"; -import type { - ChannelStatusData, - MessageStatusData, - StatusPublishEvent, -} from "@nhs-notify-client-callbacks/models"; - -import { - type SignedCallback, - awaitSignedCallbacksFromWebhookLogGroup, -} from "./cloudwatch"; -import { ensureInboundQueueIsEmpty, sendSqsEvent } from "./sqs"; - -async function processStatusEvent< - T extends MessageStatusData | ChannelStatusData, ->( - { - CloudWatchLogsClient: cloudWatchClient, - SQSClient: sqsClient, - }: { CloudWatchLogsClient: CloudWatchLogsClient; SQSClient: SQSClient }, - callbackEventQueueUrl: string, - webhookLogGroupName: string, - event: StatusPublishEvent, - callbackType: SignedCallback["payload"]["type"], - webhookPath: string, - startTime: number, -): Promise { - const sendMessageResponse = await sendSqsEvent( - sqsClient, - callbackEventQueueUrl, - event, - ); - - if (!sendMessageResponse.MessageId) { - throw new Error("Expected SQS send response to include MessageId"); - } - - await ensureInboundQueueIsEmpty(sqsClient, callbackEventQueueUrl); - - return awaitSignedCallbacksFromWebhookLogGroup( - cloudWatchClient, - webhookLogGroupName, - event.data.messageId, - callbackType, - startTime, - webhookPath, - ); -} - -export async function processMessageStatusEvent( - sqsClient: SQSClient, - cloudWatchClient: CloudWatchLogsClient, - callbackEventQueueUrl: string, - webhookLogGroupName: string, - messageStatusEvent: StatusPublishEvent, - webhookPath: string, - startTime: number, -): Promise { - return processStatusEvent( - { CloudWatchLogsClient: cloudWatchClient, SQSClient: sqsClient }, - callbackEventQueueUrl, - webhookLogGroupName, - messageStatusEvent, - "MessageStatus", - webhookPath, - startTime, - ); -} - -export async function processChannelStatusEvent( - sqsClient: SQSClient, - cloudWatchClient: CloudWatchLogsClient, - callbackEventQueueUrl: string, - webhookLogGroupName: string, - channelStatusEvent: StatusPublishEvent, - webhookPath: string, - startTime: number, -): Promise { - return processStatusEvent( - { CloudWatchLogsClient: cloudWatchClient, SQSClient: sqsClient }, - callbackEventQueueUrl, - webhookLogGroupName, - channelStatusEvent, - "ChannelStatus", - webhookPath, - startTime, - ); -} diff --git a/tests/integration/helpers/test-context.ts b/tests/integration/helpers/test-context.ts new file mode 100644 index 00000000..8277f94b --- /dev/null +++ b/tests/integration/helpers/test-context.ts @@ -0,0 +1,55 @@ +import type { CloudWatchLogsClient } from "@aws-sdk/client-cloudwatch-logs"; +import type { SQSClient } from "@aws-sdk/client-sqs"; +import type { DeploymentDetails } from "@nhs-notify-client-callbacks/test-support/helpers"; +import { + buildInboundEventDlqQueueUrl, + buildInboundEventQueueUrl, + buildLambdaLogGroupName, + createCloudWatchLogsClient, + createSqsClient, + getDeploymentDetails, +} from "@nhs-notify-client-callbacks/test-support/helpers"; +import { + buildMockClientDeliveryQueueUrl, + buildMockClientDlqQueueUrl, +} from "./sqs"; + +export type TestContext = { + sqs: SQSClient; + cwLogs: CloudWatchLogsClient; + deployment: DeploymentDetails; + inboundQueueUrl: string; + inboundDlqUrl: string; + webhookLogGroup: string; + startTime: number; + clientDlqUrl(clientId: string): string; + clientDeliveryUrl(clientId: string): string; + logGroup(name: string): string; + clientLogGroup(name: string): string; +}; + +export function createTestContext(): TestContext { + const deployment = getDeploymentDetails(); + + return { + sqs: createSqsClient(deployment), + cwLogs: createCloudWatchLogsClient(deployment), + deployment, + inboundQueueUrl: buildInboundEventQueueUrl(deployment), + inboundDlqUrl: buildInboundEventDlqQueueUrl(deployment), + webhookLogGroup: buildLambdaLogGroupName(deployment, "mock-webhook"), + startTime: Date.now(), + clientDlqUrl: (clientId) => + buildMockClientDlqQueueUrl(deployment, clientId), + clientDeliveryUrl: (clientId) => + buildMockClientDeliveryQueueUrl(deployment, clientId), + logGroup: (name) => buildLambdaLogGroupName(deployment, name), + clientLogGroup: (name) => + `/aws/lambda/${deployment.project}-${deployment.environment}-${deployment.clientComponent}-${name}`, + }; +} + +export function destroyTestContext(ctx: TestContext): void { + ctx.sqs.destroy(); + ctx.cwLogs.destroy(); +} diff --git a/tests/integration/inbound-sqs-to-webhook.test.ts b/tests/integration/inbound-sqs-to-webhook.test.ts index 4305f05e..13d8ec14 100644 --- a/tests/integration/inbound-sqs-to-webhook.test.ts +++ b/tests/integration/inbound-sqs-to-webhook.test.ts @@ -1,19 +1,9 @@ -import { CloudWatchLogsClient } from "@aws-sdk/client-cloudwatch-logs"; -import { DeleteMessageCommand, SQSClient } from "@aws-sdk/client-sqs"; -import { - type ChannelStatusData, - type MessageStatusData, - type StatusPublishEvent, +import type { + ChannelStatusData, + MessageStatusData, + StatusPublishEvent, } from "@nhs-notify-client-callbacks/models"; -import { - buildInboundEventDlqQueueUrl, - buildInboundEventQueueUrl, - buildLambdaLogGroupName, - createCloudWatchLogsClient, - createSqsClient, - getDeploymentDetails, -} from "@nhs-notify-client-callbacks/test-support/helpers"; -import { awaitSignedCallbacksByCountFromWebhookLogGroup } from "./helpers/cloudwatch"; +import { awaitCallback, awaitCallbacks } from "./helpers/cloudwatch"; import { createChannelStatusPublishEvent, createMessageStatusPublishEvent, @@ -21,143 +11,112 @@ import { import { buildMockWebhookTargetPath, buildMockWebhookTargetPaths, - getMockItClient2Config, - getMockItClientConfig, + getClientConfig, } from "./helpers/mock-client-config"; import { assertCallbackHeaders } from "./helpers/signature"; import { awaitQueueMessage, awaitQueueMessageByMessageId, - buildMockClientDlqQueueUrl, + deleteMessage, ensureInboundQueueIsEmpty, purgeQueues, sendSqsEvent, } from "./helpers/sqs"; import { - processChannelStatusEvent, - processMessageStatusEvent, -} from "./helpers/status-events"; - -function compareStrings(a: string, b: string): number { - if (a > b) return 1; - if (a < b) return -1; - return 0; -} + type TestContext, + createTestContext, + destroyTestContext, +} from "./helpers/test-context"; describe("SQS to Webhook Integration", () => { - let sqsClient: SQSClient; - let cloudWatchClient: CloudWatchLogsClient; - let callbackEventQueueUrl: string; - let clientDlqQueueUrl: string; - let inboundEventDlqQueueUrl: string; - let webhookLogGroupName: string; - let webhookTargetPath: string; - let startTime: number; + let ctx: TestContext; + let clientDlqUrl: string; + let clientDeliveryUrl: string; beforeAll(async () => { - const deploymentDetails = getDeploymentDetails(); - const { targets } = getMockItClientConfig(); - - sqsClient = createSqsClient(deploymentDetails); - cloudWatchClient = createCloudWatchLogsClient(deploymentDetails); - callbackEventQueueUrl = buildInboundEventQueueUrl(deploymentDetails); - clientDlqQueueUrl = buildMockClientDlqQueueUrl(deploymentDetails, targets); - inboundEventDlqQueueUrl = buildInboundEventDlqQueueUrl(deploymentDetails); - webhookLogGroupName = buildLambdaLogGroupName( - deploymentDetails, - "mock-webhook", - ); - webhookTargetPath = buildMockWebhookTargetPath(); - startTime = Date.now(); - await purgeQueues(sqsClient, [ - inboundEventDlqQueueUrl, - clientDlqQueueUrl, - callbackEventQueueUrl, + ctx = createTestContext(); + const { clientId } = getClientConfig("clientSingleTarget"); + clientDlqUrl = ctx.clientDlqUrl(clientId); + clientDeliveryUrl = ctx.clientDeliveryUrl(clientId); + await purgeQueues(ctx.sqs, [ + ctx.inboundDlqUrl, + clientDlqUrl, + clientDeliveryUrl, + ctx.inboundQueueUrl, ]); }); afterAll(async () => { - await purgeQueues(sqsClient, [ - inboundEventDlqQueueUrl, - clientDlqQueueUrl, - callbackEventQueueUrl, + await purgeQueues(ctx.sqs, [ + ctx.inboundDlqUrl, + clientDlqUrl, + clientDeliveryUrl, + ctx.inboundQueueUrl, ]); - - sqsClient.destroy(); - cloudWatchClient.destroy(); + destroyTestContext(ctx); }); describe("Message Status Event Flow", () => { it("should process message status event from SQS to webhook", async () => { - const messageStatusEvent: StatusPublishEvent = - createMessageStatusPublishEvent(); - - const callbacks = await processMessageStatusEvent( - sqsClient, - cloudWatchClient, - callbackEventQueueUrl, - webhookLogGroupName, - messageStatusEvent, - webhookTargetPath, - startTime, - ); + const event = createMessageStatusPublishEvent(); - expect(callbacks).toHaveLength(1); + await sendSqsEvent(ctx.sqs, ctx.inboundQueueUrl, event); + await ensureInboundQueueIsEmpty(ctx.sqs, ctx.inboundQueueUrl); - expect(callbacks[0].payload).toMatchObject({ - type: "MessageStatus", + const callback = await awaitCallback( + ctx.cwLogs, + ctx.webhookLogGroup, + event.data.messageId, + "MessageStatus", + ctx.startTime, + ); - attributes: expect.objectContaining({ - messageStatus: "delivered", - }), + expect(callback.payload).toMatchObject({ + type: "MessageStatus", + attributes: expect.objectContaining({ messageStatus: "delivered" }), }); - - assertCallbackHeaders(callbacks[0]); + assertCallbackHeaders(callback); }, 120_000); it("should fan out a message status event to subscription with multiple target endpoints", async () => { - const client2Config = getMockItClient2Config(); - const expectedPaths = buildMockWebhookTargetPaths("client2"); + const fanOutConfig = getClientConfig("clientFanOut"); + const expectedPaths = buildMockWebhookTargetPaths("clientFanOut"); - const messageStatusEvent: StatusPublishEvent = + const event: StatusPublishEvent = createMessageStatusPublishEvent({ - data: { - clientId: client2Config.clientId, - }, + data: { clientId: fanOutConfig.clientId }, }); - await sendSqsEvent(sqsClient, callbackEventQueueUrl, messageStatusEvent); - await ensureInboundQueueIsEmpty(sqsClient, callbackEventQueueUrl); + await sendSqsEvent(ctx.sqs, ctx.inboundQueueUrl, event); + await ensureInboundQueueIsEmpty(ctx.sqs, ctx.inboundQueueUrl); - const callbacks = await awaitSignedCallbacksByCountFromWebhookLogGroup( - cloudWatchClient, - webhookLogGroupName, - messageStatusEvent.data.messageId, + const callbackMap = await awaitCallbacks( + ctx.cwLogs, + ctx.webhookLogGroup, + [event.data.messageId], "MessageStatus", expectedPaths.length, - startTime, + ctx.startTime, ); - expect(callbacks).toHaveLength(expectedPaths.length); + const callbacks = callbackMap.get(event.data.messageId)!; - const actualPaths = callbacks - .map((callback) => callback.path) - .toSorted(compareStrings); - expect(actualPaths).toEqual(expectedPaths.toSorted(compareStrings)); + const paths = callbacks.map((cb) => cb.path); + expect(paths).toHaveLength(expectedPaths.length); + expect(paths).toEqual(expect.arrayContaining(expectedPaths)); for (const callback of callbacks) { expect(callback.payload).toMatchObject({ type: "MessageStatus", attributes: expect.objectContaining({ - messageId: messageStatusEvent.data.messageId, + messageId: event.data.messageId, messageStatus: "delivered", }), }); - assertCallbackHeaders( callback, - client2Config.apiKeyVar, - client2Config.applicationIdVar, + fanOutConfig.apiKeyVar, + fanOutConfig.applicationIdVar, ); } }, 120_000); @@ -165,86 +124,73 @@ describe("SQS to Webhook Integration", () => { describe("Channel Status Event Flow", () => { it("should process channel status event from SQS to webhook", async () => { - const channelStatusEvent: StatusPublishEvent = + const event: StatusPublishEvent = createChannelStatusPublishEvent(); - const callbacks = await processChannelStatusEvent( - sqsClient, - cloudWatchClient, - callbackEventQueueUrl, - webhookLogGroupName, - channelStatusEvent, - webhookTargetPath, - startTime, - ); + await sendSqsEvent(ctx.sqs, ctx.inboundQueueUrl, event); + await ensureInboundQueueIsEmpty(ctx.sqs, ctx.inboundQueueUrl); - expect(callbacks).toHaveLength(1); + const callback = await awaitCallback( + ctx.cwLogs, + ctx.webhookLogGroup, + event.data.messageId, + "ChannelStatus", + ctx.startTime, + ); - expect(callbacks[0].payload).toMatchObject({ + expect(callback.payload).toMatchObject({ type: "ChannelStatus", attributes: expect.objectContaining({ channel: "nhsapp", channelStatus: "delivered", supplierStatus: "delivered", - messageId: channelStatusEvent.data.messageId, + messageId: event.data.messageId, }), }); - - assertCallbackHeaders(callbacks[0]); + assertCallbackHeaders(callback); }, 120_000); }); describe("Client Webhook DLQ", () => { - it("should route a non-retriable (4xx) webhook response to the per-target DLQ", async () => { + it("should route a non-retriable (4xx) webhook response to the per-client DLQ", async () => { const event: StatusPublishEvent = createMessageStatusPublishEvent({ - data: { - messageId: `force-400-${Date.now()}`, - }, + data: { messageId: `force-400-${crypto.randomUUID()}` }, }); - await sendSqsEvent(sqsClient, callbackEventQueueUrl, event); + await sendSqsEvent(ctx.sqs, ctx.inboundQueueUrl, event); - const dlqMessage = await awaitQueueMessage(sqsClient, clientDlqQueueUrl); + const dlqMessage = await awaitQueueMessage(ctx.sqs, clientDlqUrl); expect(dlqMessage.Body).toBeDefined(); expect(dlqMessage.MessageAttributes?.ERROR_CODE?.StringValue).toBe( - "INVALID_PARAMETER", + "HTTP_CLIENT_ERROR", ); expect( dlqMessage.MessageAttributes?.ERROR_MESSAGE?.StringValue, ).toContain("Forced status 400"); - await sqsClient.send( - new DeleteMessageCommand({ - QueueUrl: clientDlqQueueUrl, - ReceiptHandle: dlqMessage.ReceiptHandle!, - }), - ); + await deleteMessage(ctx.sqs, clientDlqUrl, dlqMessage); }, 120_000); }); describe("Inbound Event DLQ", () => { it("should move an invalid inbound event to the inbound-event DLQ when schema validation fails", async () => { - const messageId = `invalid-schema-${Date.now()}`; + const messageId = `invalid-schema-${crypto.randomUUID()}`; const invalidEvent = createMessageStatusPublishEvent({ data: { messageId, - channels: [ - // @ts-expect-error - intentionally invalid for schema-failure DLQ path - { - channelStatus: "DELIVERED", - }, - ], + // @ts-expect-error - intentionally invalid for schema-failure DLQ path + channels: [{ channelStatus: "DELIVERED" }], }, }); - await sendSqsEvent(sqsClient, callbackEventQueueUrl, invalidEvent); - await ensureInboundQueueIsEmpty(sqsClient, callbackEventQueueUrl); + await sendSqsEvent(ctx.sqs, ctx.inboundQueueUrl, invalidEvent); + await ensureInboundQueueIsEmpty(ctx.sqs, ctx.inboundQueueUrl); const dlqMessage = await awaitQueueMessageByMessageId( - sqsClient, - inboundEventDlqQueueUrl, + ctx.sqs, + ctx.inboundDlqUrl, messageId, ); @@ -252,11 +198,43 @@ describe("SQS to Webhook Integration", () => { const dlqPayload = JSON.parse(dlqMessage.Body as string); expect(dlqPayload.data.messageId).toBe(messageId); - await sqsClient.send( - new DeleteMessageCommand({ - QueueUrl: inboundEventDlqQueueUrl, - ReceiptHandle: dlqMessage.ReceiptHandle!, + await deleteMessage(ctx.sqs, ctx.inboundDlqUrl, dlqMessage); + }, 120_000); + }); + + describe("mTLS Delivery", () => { + it("should deliver a callback via mTLS to the mTLS-secured mock webhook", async () => { + const mtlsConfig = getClientConfig("clientMtls"); + + const event: StatusPublishEvent = + createMessageStatusPublishEvent({ + data: { clientId: mtlsConfig.clientId }, + }); + + await sendSqsEvent(ctx.sqs, ctx.inboundQueueUrl, event); + await ensureInboundQueueIsEmpty(ctx.sqs, ctx.inboundQueueUrl); + + const callback = await awaitCallback( + ctx.cwLogs, + ctx.webhookLogGroup, + event.data.messageId, + "MessageStatus", + ctx.startTime, + ); + + expect(callback.path).toBe(buildMockWebhookTargetPath("clientMtls")); + expect(callback.isMtls).toBe(true); + expect(callback.payload).toMatchObject({ + type: "MessageStatus", + attributes: expect.objectContaining({ + messageId: event.data.messageId, + messageStatus: "delivered", }), + }); + assertCallbackHeaders( + callback, + mtlsConfig.apiKeyVar, + mtlsConfig.applicationIdVar, ); }, 120_000); }); diff --git a/tests/integration/metrics.test.ts b/tests/integration/metrics.test.ts index 2f314f85..cd99588b 100644 --- a/tests/integration/metrics.test.ts +++ b/tests/integration/metrics.test.ts @@ -1,77 +1,50 @@ -import { DeleteMessageCommand, SQSClient } from "@aws-sdk/client-sqs"; -import { CloudWatchLogsClient } from "@aws-sdk/client-cloudwatch-logs"; import type { MessageStatusData, StatusPublishEvent, } from "@nhs-notify-client-callbacks/models"; +import { awaitCallback, awaitEmfMetrics } from "./helpers/cloudwatch"; +import { createMessageStatusPublishEvent } from "./helpers/event-factories"; +import { getClientConfig } from "./helpers/mock-client-config"; import { - buildInboundEventDlqQueueUrl, - buildInboundEventQueueUrl, - buildLambdaLogGroupName, - createCloudWatchLogsClient, - createSqsClient, - getDeploymentDetails, -} from "@nhs-notify-client-callbacks/test-support/helpers"; -import { + awaitQueueMessage, awaitQueueMessageByMessageId, - buildMockClientDlqQueueUrl, + deleteMessage, ensureInboundQueueIsEmpty, purgeQueues, sendSqsEvent, } from "./helpers/sqs"; import { - buildMockWebhookTargetPath, - getMockItClientConfig, -} from "./helpers/mock-client-config"; -import { - awaitAllEmfMetricsInLogGroup, - awaitSignedCallbacksFromWebhookLogGroup, -} from "./helpers/cloudwatch"; -import { createMessageStatusPublishEvent } from "./helpers/event-factories"; + type TestContext, + createTestContext, + destroyTestContext, +} from "./helpers/test-context"; describe("Metrics", () => { - let sqsClient: SQSClient; - let cloudWatchClient: CloudWatchLogsClient; - let callbackEventQueueUrl: string; - let clientDlqQueueUrl: string; - let inboundEventDlqQueueUrl: string; - let logGroupName: string; - let webhookLogGroupName: string; + let ctx: TestContext; + let clientDlqUrl: string; + let transformFilterLogGroup: string; beforeAll(async () => { - const deploymentDetails = getDeploymentDetails(); - const { targets } = getMockItClientConfig(); - - sqsClient = createSqsClient(deploymentDetails); - cloudWatchClient = createCloudWatchLogsClient(deploymentDetails); - callbackEventQueueUrl = buildInboundEventQueueUrl(deploymentDetails); - clientDlqQueueUrl = buildMockClientDlqQueueUrl(deploymentDetails, targets); - inboundEventDlqQueueUrl = buildInboundEventDlqQueueUrl(deploymentDetails); - logGroupName = buildLambdaLogGroupName( - deploymentDetails, - "client-transform-filter", - ); - webhookLogGroupName = buildLambdaLogGroupName( - deploymentDetails, - "mock-webhook", - ); - - await purgeQueues(sqsClient, [ - inboundEventDlqQueueUrl, - clientDlqQueueUrl, - callbackEventQueueUrl, + ctx = createTestContext(); + const { clientId } = getClientConfig("clientSingleTarget"); + + clientDlqUrl = ctx.clientDlqUrl(clientId); + transformFilterLogGroup = ctx.logGroup("client-transform-filter"); + + await purgeQueues(ctx.sqs, [ + ctx.inboundDlqUrl, + clientDlqUrl, + ctx.inboundQueueUrl, ]); }); afterAll(async () => { - await purgeQueues(sqsClient, [ - inboundEventDlqQueueUrl, - clientDlqQueueUrl, - callbackEventQueueUrl, + await purgeQueues(ctx.sqs, [ + ctx.inboundDlqUrl, + clientDlqUrl, + ctx.inboundQueueUrl, ]); - - sqsClient.destroy(); - cloudWatchClient.destroy(); + destroyTestContext(ctx); }); describe("Successful event processing", () => { @@ -79,40 +52,38 @@ describe("Metrics", () => { const startTime = Date.now(); const event = createMessageStatusPublishEvent(); - await sendSqsEvent(sqsClient, callbackEventQueueUrl, event); - await ensureInboundQueueIsEmpty(sqsClient, callbackEventQueueUrl); + await sendSqsEvent(ctx.sqs, ctx.inboundQueueUrl, event); + await ensureInboundQueueIsEmpty(ctx.sqs, ctx.inboundQueueUrl); - // Wait for signed callback log to confirm the invocation completed before checking metrics - const callbacks = await awaitSignedCallbacksFromWebhookLogGroup( - cloudWatchClient, - webhookLogGroupName, + await awaitCallback( + ctx.cwLogs, + ctx.webhookLogGroup, event.data.messageId, "MessageStatus", startTime, - buildMockWebhookTargetPath(), ); - expect(callbacks.length).toBeGreaterThan(0); - - await awaitAllEmfMetricsInLogGroup( - cloudWatchClient, - logGroupName, - [ - "EventsReceived", - "TransformationsSuccessful", - "FilteringStarted", - "FilteringMatched", - "CallbacksInitiated", - ], - startTime, - ); + await expect( + awaitEmfMetrics( + ctx.cwLogs, + transformFilterLogGroup, + [ + "EventsReceived", + "TransformationsSuccessful", + "FilteringStarted", + "FilteringMatched", + "CallbacksInitiated", + ], + startTime, + ), + ).resolves.toBeUndefined(); }, 120_000); }); describe("Validation error", () => { it("should emit ValidationErrors metric when an invalid event fails schema validation", async () => { const startTime = Date.now(); - const messageId = `invalid-schema-metrics-${Date.now()}`; + const messageId = `invalid-schema-metrics-${crypto.randomUUID()}`; const invalidEvent: StatusPublishEvent = createMessageStatusPublishEvent({ data: { @@ -122,30 +93,83 @@ describe("Metrics", () => { }, }); - await sendSqsEvent(sqsClient, callbackEventQueueUrl, invalidEvent); + await sendSqsEvent(ctx.sqs, ctx.inboundQueueUrl, invalidEvent); - // Wait for the event to land on the DLQ, confirming the Lambda ran and failed validation const dlqMessage = await awaitQueueMessageByMessageId( - sqsClient, - inboundEventDlqQueueUrl, + ctx.sqs, + ctx.inboundDlqUrl, messageId, ); expect(dlqMessage.Body).toBeDefined(); + await deleteMessage(ctx.sqs, ctx.inboundDlqUrl, dlqMessage); - await sqsClient.send( - new DeleteMessageCommand({ - QueueUrl: inboundEventDlqQueueUrl, - ReceiptHandle: dlqMessage.ReceiptHandle!, - }), + await awaitEmfMetrics( + ctx.cwLogs, + transformFilterLogGroup, + ["EventsReceived", "ValidationErrors"], + startTime, ); + }, 120_000); + }); - await awaitAllEmfMetricsInLogGroup( - cloudWatchClient, - logGroupName, - ["EventsReceived", "ValidationErrors"], + describe("HTTPS Client Lambda metrics", () => { + let httpsClientLogGroup: string; + + beforeAll(() => { + const { clientId } = getClientConfig("clientSingleTarget"); + httpsClientLogGroup = ctx.clientLogGroup(`https-client-${clientId}`); + }); + + it("should emit DeliveryAttempt, DeliverySuccess and DeliveryDurationMs on successful delivery", async () => { + const startTime = Date.now(); + const event = createMessageStatusPublishEvent(); + + await sendSqsEvent(ctx.sqs, ctx.inboundQueueUrl, event); + await ensureInboundQueueIsEmpty(ctx.sqs, ctx.inboundQueueUrl); + + await awaitCallback( + ctx.cwLogs, + ctx.webhookLogGroup, + event.data.messageId, + "MessageStatus", startTime, ); + + await expect( + awaitEmfMetrics( + ctx.cwLogs, + httpsClientLogGroup, + ["DeliveryAttempt", "DeliverySuccess", "DeliveryDurationMs"], + startTime, + ), + ).resolves.toBeUndefined(); + }, 120_000); + + it("should emit DeliveryAttempt, DeliveryPermanentFailure and DeliveryDurationMs on 4xx response", async () => { + const startTime = Date.now(); + const messageId = `force-400-metrics-${crypto.randomUUID()}`; + + const event: StatusPublishEvent = + createMessageStatusPublishEvent({ + data: { messageId }, + }); + + await sendSqsEvent(ctx.sqs, ctx.inboundQueueUrl, event); + + const dlqMessage = await awaitQueueMessage(ctx.sqs, clientDlqUrl, 90_000); + + expect(dlqMessage.Body).toBeDefined(); + await deleteMessage(ctx.sqs, clientDlqUrl, dlqMessage); + + await expect( + awaitEmfMetrics( + ctx.cwLogs, + httpsClientLogGroup, + ["DeliveryAttempt", "DeliveryPermanentFailure", "DeliveryDurationMs"], + startTime, + ), + ).resolves.toBeUndefined(); }, 120_000); }); }); diff --git a/tests/performance/README.md b/tests/performance/README.md deleted file mode 100644 index 9a44eccf..00000000 --- a/tests/performance/README.md +++ /dev/null @@ -1,32 +0,0 @@ -# performance - -Load tests for the client-callbacks service. These tests run against a real deployed AWS environment — they are not unit tests and cannot run locally without a live stack. - -## Prerequisites - -- AWS credentials configured for the target environment -- The service deployed to the target environment - -## Environment Variables - -| Variable | Required | Default | Description | -| --- | --- | --- | --- | -| `ENVIRONMENT` | Yes | — | Target environment name (e.g. `dev`) | -| `AWS_ACCOUNT_ID` | Yes | — | AWS account ID for the target environment | -| `AWS_REGION` | No | `eu-west-2` | AWS region | -| `PROJECT` | No | `nhs` | Project name prefix used in resource naming | -| `COMPONENT` | No | `callbacks` | Component name used in resource naming | - -## Running - -From the repository root: - -```bash -ENVIRONMENT=dev AWS_ACCOUNT_ID=123456789012 npm run test:performance --workspace tests/performance -``` - -## What the Tests Do - -The load test sends ~3,000 events/s to the SQS inbound queue for 30 seconds, then reads CloudWatch Logs to assert that the p95 Lambda processing time is below 500ms. - -The global teardown removes the test client subscription config from S3. diff --git a/tests/performance/fixtures/subscriptions/perf-client-1.json b/tests/performance/fixtures/subscriptions/perf-client-1.json new file mode 100644 index 00000000..005b3319 --- /dev/null +++ b/tests/performance/fixtures/subscriptions/perf-client-1.json @@ -0,0 +1,45 @@ +{ + "clientId": "perf-client-1", + "subscriptions": [ + { + "messageStatuses": [ + "DELIVERED", + "FAILED", + "SENDING", + "PENDING_ENRICHMENT", + "ENRICHED" + ], + "subscriptionId": "sub-451afe55-2c8f-4103-a5f7-7bcf79e8e476", + "subscriptionType": "MessageStatus", + "targetIds": [ + "target-39dbd795-5909-40ab-95b2-4e88b11a2813" + ] + } + ], + "targets": [ + { + "apiKey": { + "headerName": "x-api-key", + "headerValue": "REPLACED_BY_TERRAFORM" + }, + "delivery": { + "circuitBreaker": { + "enabled": true + }, + "maxRetryDurationSeconds": 7200, + "mtls": { + "certPinning": { + "enabled": true, + "spkiHash": "REPLACED_BY_TERRAFORM" + }, + "enabled": true + } + }, + "invocationEndpoint": "https://REPLACED_BY_TERRAFORM", + "invocationMethod": "POST", + "invocationRateLimit": 100, + "targetId": "target-39dbd795-5909-40ab-95b2-4e88b11a2813", + "type": "API" + } + ] +} diff --git a/tests/performance/fixtures/subscriptions/perf-client-2.json b/tests/performance/fixtures/subscriptions/perf-client-2.json new file mode 100644 index 00000000..116c5695 --- /dev/null +++ b/tests/performance/fixtures/subscriptions/perf-client-2.json @@ -0,0 +1,55 @@ +{ + "clientId": "perf-client-2", + "subscriptions": [ + { + "channelStatuses": [ + "DELIVERED", + "FAILED", + "RETRY", + "SKIPPED", + "SENDING", + "CREATED" + ], + "channelType": "NHSAPP", + "subscriptionId": "sub-ace58855-9f6b-4491-8cee-abb99d997ced", + "subscriptionType": "ChannelStatus", + "supplierStatuses": [ + "delivered", + "permanent_failure", + "temporary_failure", + "pending", + "sending", + "sent" + ], + "targetIds": [ + "target-e3ccc2c2-7b19-4475-80d5-51a1182d239a" + ] + } + ], + "targets": [ + { + "apiKey": { + "headerName": "x-api-key", + "headerValue": "REPLACED_BY_TERRAFORM" + }, + "delivery": { + "circuitBreaker": { + "enabled": true + }, + "maxRetryDurationSeconds": 7200, + "mtls": { + "certPinning": { + "enabled": true, + "spkiHash": "REPLACED_BY_TERRAFORM" + }, + "enabled": true + } + }, + "invocationEndpoint": "https://REPLACED_BY_TERRAFORM", + "invocationMethod": "POST", + "invocationRateLimit": 5000, + "targetId": "target-e3ccc2c2-7b19-4475-80d5-51a1182d239a", + "type": "API" + } + ] +} diff --git a/tests/performance/fixtures/subscriptions/perf-client-3.json b/tests/performance/fixtures/subscriptions/perf-client-3.json new file mode 100644 index 00000000..3d67c943 --- /dev/null +++ b/tests/performance/fixtures/subscriptions/perf-client-3.json @@ -0,0 +1,66 @@ +{ + "clientId": "perf-client-3", + "subscriptions": [ + { + "messageStatuses": [ + "DELIVERED" + ], + "subscriptionId": "sub-72197a52-8f4a-4b9d-b074-90f51183b91c", + "subscriptionType": "MessageStatus", + "targetIds": [ + "target-9f81befc-8cd2-49d7-9972-40b11c932d80", + "target-42228749-1610-4862-bcbf-ffb5b3a6f7eb" + ] + } + ], + "targets": [ + { + "apiKey": { + "headerName": "x-api-key", + "headerValue": "REPLACED_BY_TERRAFORM" + }, + "delivery": { + "circuitBreaker": { + "enabled": true + }, + "maxRetryDurationSeconds": 7200, + "mtls": { + "certPinning": { + "enabled": true, + "spkiHash": "REPLACED_BY_TERRAFORM" + }, + "enabled": true + } + }, + "invocationEndpoint": "https://REPLACED_BY_TERRAFORM", + "invocationMethod": "POST", + "invocationRateLimit": 300, + "targetId": "target-9f81befc-8cd2-49d7-9972-40b11c932d80", + "type": "API" + }, + { + "apiKey": { + "headerName": "x-api-key", + "headerValue": "REPLACED_BY_TERRAFORM" + }, + "delivery": { + "circuitBreaker": { + "enabled": true + }, + "maxRetryDurationSeconds": 7200, + "mtls": { + "certPinning": { + "enabled": true, + "spkiHash": "REPLACED_BY_TERRAFORM" + }, + "enabled": true + } + }, + "invocationEndpoint": "https://REPLACED_BY_TERRAFORM", + "invocationMethod": "POST", + "invocationRateLimit": 300, + "targetId": "target-42228749-1610-4862-bcbf-ffb5b3a6f7eb", + "type": "API" + } + ] +} diff --git a/tests/performance/fixtures/subscriptions/perf-client-4.json b/tests/performance/fixtures/subscriptions/perf-client-4.json new file mode 100644 index 00000000..b6c72346 --- /dev/null +++ b/tests/performance/fixtures/subscriptions/perf-client-4.json @@ -0,0 +1,57 @@ +{ + "clientId": "perf-client-4", + "subscriptions": [ + { + "messageStatuses": [ + "DELIVERED", + "FAILED" + ], + "subscriptionId": "sub-31908329-f6ce-4655-94a0-1ceb42073f13", + "subscriptionType": "MessageStatus", + "targetIds": [ + "target-11c2d19e-e8c9-4058-8175-546eabd1def2" + ] + }, + { + "channelStatuses": [ + "DELIVERED", + "FAILED" + ], + "channelType": "NHSAPP", + "subscriptionId": "sub-4f8a6b2c-d193-47e5-b860-7a9f3c1d2e4b", + "subscriptionType": "ChannelStatus", + "supplierStatuses": [ + "delivered", + "permanent_failure" + ], + "targetIds": [ + "target-11c2d19e-e8c9-4058-8175-546eabd1def2" + ] + } + ], + "targets": [ + { + "apiKey": { + "headerName": "x-api-key", + "headerValue": "REPLACED_BY_TERRAFORM" + }, + "delivery": { + "circuitBreaker": { + "enabled": true + }, + "maxRetryDurationSeconds": 7200, + "mtls": { + "certPinning": { + "enabled": false + }, + "enabled": false + } + }, + "invocationEndpoint": "https://REPLACED_BY_TERRAFORM", + "invocationMethod": "POST", + "invocationRateLimit": 300, + "targetId": "target-11c2d19e-e8c9-4058-8175-546eabd1def2", + "type": "API" + } + ] +} diff --git a/tests/performance/helpers/cloudwatch.ts b/tests/performance/helpers/cloudwatch.ts deleted file mode 100644 index 33772ba6..00000000 --- a/tests/performance/helpers/cloudwatch.ts +++ /dev/null @@ -1,156 +0,0 @@ -import { - CloudWatchLogsClient, - FilterLogEventsCommand, - GetQueryResultsCommand, - StartQueryCommand, -} from "@aws-sdk/client-cloudwatch-logs"; -import { waitUntil } from "async-wait-until"; - -const POLL_INTERVAL_MS = 2000; -const COLLECT_TIMEOUT_MS = 120_000; - -type BatchCompletedLogEntry = { - processingTimeMs: number; - batchSize: number; - successful: number; - failed: number; - filtered: number; -}; - -export async function collectBatchProcessingTimes( - client: CloudWatchLogsClient, - logGroupName: string, - expectedCount: number, - startTime: number, -): Promise { - const collected: number[] = []; - - await waitUntil( - async () => { - const response = await client.send( - new FilterLogEventsCommand({ - logGroupName, - startTime, - filterPattern: '{ $.msg = "batch-processing-completed" }', - }), - ); - - for (const event of response.events ?? []) { - if (event.message) { - try { - const entry = JSON.parse(event.message) as BatchCompletedLogEntry; - if (typeof entry.processingTimeMs === "number") { - collected.push(entry.processingTimeMs); - } - } catch { - // skip unparseable entries - } - } - } - - return collected.length >= expectedCount; - }, - { timeout: COLLECT_TIMEOUT_MS, intervalBetweenAttempts: POLL_INTERVAL_MS }, - ); - - return collected; -} - -export function computePercentile( - samples: number[], - percentile: number, -): number { - if (samples.length === 0) { - throw new Error("Cannot compute percentile of empty array"); - } - - const sorted = [...samples].toSorted((a, b) => a - b); - const index = Math.ceil((percentile / 100) * sorted.length) - 1; - return sorted[Math.max(0, index)]; -} - -const INSIGHTS_QUERY_TIMEOUT_MS = 60_000; -const INSIGHTS_COLLECT_TIMEOUT_MS = 300_000; - -async function runInsightsQuery( - client: CloudWatchLogsClient, - logGroupName: string, - startTimeSec: number, - endTimeSec: number, - percentile: number, -): Promise<{ count: number; percentileMs: number } | null> { - const { queryId } = await client.send( - new StartQueryCommand({ - logGroupName, - startTime: startTimeSec, - endTime: endTimeSec, - queryString: [ - 'filter msg = "batch-processing-completed"', - `| stats count(*) as eventCount, pct(processingTimeMs, ${percentile}) as p`, - ].join("\n"), - }), - ); - - if (!queryId) return null; - - const deadline = Date.now() + INSIGHTS_QUERY_TIMEOUT_MS; - - while (Date.now() < deadline) { - await new Promise((resolve) => { - setTimeout(resolve, 2000); - }); - - const response = await client.send(new GetQueryResultsCommand({ queryId })); - - if (response.status === "Failed" || response.status === "Cancelled") { - return null; - } - - if (response.status === "Complete") { - const row = response.results?.[0]; - if (!row) return null; - - return { - count: Number(row.find((f) => f.field === "eventCount")?.value ?? 0), - percentileMs: Number(row.find((f) => f.field === "p")?.value ?? 0), - }; - } - } - - return null; -} - -export async function waitForBatchProcessingPercentile( - client: CloudWatchLogsClient, - logGroupName: string, - testStartTime: number, - expectedCount: number, - percentile: number, -): Promise<{ count: number; percentileMs: number }> { - const startTimeSec = Math.floor(testStartTime / 1000); - let result = { count: 0, percentileMs: 0 }; - - await waitUntil( - async () => { - const endTimeSec = Math.floor((Date.now() + 60_000) / 1000); - const queryResult = await runInsightsQuery( - client, - logGroupName, - startTimeSec, - endTimeSec, - percentile, - ); - - if (!queryResult) return false; - - result = queryResult; - return result.count >= expectedCount; - }, - { - timeout: INSIGHTS_COLLECT_TIMEOUT_MS, - intervalBetweenAttempts: POLL_INTERVAL_MS, - }, - ); - - return result; -} diff --git a/tests/performance/helpers/deployment.ts b/tests/performance/helpers/deployment.ts deleted file mode 100644 index 5d6ee82e..00000000 --- a/tests/performance/helpers/deployment.ts +++ /dev/null @@ -1,10 +0,0 @@ -import { - type DeploymentDetails, - buildLambdaLogGroupName, -} from "@nhs-notify-client-callbacks/test-support/helpers/deployment"; - -export function buildTransformFilterLambdaLogGroupName( - details: DeploymentDetails, -): string { - return buildLambdaLogGroupName(details, "client-transform-filter"); -} diff --git a/tests/performance/helpers/index.ts b/tests/performance/helpers/index.ts deleted file mode 100644 index 194022a3..00000000 --- a/tests/performance/helpers/index.ts +++ /dev/null @@ -1,4 +0,0 @@ -export * from "./cloudwatch"; -export * from "./deployment"; -export * from "./event-factories"; -export * from "./sqs"; diff --git a/tests/performance/helpers/sqs.ts b/tests/performance/helpers/sqs.ts deleted file mode 100644 index e8d5b171..00000000 --- a/tests/performance/helpers/sqs.ts +++ /dev/null @@ -1,72 +0,0 @@ -import { - SQSClient, - SendMessageBatchCommand, - SendMessageCommand, -} from "@aws-sdk/client-sqs"; -import type { StatusPublishEvent } from "@nhs-notify-client-callbacks/models"; - -export async function sendSqsEvent( - client: SQSClient, - queueUrl: string, - event: StatusPublishEvent, -): Promise { - await client.send( - new SendMessageCommand({ - QueueUrl: queueUrl, - MessageBody: JSON.stringify(event), - }), - ); -} - -const SQS_MAX_BATCH_SIZE = 10; - -export async function sendSqsBatch( - client: SQSClient, - queueUrl: string, - events: StatusPublishEvent[], -): Promise { - await client.send( - new SendMessageBatchCommand({ - QueueUrl: queueUrl, - Entries: events.map((event, index) => ({ - Id: String(index), - MessageBody: JSON.stringify(event), - })), - }), - ); -} - -export async function generateSqsLoad( - client: SQSClient, - queueUrl: string, - targetEventsPerSecond: number, - durationSeconds: number, - eventFactory: () => StatusPublishEvent, -): Promise<{ sent: number; durationMs: number }> { - const batchesPerSecond = Math.ceil( - targetEventsPerSecond / SQS_MAX_BATCH_SIZE, - ); - const start = Date.now(); - let sent = 0; - - for (let second = 0; second < durationSeconds; second++) { - const waveStart = Date.now(); - - const results = await Promise.all( - Array.from({ length: batchesPerSecond }, () => { - const batch = Array.from({ length: SQS_MAX_BATCH_SIZE }, eventFactory); - return sendSqsBatch(client, queueUrl, batch).then(() => batch.length); - }), - ); - sent += results.reduce((sum, count) => sum + count, 0); - - const remaining = 1000 - (Date.now() - waveStart); - if (remaining > 0 && second < durationSeconds - 1) { - await new Promise((resolve) => { - setTimeout(resolve, remaining); - }); - } - } - - return { sent, durationMs: Date.now() - start }; -} diff --git a/tests/performance/jest.config.ts b/tests/performance/jest.config.ts deleted file mode 100644 index 06f45e6d..00000000 --- a/tests/performance/jest.config.ts +++ /dev/null @@ -1,14 +0,0 @@ -import { nodeJestConfig } from "../../jest.config.base.ts"; - -export default { - ...nodeJestConfig, - modulePaths: [""], - collectCoverage: false, - moduleNameMapper: { - "^helpers$": "/helpers/index", - }, - // Run performance tests serially to avoid queue contention - maxWorkers: 1, - // Force exit after tests complete — real AWS SDK clients keep connections alive - forceExit: true, -}; diff --git a/tests/performance/lambda-throughput.test.ts b/tests/performance/lambda-throughput.test.ts deleted file mode 100644 index 5a543ab6..00000000 --- a/tests/performance/lambda-throughput.test.ts +++ /dev/null @@ -1,76 +0,0 @@ -import { CloudWatchLogsClient } from "@aws-sdk/client-cloudwatch-logs"; -import { SQSClient } from "@aws-sdk/client-sqs"; -import { - buildInboundEventQueueUrl, - createCloudWatchLogsClient, - createSqsClient, - getDeploymentDetails, -} from "@nhs-notify-client-callbacks/test-support/helpers"; -import { - buildTransformFilterLambdaLogGroupName, - createMessageStatusPublishEvent, - generateSqsLoad, - waitForBatchProcessingPercentile, -} from "helpers"; - -const TARGET_EPS = 3000; -const LOAD_DURATION_SECONDS = 30; -const P95_LATENCY_THRESHOLD_MS = 500; - -describe("Lambda throughput and latency under load", () => { - let sqsClient: SQSClient; - let cloudWatchClient: CloudWatchLogsClient; - let inboundQueueUrl: string; - let lambdaLogGroupName: string; - - beforeAll(() => { - const deploymentDetails = getDeploymentDetails(); - - sqsClient = createSqsClient(deploymentDetails); - cloudWatchClient = createCloudWatchLogsClient(deploymentDetails); - inboundQueueUrl = buildInboundEventQueueUrl(deploymentDetails); - lambdaLogGroupName = - buildTransformFilterLambdaLogGroupName(deploymentDetails); - }); - - afterAll(() => { - sqsClient.destroy(); - cloudWatchClient.destroy(); - }); - - it(`should sustain ~${TARGET_EPS} events/s for ${LOAD_DURATION_SECONDS}s with p95 Lambda processing time below ${P95_LATENCY_THRESHOLD_MS}ms`, async () => { - const testStartTime = Date.now(); - - const { durationMs, sent } = await generateSqsLoad( - sqsClient, - inboundQueueUrl, - TARGET_EPS, - LOAD_DURATION_SECONDS, - createMessageStatusPublishEvent, - ); - - const achievedEps = Math.round(sent / (durationMs / 1000)); - console.log( - `Load generation: ${sent} events in ${durationMs}ms (${achievedEps} eps achieved)`, - ); - - // Accept ≥90% of sent events processed — accounts for any events routed to DLQ - // due to transient Lambda errors under concurrency pressure. - const minExpectedCount = Math.floor(sent * 0.9); - - const { count, percentileMs } = await waitForBatchProcessingPercentile( - cloudWatchClient, - lambdaLogGroupName, - testStartTime, - minExpectedCount, - 95, - ); - - console.log( - `Processing: ${count} events logged, p95 Lambda processing time: ${percentileMs}ms`, - ); - - expect(count).toBeGreaterThanOrEqual(minExpectedCount); - expect(percentileMs).toBeLessThan(P95_LATENCY_THRESHOLD_MS); - }, 600_000); -}); diff --git a/tests/test-support/helpers/deployment.ts b/tests/test-support/helpers/deployment.ts index 20bf1f59..dacb3c2d 100644 --- a/tests/test-support/helpers/deployment.ts +++ b/tests/test-support/helpers/deployment.ts @@ -3,6 +3,7 @@ export type DeploymentDetails = { environment: string; project: string; component: string; + clientComponent: string; accountId: string; }; @@ -10,7 +11,8 @@ export function getDeploymentDetails(): DeploymentDetails { const region = process.env.AWS_REGION ?? "eu-west-2"; const environment = process.env.ENVIRONMENT; const project = process.env.PROJECT ?? "nhs"; - const component = process.env.COMPONENT ?? "callbacks"; + const component = process.env.COMPONENT ?? "cb"; + const clientComponent = process.env.CLIENT_COMPONENT ?? "cbc"; const accountId = process.env.AWS_ACCOUNT_ID; if (!environment) { @@ -21,17 +23,14 @@ export function getDeploymentDetails(): DeploymentDetails { throw new Error("AWS_ACCOUNT_ID environment variable must be set"); } - return { region, environment, project, component, accountId }; -} - -export function buildSubscriptionConfigBucketName({ - accountId, - component, - environment, - project, - region, -}: DeploymentDetails): string { - return `${project}-${accountId}-${region}-${environment}-${component}-subscription-config`; + return { + region, + environment, + project, + component, + clientComponent, + accountId, + }; } export function buildLambdaLogGroupName( diff --git a/tools/client-subscriptions-management/README.md b/tools/client-subscriptions-management/README.md index 7593bb07..b735c301 100644 --- a/tools/client-subscriptions-management/README.md +++ b/tools/client-subscriptions-management/README.md @@ -1,109 +1,129 @@ # client-subscriptions-management -TypeScript CLI utility for managing NHS Notify client subscription configuration in S3. +TypeScript CLI for managing NHS Notify client callback configuration in S3. ## Usage -From the repository root run: - ```bash -npm --workspace tools/client-subscriptions-management run -- [options] +pnpm --filter client-subscriptions-management run -- [options] ``` -## Example +### Common options -Deploy a message status subscription to the `dev` environment using a named AWS profile: +All commands accept: -```bash -npm --workspace tools/client-subscriptions-management run deploy -- message \ - --environment dev \ - --profile my-profile \ - --client-id my-client \ - --message-statuses DELIVERED FAILED \ - --api-endpoint https://webhook.example.invalid/callbacks \ - --api-key 1234.4321 \ - --rate-limit 20 \ - --dry-run false -``` +| Option | Default | Description | +| --------------- | ------------- | --------------------------------------------- | +| `--environment` | — | Environment name (used to derive bucket name) | +| `--bucket-name` | derived | Override the S3 bucket name | +| `--region` | `eu-west-2` | AWS region | +| `--profile` | `AWS_PROFILE` | AWS credentials profile | -## Commands +Write commands (`clients-put`, `subscriptions-add`, `subscriptions-del`, `subscriptions-set-states`, `targets-add`, `targets-del`, `applications-map-add`) also accept `--dry-run` (default `false`). -### Deploy a Subscription +## Commands -#### Message status +### Clients ```bash -npm --workspace tools/client-subscriptions-management run deploy -- message \ - --environment dev \ +# List all client IDs +pnpm --filter client-subscriptions-management run clients-list -- --environment dev + +# Get a client's full configuration +pnpm --filter client-subscriptions-management run clients-get -- --environment dev --client-id client-123 + +# Write a full client configuration from JSON +pnpm --filter client-subscriptions-management run clients-put -- --environment dev \ --client-id client-123 \ - --message-statuses DELIVERED FAILED \ - --api-endpoint https://webhook.example.invalid \ - --api-key-header-name x-api-key \ - --api-key 1234.4321 \ - --dry-run false \ - --rate-limit 20 + --file config.json ``` -#### Channel status +`clients-put` accepts `--json ` or `--file ` (mutually exclusive, one required). + +### Targets ```bash -npm --workspace tools/client-subscriptions-management run deploy -- channel \ - --environment dev \ +# List a client's callback targets +pnpm --filter client-subscriptions-management run targets-list -- --environment dev --client-id client-123 + +# Add a callback target +pnpm --filter client-subscriptions-management run targets-add -- --environment dev \ --client-id client-123 \ - --channel-type EMAIL \ - --channel-statuses DELIVERED FAILED \ - --supplier-statuses READ REJECTED \ --api-endpoint https://webhook.example.invalid \ - --api-key-header-name x-api-key \ - --api-key 1234.4321 \ - --dry-run false \ + --api-key secret-key \ --rate-limit 20 + +# Delete a callback target +pnpm --filter client-subscriptions-management run targets-del -- --environment dev \ + --client-id client-123 \ + --target-id target-abc ``` -Optional for both: `--client-name "Test Client"` (defaults to client-id if not provided), `--project ` (defaults to `nhs`), `--region ` (defaults to `eu-west-2`), `--profile `, `--bucket-name ` (override derived bucket name) +`targets-add` options: -**Note (channel)**: At least one of `--channel-statuses` or `--supplier-statuses` must be provided. +| Option | Required | Default | +| ----------------------- | -------- | ----------- | +| `--api-endpoint` | yes | — | +| `--api-key` | yes | — | +| `--api-key-header-name` | no | `x-api-key` | +| `--rate-limit` | yes | — | -### Get Client Subscriptions By Client ID +### Subscriptions ```bash -npm --workspace tools/client-subscriptions-management run get-by-client-id -- \ - --environment dev \ - --client-id client-123 -``` +# List a client's subscriptions +pnpm --filter client-subscriptions-management run subscriptions-list -- --environment dev --client-id client-123 -### Put Message Status Subscription (S3 upload only) +# Add a message status subscription +pnpm --filter client-subscriptions-management run subscriptions-add -- --environment dev \ + --client-id client-123 \ + --subscription-type MessageStatus \ + --target-id target-abc \ + --message-statuses DELIVERED FAILED -```bash -npm --workspace tools/client-subscriptions-management run put-message-status -- \ - --environment dev \ +# Add a channel status subscription +pnpm --filter client-subscriptions-management run subscriptions-add -- --environment dev \ --client-id client-123 \ - --message-statuses DELIVERED FAILED \ - --api-endpoint https://webhook.example.invalid \ - --api-key-header-name x-api-key \ - --api-key 1234.4321 \ - --dry-run false \ - --rate-limit 20 + --subscription-type ChannelStatus \ + --target-id target-abc \ + --channel-type EMAIL \ + --channel-statuses DELIVERED FAILED \ + --supplier-statuses read rejected + +# Update statuses on an existing subscription +pnpm --filter client-subscriptions-management run subscriptions-set-states -- --environment dev \ + --client-id client-123 \ + --subscription-id sub-456 \ + --message-statuses DELIVERED FAILED SENDING + +# Delete a subscription +pnpm --filter client-subscriptions-management run subscriptions-del -- --environment dev \ + --client-id client-123 \ + --subscription-id sub-456 ``` -Optional: `--client-name "Test Client"` (defaults to client-id if not provided), `--profile `, `--bucket-name ` +`subscriptions-add` options: + +| Option | Required | Notes | +| --------------------- | ----------- | --------------------------------------------------------------------------------- | +| `--subscription-type` | yes | `MessageStatus` or `ChannelStatus` | +| `--target-id` | yes | One or more target IDs | +| `--message-statuses` | conditional | Required for `MessageStatus` | +| `--channel-type` | conditional | Required for `ChannelStatus` (`NHSAPP`, `EMAIL`, `SMS`, `LETTER`) | +| `--channel-statuses` | conditional | At least one of `--channel-statuses` or `--supplier-statuses` for `ChannelStatus` | +| `--supplier-statuses` | conditional | See above | +| `--subscription-id` | no | Auto-generated UUID if omitted | -### Put Channel Status Subscription (S3 upload only) +### Applications Map ```bash -npm --workspace tools/client-subscriptions-management run put-channel-status -- \ - --environment dev \ +# Get the application ID for a client +pnpm --filter client-subscriptions-management run applications-map-get -- --environment dev --client-id client-123 + +# Add/update a client-to-application-ID mapping +pnpm --filter client-subscriptions-management run applications-map-add -- --environment dev \ --client-id client-123 \ - --channel-type EMAIL \ - --channel-statuses DELIVERED FAILED \ - --supplier-statuses READ REJECTED \ - --api-endpoint https://webhook.example.invalid \ - --api-key-header-name x-api-key \ - --api-key 1234.4321 \ - --dry-run false \ - --rate-limit 20 + --application-id app-789 ``` -Optional: `--client-name "Test Client"` (defaults to client-id if not provided), `--profile `, `--bucket-name ` - -**Note**: At least one of `--channel-statuses` or `--supplier-statuses` must be provided. +Both accept optional `--applications-map-bucket` and `--applications-map-key` to override the default S3 location. diff --git a/tools/client-subscriptions-management/package.json b/tools/client-subscriptions-management/package.json index ec4cb3b3..c2e3ebc4 100644 --- a/tools/client-subscriptions-management/package.json +++ b/tools/client-subscriptions-management/package.json @@ -25,15 +25,16 @@ }, "dependencies": { "@aws-sdk/client-s3": "catalog:aws", - "@aws-sdk/client-ssm": "catalog:aws", "@aws-sdk/client-sts": "catalog:aws", "@aws-sdk/credential-providers": "catalog:aws", "@nhs-notify-client-callbacks/models": "workspace:*", + "picocolors": "catalog:app", "table": "catalog:app", "yargs": "catalog:app", "zod": "catalog:app" }, "devDependencies": { + "@smithy/types": "catalog:aws", "@types/jest": "catalog:test", "@types/node": "catalog:tools", "@types/yargs": "catalog:tools", diff --git a/tools/client-subscriptions-management/src/__tests__/aws.test.ts b/tools/client-subscriptions-management/src/__tests__/aws.test.ts index f08d0bda..528acc31 100644 --- a/tools/client-subscriptions-management/src/__tests__/aws.test.ts +++ b/tools/client-subscriptions-management/src/__tests__/aws.test.ts @@ -1,8 +1,10 @@ import { + deriveApplicationsMapBucketName, + deriveApplicationsMapKey, deriveBucketName, - deriveParameterName, + resolveApplicationsMapLocation, resolveBucketName, - resolveParameterName, + resolveDeploymentContext, resolveProfile, resolveRegion, } from "src/aws"; @@ -24,20 +26,18 @@ describe("aws", () => { it("derives bucket name from environment using STS account ID", async () => { await expect( resolveBucketName({ environment: "dev", region: "eu-west-2" }), - ).resolves.toBe( - "nhs-123456789012-eu-west-2-dev-callbacks-subscription-config", - ); + ).resolves.toBe("nhs-123456789012-eu-west-2-main-acct-clie-client-configs"); }); it("uses default region eu-west-2 when region is not provided", async () => { await expect(resolveBucketName({ environment: "dev" })).resolves.toBe( - "nhs-123456789012-eu-west-2-dev-callbacks-subscription-config", + "nhs-123456789012-eu-west-2-main-acct-clie-client-configs", ); }); it("derives bucket name correctly", () => { expect(deriveBucketName("123456789012", "dev", "eu-west-2")).toBe( - "nhs-123456789012-eu-west-2-dev-callbacks-subscription-config", + "nhs-123456789012-eu-west-2-main-acct-clie-client-configs", ); }); @@ -81,37 +81,85 @@ describe("aws", () => { expect(resolveRegion(undefined, {} as NodeJS.ProcessEnv)).toBeUndefined(); }); - it("derives parameter name from environment", () => { - expect(deriveParameterName("dev")).toBe( - "/nhs/dev/callbacks/applications-map", + it("derives applications map bucket name", () => { + expect(deriveApplicationsMapBucketName("123456789012", "eu-west-2")).toBe( + "nhs-123456789012-eu-west-2-main-acct-clie-apps-map", ); }); - it("resolves parameter name from explicit argument", () => { - expect(resolveParameterName({ parameterName: "/custom/path" })).toBe( - "/custom/path", - ); + it("derives applications map key from environment", () => { + expect(deriveApplicationsMapKey("dev")).toBe("dev/applications-map.json"); }); - it("derives parameter name from environment argument", () => { - expect(resolveParameterName({ environment: "dev" })).toBe( - "/nhs/dev/callbacks/applications-map", - ); + it("resolves applications map location from explicit arguments", async () => { + await expect( + resolveApplicationsMapLocation({ + bucketName: "my-bucket", + key: "my-key.json", + }), + ).resolves.toEqual({ bucket: "my-bucket", key: "my-key.json" }); }); - it("derives parameter name from ENVIRONMENT env var", () => { - expect( - resolveParameterName({ - env: { ENVIRONMENT: "staging" } as NodeJS.ProcessEnv, + it("derives applications map location from environment", async () => { + await expect( + resolveApplicationsMapLocation({ + environment: "dev", + region: "eu-west-2", }), - ).toBe("/nhs/staging/callbacks/applications-map"); + ).resolves.toEqual({ + bucket: "nhs-123456789012-eu-west-2-main-acct-clie-apps-map", + key: "dev/applications-map.json", + }); }); - it("throws when no parameter name can be resolved", () => { - expect(() => - resolveParameterName({ env: {} as NodeJS.ProcessEnv }), - ).toThrow( - "Environment is required to derive parameter name. Please provide via --environment or ENVIRONMENT env var.", + it("throws when no environment for applications map location", async () => { + await expect( + resolveApplicationsMapLocation({ + env: {} as NodeJS.ProcessEnv, + } as Parameters[0]), + ).rejects.toThrow("Environment is required"); + }); + + it("resolves deployment context with all fields", async () => { + const ctx = await resolveDeploymentContext({ + environment: "dev", + region: "eu-west-2", + profile: "my-profile", + }); + + expect(ctx).toEqual({ + environment: "dev", + region: "eu-west-2", + accountId: "123456789012", + profile: "my-profile", + }); + }); + + it("resolves deployment context with defaults", async () => { + const savedProfile = process.env.AWS_PROFILE; + delete process.env.AWS_PROFILE; + + try { + const ctx = await resolveDeploymentContext({ + environment: "staging", + }); + + expect(ctx).toEqual({ + environment: "staging", + region: "eu-west-2", + accountId: "123456789012", + profile: undefined, + }); + } finally { + if (savedProfile !== undefined) { + process.env.AWS_PROFILE = savedProfile; + } + } + }); + + it("throws when environment is missing from deployment context", async () => { + await expect(resolveDeploymentContext({})).rejects.toThrow( + "Environment is required", ); }); }); diff --git a/tools/client-subscriptions-management/src/__tests__/container.test.ts b/tools/client-subscriptions-management/src/__tests__/container.test.ts index 1838175f..7066307e 100644 --- a/tools/client-subscriptions-management/src/__tests__/container.test.ts +++ b/tools/client-subscriptions-management/src/__tests__/container.test.ts @@ -20,12 +20,14 @@ describe("createRepository", () => { const result = createRepository({ bucketName: "bucket-1", + environment: "dev", region: "eu-west-2", }); expect(mockS3Repository).toHaveBeenCalledWith( "bucket-1", expect.any(S3Client), + "dev/", ); expect(mockRepository).toHaveBeenCalledWith( mockS3Repository.mock.instances[0], diff --git a/tools/client-subscriptions-management/src/__tests__/domain/client-subscription-builder.test.ts b/tools/client-subscriptions-management/src/__tests__/domain/client-subscription-builder.test.ts index 10fcb111..edc4b857 100644 --- a/tools/client-subscriptions-management/src/__tests__/domain/client-subscription-builder.test.ts +++ b/tools/client-subscriptions-management/src/__tests__/domain/client-subscription-builder.test.ts @@ -8,6 +8,16 @@ const UUID_REGEX = /^[\da-f]{8}-[\da-f]{4}-4[\da-f]{3}-[89ab][\da-f]{3}-[\da-f]{12}$/i; describe("buildTarget", () => { + let warnSpy: jest.SpyInstance; + + beforeEach(() => { + warnSpy = jest.spyOn(console, "warn").mockImplementation(); + }); + + afterEach(() => { + warnSpy.mockRestore(); + }); + it("builds a target with required fields", () => { const result = buildTarget({ apiEndpoint: "https://example.com/webhook", @@ -22,6 +32,9 @@ describe("buildTarget", () => { invocationMethod: "POST", invocationRateLimit: 10, apiKey: { headerName: "x-api-key", headerValue: "secret" }, + delivery: { + mtls: { enabled: false, certPinning: { enabled: false } }, + }, }); expect(result.targetId).toMatch(UUID_REGEX); }); @@ -35,8 +48,122 @@ describe("buildTarget", () => { expect(result.apiKey.headerName).toBe("x-api-key"); }); -}); + it("emits warning when mtls is disabled", () => { + buildTarget({ + apiEndpoint: "https://example.com/webhook", + apiKey: "secret", + rateLimit: 10, + mtls: { enabled: false }, + }); + + expect(warnSpy).toHaveBeenCalledWith( + expect.stringContaining("mTLS is disabled"), + ); + }); + + it("emits warning when mtls enabled but certPinning disabled", () => { + buildTarget({ + apiEndpoint: "https://example.com/webhook", + apiKey: "secret", + rateLimit: 10, + mtls: { enabled: true }, + certPinning: { enabled: false }, + }); + + expect(warnSpy).toHaveBeenCalledWith( + expect.stringContaining("certificate pinning is disabled"), + ); + }); + + it("throws when certPinning enabled without spkiHash", () => { + expect(() => + buildTarget({ + apiEndpoint: "https://example.com/webhook", + apiKey: "secret", + rateLimit: 10, + mtls: { enabled: true }, + certPinning: { enabled: true }, + }), + ).toThrow("Certificate pinning cannot be enabled without an SPKI hash"); + }); + + it("emits warning when certPinning enabled but mtls disabled", () => { + buildTarget({ + apiEndpoint: "https://example.com/webhook", + apiKey: "secret", + rateLimit: 10, + mtls: { enabled: false }, + certPinning: { + enabled: true, + spkiHash: "dGVzdGhhc2g9PT09PT09PT09PT09PT09PT09PT09PQ==", + }, + }); + + expect(warnSpy).toHaveBeenCalledWith( + expect.stringContaining("mTLS is disabled"), + ); + }); + + it("emits no warnings for fully secure config", () => { + buildTarget({ + apiEndpoint: "https://example.com/webhook", + apiKey: "secret", + rateLimit: 10, + mtls: { enabled: true }, + certPinning: { enabled: true, spkiHash: "abc123" }, + }); + + expect(warnSpy).not.toHaveBeenCalled(); + }); + + it("emits warning when maxRetryDurationSeconds is below 60", () => { + buildTarget({ + apiEndpoint: "https://example.com/webhook", + apiKey: "secret", + rateLimit: 10, + maxRetryDurationSeconds: 30, + }); + + expect(warnSpy).toHaveBeenCalledWith( + expect.stringContaining("maxRetryDurationSeconds is 30s"), + ); + }); + + it("does not emit warning when maxRetryDurationSeconds is 60 or above", () => { + buildTarget({ + apiEndpoint: "https://example.com/webhook", + apiKey: "secret", + rateLimit: 10, + maxRetryDurationSeconds: 60, + mtls: { enabled: true }, + certPinning: { enabled: true, spkiHash: "abc123" }, + }); + + expect(warnSpy).not.toHaveBeenCalled(); + }); + + it("includes maxRetryDurationSeconds in delivery when provided", () => { + const result = buildTarget({ + apiEndpoint: "https://example.com/webhook", + apiKey: "secret", + rateLimit: 10, + maxRetryDurationSeconds: 3600, + }); + + expect(result.delivery?.maxRetryDurationSeconds).toBe(3600); + }); + + it("omits maxRetryDurationSeconds from delivery when not provided", () => { + const result = buildTarget({ + apiEndpoint: "https://example.com/webhook", + apiKey: "secret", + rateLimit: 10, + }); + + expect(result.delivery).not.toHaveProperty("maxRetryDurationSeconds"); + }); +}); describe("buildMessageStatusSubscription", () => { it("builds message status subscription", () => { const result = buildMessageStatusSubscription({ diff --git a/tools/client-subscriptions-management/src/__tests__/entrypoint/cli/applications-map-add.test.ts b/tools/client-subscriptions-management/src/__tests__/entrypoint/cli/applications-map-add.test.ts index 99b08ca9..27a6a28f 100644 --- a/tools/client-subscriptions-management/src/__tests__/entrypoint/cli/applications-map-add.test.ts +++ b/tools/client-subscriptions-management/src/__tests__/entrypoint/cli/applications-map-add.test.ts @@ -12,7 +12,7 @@ const mockFormatApplicationsMap = jest.fn(); jest.mock("src/entrypoint/cli/helper", () => ({ ...jest.requireActual("src/entrypoint/cli/helper"), - createSsmApplicationsMapRepository: jest.fn(), + createS3ApplicationsMapRepository: jest.fn(), })); jest.mock("src/format", () => ({ @@ -21,8 +21,8 @@ jest.mock("src/format", () => ({ mockFormatApplicationsMap(...args), })); -const mockCreateSsmApplicationsMapRepository = - helper.createSsmApplicationsMapRepository as jest.Mock; +const mockCreateS3ApplicationsMapRepository = + helper.createS3ApplicationsMapRepository as jest.Mock; describe("applications-map-add CLI", () => { const originalCliConsoleState = captureCliConsoleState(); @@ -34,8 +34,10 @@ describe("applications-map-add CLI", () => { "client-1", "--application-id", "app-1", - "--parameter-name", - "/nhs/dev/callbacks/applications-map", + "--applications-map-bucket", + "test-bucket", + "--applications-map-key", + "dev/applications-map.json", ]; const resultMap = new Map([["client-1", "app-1"]]); @@ -45,8 +47,8 @@ describe("applications-map-add CLI", () => { mockAddApplication.mockResolvedValue(resultMap); mockFormatApplicationsMap.mockReset(); mockFormatApplicationsMap.mockReturnValue("masked-map-output"); - mockCreateSsmApplicationsMapRepository.mockReset(); - mockCreateSsmApplicationsMapRepository.mockReturnValue({ + mockCreateS3ApplicationsMapRepository.mockReset(); + mockCreateS3ApplicationsMapRepository.mockResolvedValue({ addApplication: mockAddApplication, }); resetCliConsoleState(); @@ -59,11 +61,12 @@ describe("applications-map-add CLI", () => { it("adds application and logs output", async () => { await cli.main(baseArgs); - expect(mockCreateSsmApplicationsMapRepository).toHaveBeenCalledWith( + expect(mockCreateS3ApplicationsMapRepository).toHaveBeenCalledWith( expect.objectContaining({ "client-id": "client-1", "application-id": "app-1", - "parameter-name": "/nhs/dev/callbacks/applications-map", + "applications-map-bucket": "test-bucket", + "applications-map-key": "dev/applications-map.json", }), ); expect(mockAddApplication).toHaveBeenCalledWith("client-1", "app-1", false); @@ -85,7 +88,7 @@ describe("applications-map-add CLI", () => { await cli.main(baseArgs); expect(console.log).not.toHaveBeenCalledWith( - "Dry run — no changes written to SSM.", + "Dry run — no changes written.", ); }); @@ -93,14 +96,12 @@ describe("applications-map-add CLI", () => { await cli.main([...baseArgs, "--dry-run"]); expect(mockAddApplication).toHaveBeenCalledWith("client-1", "app-1", true); - expect(console.log).toHaveBeenCalledWith( - "Dry run — no changes written to SSM.", - ); + expect(console.log).toHaveBeenCalledWith("Dry run — no changes written."); }); it("handles errors in wrapped CLI", async () => { expect.hasAssertions(); - mockCreateSsmApplicationsMapRepository.mockReturnValue({ + mockCreateS3ApplicationsMapRepository.mockResolvedValue({ addApplication: jest.fn().mockRejectedValue(new Error("Boom")), }); diff --git a/tools/client-subscriptions-management/src/__tests__/entrypoint/cli/applications-map-get.test.ts b/tools/client-subscriptions-management/src/__tests__/entrypoint/cli/applications-map-get.test.ts index 3ddb8041..c1f62cb3 100644 --- a/tools/client-subscriptions-management/src/__tests__/entrypoint/cli/applications-map-get.test.ts +++ b/tools/client-subscriptions-management/src/__tests__/entrypoint/cli/applications-map-get.test.ts @@ -11,11 +11,11 @@ const mockGetApplication = jest.fn(); jest.mock("src/entrypoint/cli/helper", () => ({ ...jest.requireActual("src/entrypoint/cli/helper"), - createSsmApplicationsMapRepository: jest.fn(), + createS3ApplicationsMapRepository: jest.fn(), })); -const mockCreateSsmApplicationsMapRepository = - helper.createSsmApplicationsMapRepository as jest.Mock; +const mockCreateS3ApplicationsMapRepository = + helper.createS3ApplicationsMapRepository as jest.Mock; describe("applications-map-get CLI", () => { const originalCliConsoleState = captureCliConsoleState(); @@ -25,14 +25,16 @@ describe("applications-map-get CLI", () => { "script", "--client-id", "client-1", - "--parameter-name", - "/nhs/dev/callbacks/applications-map", + "--applications-map-bucket", + "test-bucket", + "--applications-map-key", + "dev/applications-map.json", ]; beforeEach(() => { mockGetApplication.mockReset(); - mockCreateSsmApplicationsMapRepository.mockReset(); - mockCreateSsmApplicationsMapRepository.mockReturnValue({ + mockCreateS3ApplicationsMapRepository.mockReset(); + mockCreateS3ApplicationsMapRepository.mockResolvedValue({ getApplication: mockGetApplication, }); resetCliConsoleState(); @@ -47,10 +49,11 @@ describe("applications-map-get CLI", () => { await cli.main(baseArgs); - expect(mockCreateSsmApplicationsMapRepository).toHaveBeenCalledWith( + expect(mockCreateS3ApplicationsMapRepository).toHaveBeenCalledWith( expect.objectContaining({ "client-id": "client-1", - "parameter-name": "/nhs/dev/callbacks/applications-map", + "applications-map-bucket": "test-bucket", + "applications-map-key": "dev/applications-map.json", }), ); expect(mockGetApplication).toHaveBeenCalledWith("client-1"); diff --git a/tools/client-subscriptions-management/src/__tests__/entrypoint/cli/clients-put.test.ts b/tools/client-subscriptions-management/src/__tests__/entrypoint/cli/clients-put.test.ts index 39b800ab..4e0eb8a8 100644 --- a/tools/client-subscriptions-management/src/__tests__/entrypoint/cli/clients-put.test.ts +++ b/tools/client-subscriptions-management/src/__tests__/entrypoint/cli/clients-put.test.ts @@ -243,10 +243,15 @@ describe("clients-put CLI", () => { validConfig, true, ); - expect(console.log).toHaveBeenCalledWith("Dry run: config is valid"); + expect(console.log).toHaveBeenCalledWith( + "Dry run \u2014 no changes written.", + ); expect(console.log).toHaveBeenCalledWith( JSON.stringify(validConfig, null, 2), ); + expect(console.log).not.toHaveBeenCalledWith( + `Config written for client: client-1`, + ); }); it("handles errors in wrapped CLI", async () => { diff --git a/tools/client-subscriptions-management/src/__tests__/entrypoint/cli/helper.test.ts b/tools/client-subscriptions-management/src/__tests__/entrypoint/cli/helper.test.ts index fc33aff7..0f60ad34 100644 --- a/tools/client-subscriptions-management/src/__tests__/entrypoint/cli/helper.test.ts +++ b/tools/client-subscriptions-management/src/__tests__/entrypoint/cli/helper.test.ts @@ -1,13 +1,11 @@ const mockCreateRepositoryFromOptions = jest.fn(); const mockResolveBucketName = jest.fn(); -const mockResolveProfile = jest.fn(); -const mockResolveRegion = jest.fn(); +const mockResolveDeploymentContext = jest.fn(); jest.mock("src/aws", () => ({ createRepository: mockCreateRepositoryFromOptions, resolveBucketName: mockResolveBucketName, - resolveProfile: mockResolveProfile, - resolveRegion: mockResolveRegion, + resolveDeploymentContext: mockResolveDeploymentContext, })); import { @@ -17,10 +15,14 @@ import { } from "src/entrypoint/cli/helper"; describe("createRepository", () => { - it("resolves region, profile and bucket then delegates to createRepository from aws", async () => { + it("resolves deployment context and bucket then delegates to createRepository from aws", async () => { const fakeRepo = { listClientIds: jest.fn() }; - mockResolveRegion.mockReturnValue("eu-west-2"); - mockResolveProfile.mockReturnValue("my-profile"); + mockResolveDeploymentContext.mockResolvedValue({ + environment: "my-env", + region: "eu-west-2", + accountId: "123456789012", + profile: "my-profile", + }); mockResolveBucketName.mockResolvedValue("my-bucket"); mockCreateRepositoryFromOptions.mockReturnValue(fakeRepo); @@ -31,8 +33,11 @@ describe("createRepository", () => { environment: "my-env", }); - expect(mockResolveRegion).toHaveBeenCalledWith("eu-west-2"); - expect(mockResolveProfile).toHaveBeenCalledWith("my-profile"); + expect(mockResolveDeploymentContext).toHaveBeenCalledWith({ + environment: "my-env", + region: "eu-west-2", + profile: "my-profile", + }); expect(mockResolveBucketName).toHaveBeenCalledWith({ bucketName: "my-bucket", environment: "my-env", @@ -41,11 +46,25 @@ describe("createRepository", () => { }); expect(mockCreateRepositoryFromOptions).toHaveBeenCalledWith({ bucketName: "my-bucket", + environment: "my-env", region: "eu-west-2", profile: "my-profile", }); expect(result).toBe(fakeRepo); }); + + it("throws when environment is not provided", async () => { + mockResolveDeploymentContext.mockRejectedValue( + new Error("Environment is required"), + ); + + await expect( + createRepository({ + "bucket-name": "my-bucket", + region: "eu-west-2", + } as Parameters[0]), + ).rejects.toThrow("Environment is required"); + }); }); describe("runCommands", () => { diff --git a/tools/client-subscriptions-management/src/__tests__/entrypoint/cli/targets-set-certificate.test.ts b/tools/client-subscriptions-management/src/__tests__/entrypoint/cli/targets-set-certificate.test.ts new file mode 100644 index 00000000..a902ed0c --- /dev/null +++ b/tools/client-subscriptions-management/src/__tests__/entrypoint/cli/targets-set-certificate.test.ts @@ -0,0 +1,131 @@ +import path from "node:path"; +import { mkdtempSync, unlinkSync, writeFileSync } from "node:fs"; +import os from "node:os"; +import * as cli from "src/entrypoint/cli/targets-set-certificate"; +import { + captureCliConsoleState, + expectWrappedCliError, + getMockCreateRepository, + resetCliConsoleState, + resetMockCreateRepository, + restoreCliConsoleState, +} from "src/__tests__/entrypoint/cli/test-utils"; +import { + createClientSubscriptionConfig, + createTarget, +} from "src/__tests__/helpers/client-subscription-fixtures"; + +const mockGetClientConfig = jest.fn(); +const mockPutClientConfig = jest.fn(); +const mockFormatClientConfig = jest.fn(); + +jest.mock("src/entrypoint/cli/helper", () => ({ + ...jest.requireActual("src/entrypoint/cli/helper"), + createRepository: jest.fn(), +})); +jest.mock("src/format", () => ({ + formatClientConfig: (...args: unknown[]) => mockFormatClientConfig(...args), +})); + +const FIXTURE_CERT_PATH = path.join(__dirname, "../../fixtures/test-cert.pem"); +const EXPECTED_SPKI_HASH = "SpGTft7LNMxLIx5s9GMAaHTo1uz4eqMtrAFws3Exs8I="; + +const target = createTarget(); +const config = createClientSubscriptionConfig({ targets: [target] }); +const mockCreateRepository = getMockCreateRepository(); + +describe("targets-set-certificate CLI", () => { + const originalCliConsoleState = captureCliConsoleState(); + + const baseArgs = [ + "node", + "script", + "--client-id", + "client-1", + "--bucket-name", + "bucket-1", + "--target-id", + target.targetId, + ]; + + beforeEach(() => { + mockGetClientConfig.mockReset(); + mockGetClientConfig.mockResolvedValue(config); + mockPutClientConfig.mockReset(); + mockPutClientConfig.mockResolvedValue(config); + mockFormatClientConfig.mockReset(); + mockFormatClientConfig.mockReturnValue("formatted-output"); + resetMockCreateRepository({ + getClientConfig: mockGetClientConfig, + putClientConfig: mockPutClientConfig, + }); + resetCliConsoleState(); + }); + + afterAll(() => { + restoreCliConsoleState(originalCliConsoleState); + }); + + it("extracts SPKI hash from valid PEM and stores it", async () => { + await cli.main([...baseArgs, "--pem-file", FIXTURE_CERT_PATH]); + + expect(mockPutClientConfig).toHaveBeenCalledWith( + "client-1", + expect.objectContaining({ + targets: [ + expect.objectContaining({ + delivery: expect.objectContaining({ + mtls: expect.objectContaining({ + certPinning: expect.objectContaining({ + spkiHash: EXPECTED_SPKI_HASH, + }), + }), + }), + }), + ], + }), + false, + ); + }); + + it("errors for invalid PEM file", async () => { + const tmpDir = mkdtempSync(path.join(os.tmpdir(), "cert-test-")); + const invalidPath = path.join(tmpDir, "invalid.pem"); + // eslint-disable-next-line security/detect-non-literal-fs-filename + writeFileSync(invalidPath, "not-a-pem"); + + await cli.main([...baseArgs, "--pem-file", invalidPath]).catch(() => {}); + + expect(mockPutClientConfig).not.toHaveBeenCalled(); + + // eslint-disable-next-line security/detect-non-literal-fs-filename + unlinkSync(invalidPath); + }); + + it("passes dry-run to putClientConfig", async () => { + await cli.main([ + ...baseArgs, + "--pem-file", + FIXTURE_CERT_PATH, + "--dry-run", + "true", + ]); + + expect(mockPutClientConfig).toHaveBeenCalledWith( + "client-1", + expect.any(Object), + true, + ); + }); + + it("handles repository errors in wrapped CLI", async () => { + expect.hasAssertions(); + mockCreateRepository.mockRejectedValue(new Error("Boom")); + + await expectWrappedCliError(cli.main, [ + ...baseArgs, + "--pem-file", + FIXTURE_CERT_PATH, + ]); + }); +}); diff --git a/tools/client-subscriptions-management/src/__tests__/entrypoint/cli/targets-set-mtls.test.ts b/tools/client-subscriptions-management/src/__tests__/entrypoint/cli/targets-set-mtls.test.ts new file mode 100644 index 00000000..a0e53092 --- /dev/null +++ b/tools/client-subscriptions-management/src/__tests__/entrypoint/cli/targets-set-mtls.test.ts @@ -0,0 +1,119 @@ +import * as cli from "src/entrypoint/cli/targets-set-mtls"; +import { + captureCliConsoleState, + expectWrappedCliError, + getMockCreateRepository, + resetCliConsoleState, + resetMockCreateRepository, + restoreCliConsoleState, +} from "src/__tests__/entrypoint/cli/test-utils"; +import { + createClientSubscriptionConfig, + createTarget, +} from "src/__tests__/helpers/client-subscription-fixtures"; + +const mockGetClientConfig = jest.fn(); +const mockPutClientConfig = jest.fn(); +const mockFormatClientConfig = jest.fn(); + +jest.mock("src/entrypoint/cli/helper", () => ({ + ...jest.requireActual("src/entrypoint/cli/helper"), + createRepository: jest.fn(), +})); +jest.mock("src/format", () => ({ + formatClientConfig: (...args: unknown[]) => mockFormatClientConfig(...args), +})); + +const target = createTarget(); +const config = createClientSubscriptionConfig({ targets: [target] }); +const mockCreateRepository = getMockCreateRepository(); + +describe("targets-set-mtls CLI", () => { + const originalCliConsoleState = captureCliConsoleState(); + + const baseArgs = [ + "node", + "script", + "--client-id", + "client-1", + "--bucket-name", + "bucket-1", + "--target-id", + target.targetId, + ]; + + beforeEach(() => { + mockGetClientConfig.mockReset(); + mockGetClientConfig.mockResolvedValue(config); + mockPutClientConfig.mockReset(); + mockPutClientConfig.mockResolvedValue(config); + mockFormatClientConfig.mockReset(); + mockFormatClientConfig.mockReturnValue("formatted-output"); + resetMockCreateRepository({ + getClientConfig: mockGetClientConfig, + putClientConfig: mockPutClientConfig, + }); + resetCliConsoleState(); + console.warn = jest.fn(); + }); + + afterAll(() => { + restoreCliConsoleState(originalCliConsoleState); + }); + + it("enables mTLS with --enable flag", async () => { + await cli.main([...baseArgs, "--enable"]); + + expect(mockPutClientConfig).toHaveBeenCalledWith( + "client-1", + expect.objectContaining({ + targets: [ + expect.objectContaining({ + delivery: expect.objectContaining({ + mtls: expect.objectContaining({ enabled: true }), + }), + }), + ], + }), + false, + ); + }); + + it("disables mTLS with --no-enable flag and emits ANSI warning", async () => { + await cli.main([...baseArgs, "--no-enable"]); + + expect(console.warn).toHaveBeenCalledWith( + expect.stringContaining("Disabling mTLS"), + ); + expect(mockPutClientConfig).toHaveBeenCalledWith( + "client-1", + expect.objectContaining({ + targets: [ + expect.objectContaining({ + delivery: expect.objectContaining({ + mtls: expect.objectContaining({ enabled: false }), + }), + }), + ], + }), + false, + ); + }); + + it("passes dry-run to putClientConfig", async () => { + await cli.main([...baseArgs, "--enable", "--dry-run", "true"]); + + expect(mockPutClientConfig).toHaveBeenCalledWith( + "client-1", + expect.any(Object), + true, + ); + }); + + it("handles errors in wrapped CLI", async () => { + expect.hasAssertions(); + mockCreateRepository.mockRejectedValue(new Error("Boom")); + + await expectWrappedCliError(cli.main, [...baseArgs, "--enable"]); + }); +}); diff --git a/tools/client-subscriptions-management/src/__tests__/entrypoint/cli/targets-set-pinning.test.ts b/tools/client-subscriptions-management/src/__tests__/entrypoint/cli/targets-set-pinning.test.ts new file mode 100644 index 00000000..051dcbdd --- /dev/null +++ b/tools/client-subscriptions-management/src/__tests__/entrypoint/cli/targets-set-pinning.test.ts @@ -0,0 +1,177 @@ +import * as cli from "src/entrypoint/cli/targets-set-pinning"; +import { + captureCliConsoleState, + expectWrappedCliError, + getMockCreateRepository, + resetCliConsoleState, + resetMockCreateRepository, + restoreCliConsoleState, +} from "src/__tests__/entrypoint/cli/test-utils"; +import { + createClientSubscriptionConfig, + createTarget, +} from "src/__tests__/helpers/client-subscription-fixtures"; + +const mockGetClientConfig = jest.fn(); +const mockPutClientConfig = jest.fn(); +const mockFormatClientConfig = jest.fn(); + +jest.mock("src/entrypoint/cli/helper", () => ({ + ...jest.requireActual("src/entrypoint/cli/helper"), + createRepository: jest.fn(), +})); +jest.mock("src/format", () => ({ + formatClientConfig: (...args: unknown[]) => mockFormatClientConfig(...args), +})); + +const target = createTarget({ + delivery: { + mtls: { + enabled: true, + certPinning: { enabled: true, spkiHash: "existing-hash" }, + }, + }, +}); +const config = createClientSubscriptionConfig({ targets: [target] }); +const mockCreateRepository = getMockCreateRepository(); + +describe("targets-set-pinning CLI", () => { + const originalCliConsoleState = captureCliConsoleState(); + + const baseArgs = [ + "node", + "script", + "--client-id", + "client-1", + "--bucket-name", + "bucket-1", + "--target-id", + target.targetId, + ]; + + beforeEach(() => { + mockGetClientConfig.mockReset(); + mockGetClientConfig.mockResolvedValue( + createClientSubscriptionConfig({ + targets: [ + createTarget({ + delivery: { + mtls: { + enabled: true, + certPinning: { enabled: true, spkiHash: "existing-hash" }, + }, + }, + }), + ], + }), + ); + mockPutClientConfig.mockReset(); + mockPutClientConfig.mockResolvedValue(config); + mockFormatClientConfig.mockReset(); + mockFormatClientConfig.mockReturnValue("formatted-output"); + resetMockCreateRepository({ + getClientConfig: mockGetClientConfig, + putClientConfig: mockPutClientConfig, + }); + resetCliConsoleState(); + console.warn = jest.fn(); + }); + + afterAll(() => { + restoreCliConsoleState(originalCliConsoleState); + }); + + it("enables certificate pinning with --enable flag", async () => { + await cli.main([...baseArgs, "--enable"]); + + expect(mockPutClientConfig).toHaveBeenCalledWith( + "client-1", + expect.objectContaining({ + targets: [ + expect.objectContaining({ + delivery: expect.objectContaining({ + mtls: expect.objectContaining({ + certPinning: { enabled: true, spkiHash: "existing-hash" }, + }), + }), + }), + ], + }), + false, + ); + }); + + it("disables pinning with --no-enable flag and emits ANSI warning", async () => { + await cli.main([...baseArgs, "--no-enable"]); + + expect(console.warn).toHaveBeenCalledWith( + expect.stringContaining("Disabling certificate pinning"), + ); + expect(mockPutClientConfig).toHaveBeenCalledWith( + "client-1", + expect.objectContaining({ + targets: [ + expect.objectContaining({ + delivery: expect.objectContaining({ + mtls: expect.objectContaining({ + certPinning: { enabled: false, spkiHash: "existing-hash" }, + }), + }), + }), + ], + }), + false, + ); + }); + + it("preserves existing spkiHash when disabling", async () => { + await cli.main([...baseArgs, "--no-enable"]); + + const putCall = mockPutClientConfig.mock.calls[0]; + const updatedTarget = putCall[1].targets[0]; + expect(updatedTarget.delivery.mtls.certPinning.spkiHash).toBe( + "existing-hash", + ); + }); + + it("passes dry-run to putClientConfig", async () => { + await cli.main([...baseArgs, "--enable", "--dry-run", "true"]); + + expect(mockPutClientConfig).toHaveBeenCalledWith( + "client-1", + expect.any(Object), + true, + ); + }); + + it("handles errors in wrapped CLI", async () => { + expect.hasAssertions(); + mockCreateRepository.mockRejectedValue(new Error("Boom")); + + await expectWrappedCliError(cli.main, [...baseArgs, "--enable"]); + }); + + it("throws when enabling pinning but target has no spkiHash", async () => { + expect.hasAssertions(); + mockGetClientConfig.mockResolvedValue( + createClientSubscriptionConfig({ + targets: [ + createTarget({ + delivery: { + mtls: { + enabled: true, + certPinning: { enabled: false }, + }, + }, + }), + ], + }), + ); + + await expectWrappedCliError( + cli.main, + [...baseArgs, "--enable"], + `Target '${target.targetId}' has no SPKI hash stored. Run 'targets-set-certificate' first to configure a certificate hash before enabling pinning.`, + ); + }); +}); diff --git a/tools/client-subscriptions-management/src/__tests__/fixtures/test-cert.pem b/tools/client-subscriptions-management/src/__tests__/fixtures/test-cert.pem new file mode 100644 index 00000000..66accebb --- /dev/null +++ b/tools/client-subscriptions-management/src/__tests__/fixtures/test-cert.pem @@ -0,0 +1,14 @@ +-----BEGIN CERTIFICATE----- +MIICETCCAbgCCQD0bFWfktPerzAKBggqhkjOPQQDAjAXMRUwEwYDVQQDDAx0ZXN0 +LWZpeHR1cmUwHhcNMjYwNDE3MDgzMjAzWhcNMzYwNDE0MDgzMjAzWjAXMRUwEwYD +VQQDDAx0ZXN0LWZpeHR1cmUwggFLMIIBAwYHKoZIzj0CATCB9wIBATAsBgcqhkjO +PQEBAiEA/////wAAAAEAAAAAAAAAAAAAAAD///////////////8wWwQg/////wAA +AAEAAAAAAAAAAAAAAAD///////////////wEIFrGNdiqOpPns+u9VXaYhrxlHQaw +zFOw9jvOPD4n0mBLAxUAxJ02CIbnBJNqZnjhE50mt4GffpAEQQRrF9Hy4SxCR/i8 +5uVjpEDydwN9gS3rM6D0oTlF2JjClk/jQuL+Gn+bjufrSnwPnhYrzjNXazFezsu2 +QGg3v1H1AiEA/////wAAAAD//////////7zm+q2nF56E87nKwvxjJVECAQEDQgAE +WpOtSSCENuhBWnPFy4csFZkUT/t77xiQxJr/mrClSsNs4H7vwgXRc5OCT+BuTruT +J/kv6ipp/9s/c5/WP8Ln9zAKBggqhkjOPQQDAgNHADBEAiA46dblj9UZZe163Me1 +sydmzRkzLrtAy1pLCcPp86Z4WwIgRD6/Oa4UQ/C6tCDpdLquzAtRxSNKuHMPLXK9 +vkFt930= +-----END CERTIFICATE----- diff --git a/tools/client-subscriptions-management/src/__tests__/repository/ssm-applications-map.test.ts b/tools/client-subscriptions-management/src/__tests__/repository/s3-applications-map.test.ts similarity index 67% rename from tools/client-subscriptions-management/src/__tests__/repository/ssm-applications-map.test.ts rename to tools/client-subscriptions-management/src/__tests__/repository/s3-applications-map.test.ts index afb94e41..9e9084f4 100644 --- a/tools/client-subscriptions-management/src/__tests__/repository/ssm-applications-map.test.ts +++ b/tools/client-subscriptions-management/src/__tests__/repository/s3-applications-map.test.ts @@ -1,38 +1,48 @@ import { - GetParameterCommand, - PutParameterCommand, - type SSMClient, -} from "@aws-sdk/client-ssm"; -import SsmApplicationsMapRepository from "src/repository/ssm-applications-map"; + GetObjectCommand, + PutObjectCommand, + type S3Client, +} from "@aws-sdk/client-s3"; +import type { SdkStream } from "@smithy/types"; +import S3ApplicationsMapRepository from "src/repository/s3-applications-map"; + +const mockBody = (content: string) => + ({ + transformToString: jest.fn().mockResolvedValue(content), + }) as unknown as SdkStream; const createRepository = (send: jest.Mock = jest.fn()) => { - const client = { send } as unknown as SSMClient; + const client = { send } as unknown as S3Client; return { - repository: new SsmApplicationsMapRepository(client, "/test/param"), + repository: new S3ApplicationsMapRepository( + client, + "test-bucket", + "test/applications-map.json", + ), send, }; }; -describe("SsmApplicationsMapRepository", () => { +describe("S3ApplicationsMapRepository", () => { describe("getApplication", () => { it("returns the application ID for an existing client", async () => { const { repository, send } = createRepository(); send.mockResolvedValueOnce({ - Parameter: { - Value: JSON.stringify({ "client-1": "app-1", "client-2": "app-2" }), - }, + Body: mockBody( + JSON.stringify({ "client-1": "app-1", "client-2": "app-2" }), + ), }); const result = await repository.getApplication("client-1"); - expect(send).toHaveBeenCalledWith(expect.any(GetParameterCommand)); + expect(send).toHaveBeenCalledWith(expect.any(GetObjectCommand)); expect(result).toBe("app-1"); }); it("returns undefined when the client is not in the map", async () => { const { repository, send } = createRepository(); send.mockResolvedValueOnce({ - Parameter: { Value: JSON.stringify({ "other-client": "app-1" }) }, + Body: mockBody(JSON.stringify({ "other-client": "app-1" })), }); const result = await repository.getApplication("client-1"); @@ -40,10 +50,10 @@ describe("SsmApplicationsMapRepository", () => { expect(result).toBeUndefined(); }); - it("returns undefined when parameter does not exist", async () => { + it("returns undefined when object does not exist", async () => { const { repository, send } = createRepository(); const error = Object.assign(new Error("not found"), { - name: "ParameterNotFound", + name: "NoSuchKey", }); send.mockRejectedValueOnce(error); @@ -52,16 +62,16 @@ describe("SsmApplicationsMapRepository", () => { expect(result).toBeUndefined(); }); - it("returns undefined when parameter has no value", async () => { + it("returns undefined when object body is empty", async () => { const { repository, send } = createRepository(); - send.mockResolvedValueOnce({ Parameter: {} }); + send.mockResolvedValueOnce({ Body: undefined }); const result = await repository.getApplication("client-1"); expect(result).toBeUndefined(); }); - it("rethrows unexpected SSM errors", async () => { + it("rethrows unexpected S3 errors", async () => { const { repository, send } = createRepository(); send.mockRejectedValueOnce( Object.assign(new Error("Network failure"), { name: "NetworkError" }), @@ -78,16 +88,14 @@ describe("SsmApplicationsMapRepository", () => { const { repository, send } = createRepository(); send .mockResolvedValueOnce({ - Parameter: { - Value: JSON.stringify({ "existing-client": "existing-app" }), - }, + Body: mockBody(JSON.stringify({ "existing-client": "existing-app" })), }) .mockResolvedValueOnce({}); const result = await repository.addApplication("client-1", "app-1"); - expect(send).toHaveBeenNthCalledWith(1, expect.any(GetParameterCommand)); - expect(send).toHaveBeenNthCalledWith(2, expect.any(PutParameterCommand)); + expect(send).toHaveBeenNthCalledWith(1, expect.any(GetObjectCommand)); + expect(send).toHaveBeenNthCalledWith(2, expect.any(PutObjectCommand)); expect(result).toEqual( new Map([ ["existing-client", "existing-app"], @@ -96,10 +104,10 @@ describe("SsmApplicationsMapRepository", () => { ); }); - it("starts from empty map when parameter does not exist", async () => { + it("starts from empty map when object does not exist", async () => { const { repository, send } = createRepository(); const error = Object.assign(new Error("not found"), { - name: "ParameterNotFound", + name: "NoSuchKey", }); send.mockRejectedValueOnce(error).mockResolvedValueOnce({}); @@ -109,9 +117,9 @@ describe("SsmApplicationsMapRepository", () => { expect(send).toHaveBeenCalledTimes(2); }); - it("starts from empty map when parameter has no value", async () => { + it("starts from empty map when object body is empty", async () => { const { repository, send } = createRepository(); - send.mockResolvedValueOnce({ Parameter: {} }).mockResolvedValueOnce({}); + send.mockResolvedValueOnce({ Body: undefined }).mockResolvedValueOnce({}); const result = await repository.addApplication("client-1", "app-1"); @@ -122,7 +130,7 @@ describe("SsmApplicationsMapRepository", () => { const { repository, send } = createRepository(); send .mockResolvedValueOnce({ - Parameter: { Value: JSON.stringify({ "client-1": "old-app" }) }, + Body: mockBody(JSON.stringify({ "client-1": "old-app" })), }) .mockResolvedValueOnce({}); @@ -134,7 +142,7 @@ describe("SsmApplicationsMapRepository", () => { it("skips the put when dry-run is true", async () => { const { repository, send } = createRepository(); send.mockResolvedValueOnce({ - Parameter: { Value: JSON.stringify({}) }, + Body: mockBody(JSON.stringify({})), }); const result = await repository.addApplication("client-1", "app-1", true); @@ -143,7 +151,7 @@ describe("SsmApplicationsMapRepository", () => { expect(result).toEqual(new Map([["client-1", "app-1"]])); }); - it("rethrows unexpected SSM errors", async () => { + it("rethrows unexpected S3 errors", async () => { const { repository, send } = createRepository(); send.mockRejectedValueOnce( Object.assign(new Error("Network failure"), { name: "NetworkError" }), diff --git a/tools/client-subscriptions-management/src/__tests__/repository/s3.test.ts b/tools/client-subscriptions-management/src/__tests__/repository/s3.test.ts index 30a2ad43..ebd6c252 100644 --- a/tools/client-subscriptions-management/src/__tests__/repository/s3.test.ts +++ b/tools/client-subscriptions-management/src/__tests__/repository/s3.test.ts @@ -22,6 +22,60 @@ describe("S3Repository", () => { expect(send.mock.calls[0][0]).toBeInstanceOf(GetObjectCommand); }); + it("prepends keyPrefix to getObject key", async () => { + const send = jest.fn().mockResolvedValue({ + Body: { transformToString: jest.fn().mockResolvedValue("content") }, + }); + const repository = new S3Repository( + "bucket", + { send } as unknown as S3Client, + "dev/", + ); + + await repository.getObject("client_subscriptions/client-1.json"); + + const command = send.mock.calls[0][0] as GetObjectCommand; + expect(command.input.Key).toBe("dev/client_subscriptions/client-1.json"); + }); + + it("prepends keyPrefix to putRawData key", async () => { + const send = jest.fn().mockResolvedValue({}); + const repository = new S3Repository( + "bucket", + { send } as unknown as S3Client, + "abc/", + ); + + await repository.putRawData("payload", "client_subscriptions/c.json"); + + const command = send.mock.calls[0][0] as PutObjectCommand; + expect(command.input.Key).toBe("abc/client_subscriptions/c.json"); + }); + + it("prepends keyPrefix to listObjectKeys prefix and strips it from returned keys", async () => { + const send = jest.fn().mockResolvedValue({ + Contents: [ + { Key: "main/client_subscriptions/a.json" }, + { Key: "main/client_subscriptions/b.json" }, + ], + NextContinuationToken: undefined, + }); + const repository = new S3Repository( + "bucket", + { send } as unknown as S3Client, + "main/", + ); + + const keys = await repository.listObjectKeys("client_subscriptions/"); + + const command = send.mock.calls[0][0] as ListObjectsV2Command; + expect(command.input.Prefix).toBe("main/client_subscriptions/"); + expect(keys).toEqual([ + "client_subscriptions/a.json", + "client_subscriptions/b.json", + ]); + }); + it("throws when body is missing", async () => { const send = jest.fn().mockResolvedValue({}); const repository = new S3Repository("bucket", { diff --git a/tools/client-subscriptions-management/src/aws.ts b/tools/client-subscriptions-management/src/aws.ts index 5599b50b..57ae1c85 100644 --- a/tools/client-subscriptions-management/src/aws.ts +++ b/tools/client-subscriptions-management/src/aws.ts @@ -1,9 +1,8 @@ import { S3Client } from "@aws-sdk/client-s3"; -import { SSMClient } from "@aws-sdk/client-ssm"; import { GetCallerIdentityCommand, STSClient } from "@aws-sdk/client-sts"; import { fromIni } from "@aws-sdk/credential-providers"; import { ClientSubscriptionRepository } from "src/repository/client-subscriptions"; -import SsmApplicationsMapRepository from "src/repository/ssm-applications-map"; +import S3ApplicationsMapRepository from "src/repository/s3-applications-map"; import { S3Repository } from "src/repository/s3"; export const resolveProfile = ( @@ -28,43 +27,55 @@ export const deriveBucketName = ( accountId: string, environment: string, region: string, -): string => - `nhs-${accountId}-${region}-${environment}-callbacks-subscription-config`; +): string => `nhs-${accountId}-${region}-main-acct-clie-client-configs`; export const resolveRegion = ( regionArg?: string, env: NodeJS.ProcessEnv = process.env, ): string | undefined => regionArg ?? env.AWS_REGION ?? env.AWS_DEFAULT_REGION; +export interface DeploymentContext { + environment: string; + region: string; + accountId: string; + profile?: string; +} + +export const resolveDeploymentContext = async (args: { + environment?: string; + region?: string; + profile?: string; +}): Promise => { + const profile = resolveProfile(args.profile); + const region = resolveRegion(args.region) ?? "eu-west-2"; + const environment = args.environment ?? process.env.ENVIRONMENT; + + if (!environment) { + throw new Error( + "Environment is required. Please provide via --environment or ENVIRONMENT env var.", + ); + } + + const accountId = + process.env.AWS_ACCOUNT_ID ?? (await resolveAccountId(profile, region)); + + return { environment, region, accountId, profile }; +}; + export const resolveBucketName = async (args: { bucketName?: string; environment?: string; profile?: string; region?: string; }): Promise => { - const { bucketName, environment, profile, region } = args; - - if (bucketName) { - return bucketName; + if (args.bucketName) { + return args.bucketName; } - const resolvedEnvironment = environment ?? process.env.ENVIRONMENT; - const resolvedRegion = resolveRegion(region) ?? "eu-west-2"; - const resolvedAccountId = - process.env.AWS_ACCOUNT_ID ?? - (await resolveAccountId(profile, resolvedRegion)); - - if (!resolvedEnvironment) { - throw new Error( - "Environment is required to derive bucket name. Please provide via --environment or ENVIRONMENT env var.", - ); - } + const { accountId, environment, region } = + await resolveDeploymentContext(args); - return deriveBucketName( - resolvedAccountId, - resolvedEnvironment, - resolvedRegion, - ); + return deriveBucketName(accountId, environment, region); }; export const createS3Client = ( @@ -80,58 +91,57 @@ export const createS3Client = ( export const createRepository = (options: { bucketName: string; + environment: string; region?: string; profile?: string; }): ClientSubscriptionRepository => { const s3Repository = new S3Repository( options.bucketName, createS3Client(options.region, options.profile), + `${options.environment}/`, ); return new ClientSubscriptionRepository(s3Repository); }; -export const createSsmClient = ( - region?: string, - profile?: string, - env: NodeJS.ProcessEnv = process.env, -): SSMClient => { - const endpoint = env.AWS_ENDPOINT_URL; - const credentials = profile ? fromIni({ profile }) : undefined; - return new SSMClient({ region, endpoint, credentials }); -}; +export const deriveApplicationsMapBucketName = ( + accountId: string, + region: string, +): string => `nhs-${accountId}-${region}-main-acct-clie-apps-map`; -export const deriveParameterName = (environment: string): string => - `/nhs/${environment}/callbacks/applications-map`; +export const deriveApplicationsMapKey = (environment: string): string => + `${environment}/applications-map.json`; -export const resolveParameterName = (args: { - parameterName?: string; +export const resolveApplicationsMapLocation = async (args: { + bucketName?: string; + key?: string; environment?: string; - env?: NodeJS.ProcessEnv; -}): string => { - const { env = process.env, environment, parameterName } = args; - - if (parameterName) { - return parameterName; + profile?: string; + region?: string; +}): Promise<{ bucket: string; key: string }> => { + if (args.bucketName && args.key) { + return { bucket: args.bucketName, key: args.key }; } - const resolvedEnvironment = environment ?? env.ENVIRONMENT; - if (!resolvedEnvironment) { - throw new Error( - "Environment is required to derive parameter name. Please provide via --environment or ENVIRONMENT env var.", - ); - } + const { accountId, environment, region } = + await resolveDeploymentContext(args); - return deriveParameterName(resolvedEnvironment); + return { + bucket: + args.bucketName ?? deriveApplicationsMapBucketName(accountId, region), + key: args.key ?? deriveApplicationsMapKey(environment), + }; }; -export const createSsmApplicationsMapRepository = (options: { - parameterName: string; +export const createS3ApplicationsMapRepository = (options: { + bucket: string; + key: string; region?: string; profile?: string; -}): SsmApplicationsMapRepository => - new SsmApplicationsMapRepository( - createSsmClient(options.region, options.profile), - options.parameterName, +}): S3ApplicationsMapRepository => + new S3ApplicationsMapRepository( + createS3Client(options.region, options.profile), + options.bucket, + options.key, ); -export { default as SsmApplicationsMapRepository } from "src/repository/ssm-applications-map"; +export { default as S3ApplicationsMapRepository } from "src/repository/s3-applications-map"; diff --git a/tools/client-subscriptions-management/src/domain/client-subscription-builder.ts b/tools/client-subscriptions-management/src/domain/client-subscription-builder.ts index f91ee5a4..514c48d7 100644 --- a/tools/client-subscriptions-management/src/domain/client-subscription-builder.ts +++ b/tools/client-subscriptions-management/src/domain/client-subscription-builder.ts @@ -7,12 +7,16 @@ import type { MessageStatusSubscriptionConfiguration, SupplierStatus, } from "@nhs-notify-client-callbacks/models"; +import pc from "picocolors"; export type BuildTargetArgs = { apiEndpoint: string; apiKey: string; apiKeyHeaderName?: string; rateLimit: number; + maxRetryDurationSeconds?: number; + mtls?: { enabled: boolean }; + certPinning?: { enabled: boolean; spkiHash?: string }; }; export type BuildMessageStatusSubscriptionArgs = { @@ -30,6 +34,42 @@ export type BuildChannelStatusSubscriptionArgs = { }; export function buildTarget(args: BuildTargetArgs): CallbackTarget { + const mtls = args.mtls ?? { enabled: false }; + const certPinning = args.certPinning ?? { enabled: false }; + + const warnings: string[] = []; + + if (!mtls.enabled) { + warnings.push("mTLS is disabled — callbacks will not use mutual TLS"); + } + + if (mtls.enabled && !certPinning.enabled) { + warnings.push("mTLS is enabled but certificate pinning is disabled"); + } + + if (certPinning.enabled && !certPinning.spkiHash) { + throw new Error( + "Certificate pinning cannot be enabled without an SPKI hash. Run 'targets-set-certificate' first.", + ); + } + + if (!mtls.enabled && certPinning.enabled) { + warnings.push("Certificate pinning is enabled but mTLS is disabled"); + } + + if ( + args.maxRetryDurationSeconds !== undefined && + args.maxRetryDurationSeconds < 60 + ) { + warnings.push( + `maxRetryDurationSeconds is ${args.maxRetryDurationSeconds}s — values below 60s may exhaust the retry window before a single delivery attempt completes`, + ); + } + + for (const warning of warnings) { + console.warn(pc.bold(pc.red(`WARNING: ${warning}`))); + } + return { targetId: crypto.randomUUID(), type: "API", @@ -40,6 +80,15 @@ export function buildTarget(args: BuildTargetArgs): CallbackTarget { headerName: args.apiKeyHeaderName ?? "x-api-key", headerValue: args.apiKey, }, + delivery: { + ...(args.maxRetryDurationSeconds !== undefined && { + maxRetryDurationSeconds: args.maxRetryDurationSeconds, + }), + mtls: { + ...mtls, + certPinning, + }, + }, }; } diff --git a/tools/client-subscriptions-management/src/entrypoint/cli/applications-map-add.ts b/tools/client-subscriptions-management/src/entrypoint/cli/applications-map-add.ts index a98e574f..f1bfcf8b 100644 --- a/tools/client-subscriptions-management/src/entrypoint/cli/applications-map-add.ts +++ b/tools/client-subscriptions-management/src/entrypoint/cli/applications-map-add.ts @@ -1,20 +1,20 @@ import type { Argv } from "yargs"; import { + type ApplicationsMapCliArgs, type CliCommand, type ClientCliArgs, - type SsmCliArgs, type WriteCliArgs, + applicationsMapOptions, clientIdOption, commonOptions, - createSsmApplicationsMapRepository, - parameterNameOption, + createS3ApplicationsMapRepository, runCommand, writeOptions, } from "src/entrypoint/cli/helper"; import { formatApplicationsMap } from "src/format"; type ApplicationsMapAddArgs = ClientCliArgs & - SsmCliArgs & + ApplicationsMapCliArgs & WriteCliArgs & { "application-id": string; }; @@ -23,7 +23,7 @@ export const builder = (yargs: Argv) => yargs.options({ ...commonOptions, ...clientIdOption, - ...parameterNameOption, + ...applicationsMapOptions, ...writeOptions, "application-id": { type: "string", @@ -35,22 +35,23 @@ export const builder = (yargs: Argv) => export const handler: CliCommand["handler"] = async ( argv, ) => { - const repository = createSsmApplicationsMapRepository(argv); + const repository = await createS3ApplicationsMapRepository(argv); const result = await repository.addApplication( argv["client-id"], argv["application-id"], argv["dry-run"], ); - console.log(`Applications map updated for client '${argv["client-id"]}'.`); if (argv["dry-run"]) { - console.log("Dry run — no changes written to SSM."); + console.log("Dry run \u2014 no changes written."); + } else { + console.log(`Applications map updated for client '${argv["client-id"]}'.`); } console.log(formatApplicationsMap(result)); }; export const command: CliCommand = { command: "applications-map-add", - describe: "Add or update a client-to-application-ID mapping in SSM", + describe: "Add or update a client-to-application-ID mapping in S3", builder, handler, }; diff --git a/tools/client-subscriptions-management/src/entrypoint/cli/applications-map-get.ts b/tools/client-subscriptions-management/src/entrypoint/cli/applications-map-get.ts index 5ffe2192..3e22db39 100644 --- a/tools/client-subscriptions-management/src/entrypoint/cli/applications-map-get.ts +++ b/tools/client-subscriptions-management/src/entrypoint/cli/applications-map-get.ts @@ -1,28 +1,28 @@ import type { Argv } from "yargs"; import { + type ApplicationsMapCliArgs, type CliCommand, type ClientCliArgs, - type SsmCliArgs, + applicationsMapOptions, clientIdOption, commonOptions, - createSsmApplicationsMapRepository, - parameterNameOption, + createS3ApplicationsMapRepository, runCommand, } from "src/entrypoint/cli/helper"; -type ApplicationsMapGetArgs = ClientCliArgs & SsmCliArgs; +type ApplicationsMapGetArgs = ClientCliArgs & ApplicationsMapCliArgs; export const builder = (yargs: Argv) => yargs.options({ ...commonOptions, ...clientIdOption, - ...parameterNameOption, + ...applicationsMapOptions, }); export const handler: CliCommand["handler"] = async ( argv, ) => { - const repository = createSsmApplicationsMapRepository(argv); + const repository = await createS3ApplicationsMapRepository(argv); const applicationId = await repository.getApplication(argv["client-id"]); if (applicationId) { diff --git a/tools/client-subscriptions-management/src/entrypoint/cli/clients-put.ts b/tools/client-subscriptions-management/src/entrypoint/cli/clients-put.ts index c0d13554..ef7cf65f 100644 --- a/tools/client-subscriptions-management/src/entrypoint/cli/clients-put.ts +++ b/tools/client-subscriptions-management/src/entrypoint/cli/clients-put.ts @@ -56,8 +56,7 @@ export const handler: CliCommand["handler"] = async (argv) => { return; } - // Safe as this is an internal tool and this CLI option we are expecting the user will run locally and manually - // eslint-disable-next-line security/detect-non-literal-fs-filename + // eslint-disable-next-line security/detect-non-literal-fs-filename -- path is provided directly by the operator via CLI arg const rawJson = argv.json ?? readFileSync(argv.file!, "utf8"); let parsed: unknown; @@ -94,11 +93,11 @@ export const handler: CliCommand["handler"] = async (argv) => { argv["dry-run"], ); - console.log(`Config written for client: ${argv["client-id"]}`); - if (argv["dry-run"]) { - console.log("Dry run: config is valid"); + console.log("Dry run \u2014 no changes written."); console.log(JSON.stringify(result, null, 2)); + } else { + console.log(`Config written for client: ${argv["client-id"]}`); } }; diff --git a/tools/client-subscriptions-management/src/entrypoint/cli/helper.ts b/tools/client-subscriptions-management/src/entrypoint/cli/helper.ts index 14e998dd..514626ca 100644 --- a/tools/client-subscriptions-management/src/entrypoint/cli/helper.ts +++ b/tools/client-subscriptions-management/src/entrypoint/cli/helper.ts @@ -1,10 +1,13 @@ +import type { + CallbackTarget, + ClientSubscriptionConfiguration, +} from "@nhs-notify-client-callbacks/models"; import { createRepository as createRepositoryFromOptions, - createSsmApplicationsMapRepository as createSsmApplicationsMapRepositoryFromOptions, + createS3ApplicationsMapRepository as createS3ApplicationsMapRepositoryFromOptions, + resolveApplicationsMapLocation as resolveApplicationsMapLocationFromAws, resolveBucketName, - resolveParameterName as resolveParameterNameFromAws, - resolveProfile, - resolveRegion, + resolveDeploymentContext, } from "src/aws"; import { hideBin } from "yargs/helpers"; import yargs from "yargs/yargs"; @@ -37,15 +40,24 @@ export type WriteCliArgs = { }; export const createRepository = async (argv: CommonCliArgs) => { - const region = resolveRegion(argv.region); - const profile = resolveProfile(argv.profile); + const { environment, profile, region } = await resolveDeploymentContext({ + environment: argv.environment, + region: argv.region, + profile: argv.profile, + }); + const bucketName = await resolveBucketName({ bucketName: argv["bucket-name"], - environment: argv.environment, + environment, + region, + profile, + }); + return createRepositoryFromOptions({ + bucketName, + environment, region, profile, }); - return createRepositoryFromOptions({ bucketName, region, profile }); }; type BaseArgs = Record; @@ -124,6 +136,14 @@ export const clientIdOption = { }, }; +export const targetIdOption = { + "target-id": { + type: "string" as const, + demandOption: true as const, + description: "Target identifier", + }, +}; + export const writeOptions = { "dry-run": { type: "boolean" as const, @@ -133,29 +153,72 @@ export const writeOptions = { }, }; -export type SsmCliArgs = CommonCliArgs & { - "parameter-name"?: string; +export type ApplicationsMapCliArgs = CommonCliArgs & { + "applications-map-bucket"?: string; + "applications-map-key"?: string; }; -export const parameterNameOption = { - "parameter-name": { +export const applicationsMapOptions = { + "applications-map-bucket": { type: "string" as const, demandOption: false as const, description: - "Explicit SSM parameter name for the applications map (overrides derived name)", + "Explicit S3 bucket name for the applications map (overrides derived name)", + }, + "applications-map-key": { + type: "string" as const, + demandOption: false as const, + description: + "Explicit S3 key for the applications map (overrides derived key)", }, }; -export const createSsmApplicationsMapRepository = (argv: SsmCliArgs) => { - const region = resolveRegion(argv.region); - const profile = resolveProfile(argv.profile); - const parameterName = resolveParameterNameFromAws({ - parameterName: argv["parameter-name"], +export const createS3ApplicationsMapRepository = async ( + argv: ApplicationsMapCliArgs, +) => { + const { environment, profile, region } = await resolveDeploymentContext({ environment: argv.environment, + region: argv.region, + profile: argv.profile, }); - return createSsmApplicationsMapRepositoryFromOptions({ - parameterName, + const { bucket, key } = await resolveApplicationsMapLocationFromAws({ + bucketName: argv["applications-map-bucket"], + key: argv["applications-map-key"], + environment, + region, + profile, + }); + return createS3ApplicationsMapRepositoryFromOptions({ + bucket, + key, region, profile, }); }; + +export async function requireClientConfig( + repository: { + getClientConfig: ( + clientId: string, + ) => Promise; + }, + clientId: string, +): Promise { + const config = await repository.getClientConfig(clientId); + if (!config) { + throw new Error(`No configuration found for client: ${clientId}`); + } + return config; +} + +export function requireTargetConfig( + config: ClientSubscriptionConfiguration, + clientId: string, + targetId: string, +): CallbackTarget { + const target = config.targets.find((t) => t.targetId === targetId); + if (!target) { + throw new Error(`Target '${targetId}' not found for client '${clientId}'`); + } + return target; +} diff --git a/tools/client-subscriptions-management/src/entrypoint/cli/subscriptions-add.ts b/tools/client-subscriptions-management/src/entrypoint/cli/subscriptions-add.ts index fc34cd2b..b367fa2f 100644 --- a/tools/client-subscriptions-management/src/entrypoint/cli/subscriptions-add.ts +++ b/tools/client-subscriptions-management/src/entrypoint/cli/subscriptions-add.ts @@ -146,6 +146,9 @@ export const handler: CliCommand["handler"] = async ( argv["dry-run"], ); + if (argv["dry-run"]) { + console.log("Dry run \u2014 no changes written."); + } console.log(formatClientConfig(result)); }; diff --git a/tools/client-subscriptions-management/src/entrypoint/cli/subscriptions-del.ts b/tools/client-subscriptions-management/src/entrypoint/cli/subscriptions-del.ts index 74c07da0..a5f675f8 100644 --- a/tools/client-subscriptions-management/src/entrypoint/cli/subscriptions-del.ts +++ b/tools/client-subscriptions-management/src/entrypoint/cli/subscriptions-del.ts @@ -39,6 +39,9 @@ export const handler: CliCommand["handler"] = async ( argv["dry-run"], ); + if (argv["dry-run"]) { + console.log("Dry run \u2014 no changes written."); + } console.log(formatClientConfig(result)); }; diff --git a/tools/client-subscriptions-management/src/entrypoint/cli/subscriptions-set-states.ts b/tools/client-subscriptions-management/src/entrypoint/cli/subscriptions-set-states.ts index ee17a979..1d6f8228 100644 --- a/tools/client-subscriptions-management/src/entrypoint/cli/subscriptions-set-states.ts +++ b/tools/client-subscriptions-management/src/entrypoint/cli/subscriptions-set-states.ts @@ -89,6 +89,9 @@ export const handler: CliCommand["handler"] = argv["dry-run"], ); + if (argv["dry-run"]) { + console.log("Dry run \u2014 no changes written."); + } console.log(formatClientConfig(result)); }; diff --git a/tools/client-subscriptions-management/src/entrypoint/cli/targets-add.ts b/tools/client-subscriptions-management/src/entrypoint/cli/targets-add.ts index 524d51d7..35724d1a 100644 --- a/tools/client-subscriptions-management/src/entrypoint/cli/targets-add.ts +++ b/tools/client-subscriptions-management/src/entrypoint/cli/targets-add.ts @@ -65,6 +65,9 @@ export const handler: CliCommand["handler"] = async (argv) => { target, argv["dry-run"], ); + if (argv["dry-run"]) { + console.log("Dry run \u2014 no changes written."); + } console.log(`Target added with ID: ${target.targetId}`); console.log(formatClientConfig(result)); }; diff --git a/tools/client-subscriptions-management/src/entrypoint/cli/targets-del.ts b/tools/client-subscriptions-management/src/entrypoint/cli/targets-del.ts index 6fe56ac2..d7030dec 100644 --- a/tools/client-subscriptions-management/src/entrypoint/cli/targets-del.ts +++ b/tools/client-subscriptions-management/src/entrypoint/cli/targets-del.ts @@ -7,6 +7,7 @@ import { commonOptions, createRepository, runCommand, + targetIdOption, writeOptions, } from "src/entrypoint/cli/helper"; import { formatClientConfig } from "src/format"; @@ -20,12 +21,8 @@ export const builder = (yargs: Argv) => yargs.options({ ...commonOptions, ...clientIdOption, + ...targetIdOption, ...writeOptions, - "target-id": { - type: "string", - demandOption: true, - description: "Target identifier to delete", - }, }); export const handler: CliCommand["handler"] = async (argv) => { @@ -37,6 +34,9 @@ export const handler: CliCommand["handler"] = async (argv) => { argv["dry-run"], ); + if (argv["dry-run"]) { + console.log("Dry run \u2014 no changes written."); + } console.log(formatClientConfig(result)); }; diff --git a/tools/client-subscriptions-management/src/entrypoint/cli/targets-set-certificate.ts b/tools/client-subscriptions-management/src/entrypoint/cli/targets-set-certificate.ts new file mode 100644 index 00000000..857d2991 --- /dev/null +++ b/tools/client-subscriptions-management/src/entrypoint/cli/targets-set-certificate.ts @@ -0,0 +1,94 @@ +import { X509Certificate, createHash } from "node:crypto"; +import { readFileSync } from "node:fs"; +import type { Argv } from "yargs"; +import { + type CliCommand, + type ClientCliArgs, + type WriteCliArgs, + clientIdOption, + commonOptions, + createRepository, + requireClientConfig, + requireTargetConfig, + runCommand, + targetIdOption, + writeOptions, +} from "src/entrypoint/cli/helper"; +import { formatClientConfig } from "src/format"; + +type TargetsSetCertificateArgs = ClientCliArgs & + WriteCliArgs & { + "target-id": string; + "pem-file": string; + }; + +export const builder = (yargs: Argv) => + yargs.options({ + ...commonOptions, + ...clientIdOption, + ...targetIdOption, + ...writeOptions, + "pem-file": { + type: "string", + demandOption: true, + description: "Path to PEM certificate file", + }, + }); + +function extractSpkiHash(pemPath: string): string { + // eslint-disable-next-line security/detect-non-literal-fs-filename -- path is provided directly by the operator via CLI arg + const pemBuffer = readFileSync(pemPath); + const x509 = new X509Certificate(pemBuffer); + const spkiDer = x509.publicKey.export({ + type: "spki", + format: "der", + }) as Buffer; + return createHash("sha256").update(spkiDer).digest("base64"); +} + +export const handler: CliCommand["handler"] = async ( + argv, +) => { + const spkiHash = extractSpkiHash(argv["pem-file"]); + console.log(`Extracted SPKI hash: ${spkiHash}`); + + const repository = await createRepository(argv); + const config = await requireClientConfig(repository, argv["client-id"]); + const target = requireTargetConfig( + config, + argv["client-id"], + argv["target-id"], + ); + + const mtls = target.delivery?.mtls ?? { enabled: false }; + const certPinning = mtls.certPinning ?? { enabled: false }; + target.delivery = { + ...target.delivery, + mtls: { + ...mtls, + certPinning: { + ...certPinning, + spkiHash, + }, + }, + }; + + const result = await repository.putClientConfig( + argv["client-id"], + config, + argv["dry-run"], + ); + console.log("Certificate SPKI hash stored successfully"); + console.log(formatClientConfig(result)); +}; + +export const command: CliCommand = { + command: "targets-set-certificate", + describe: "Extract and store SPKI hash from a PEM certificate for a target", + builder, + handler, +}; + +export async function main(args: string[] = process.argv) { + await runCommand(command, args); +} diff --git a/tools/client-subscriptions-management/src/entrypoint/cli/targets-set-mtls.ts b/tools/client-subscriptions-management/src/entrypoint/cli/targets-set-mtls.ts new file mode 100644 index 00000000..ae9127ef --- /dev/null +++ b/tools/client-subscriptions-management/src/entrypoint/cli/targets-set-mtls.ts @@ -0,0 +1,87 @@ +import type { Argv } from "yargs"; +import pc from "picocolors"; +import { + type CliCommand, + type ClientCliArgs, + type WriteCliArgs, + clientIdOption, + commonOptions, + createRepository, + requireClientConfig, + requireTargetConfig, + runCommand, + targetIdOption, + writeOptions, +} from "src/entrypoint/cli/helper"; +import { formatClientConfig } from "src/format"; + +type TargetsSetMtlsArgs = ClientCliArgs & + WriteCliArgs & { + "target-id": string; + enable: boolean; + }; + +export const builder = (yargs: Argv) => + yargs.options({ + ...commonOptions, + ...clientIdOption, + ...targetIdOption, + ...writeOptions, + enable: { + type: "boolean", + demandOption: true, + description: + "Enable or disable mTLS for this target (use --no-enable to disable)", + }, + }); + +export const handler: CliCommand["handler"] = async ( + argv, +) => { + const enabled = argv.enable; + + if (!enabled) { + console.warn( + pc.bold( + pc.red("WARNING: Disabling mTLS — callbacks will not use mutual TLS"), + ), + ); + } + + const repository = await createRepository(argv); + const config = await requireClientConfig(repository, argv["client-id"]); + const target = requireTargetConfig( + config, + argv["client-id"], + argv["target-id"], + ); + + target.delivery = { + ...target.delivery, + mtls: { + ...target.delivery?.mtls, + enabled, + }, + }; + + const result = await repository.putClientConfig( + argv["client-id"], + config, + argv["dry-run"], + ); + console.log( + `mTLS ${enabled ? "enabled" : "disabled"} for target ${argv["target-id"]}`, + ); + console.log(formatClientConfig(result)); +}; + +export const command: CliCommand = { + command: "targets-set-mtls", + describe: "Enable or disable mTLS for a callback target", + builder, + handler, +}; + +export async function main(args: string[] = process.argv) { + await runCommand(command, args); +} diff --git a/tools/client-subscriptions-management/src/entrypoint/cli/targets-set-pinning.ts b/tools/client-subscriptions-management/src/entrypoint/cli/targets-set-pinning.ts new file mode 100644 index 00000000..6816d85f --- /dev/null +++ b/tools/client-subscriptions-management/src/entrypoint/cli/targets-set-pinning.ts @@ -0,0 +1,94 @@ +import type { Argv } from "yargs"; +import pc from "picocolors"; +import { + type CliCommand, + type ClientCliArgs, + type WriteCliArgs, + clientIdOption, + commonOptions, + createRepository, + requireClientConfig, + requireTargetConfig, + runCommand, + targetIdOption, + writeOptions, +} from "src/entrypoint/cli/helper"; +import { formatClientConfig } from "src/format"; + +type TargetsSetPinningArgs = ClientCliArgs & + WriteCliArgs & { + "target-id": string; + enable: boolean; + }; + +export const builder = (yargs: Argv) => + yargs.options({ + ...commonOptions, + ...clientIdOption, + ...targetIdOption, + ...writeOptions, + enable: { + type: "boolean", + demandOption: true, + description: + "Enable or disable certificate pinning for this target (use --no-enable to disable)", + }, + }); + +export const handler: CliCommand["handler"] = async ( + argv, +) => { + const enabled = argv.enable; + + if (!enabled) { + console.warn(pc.bold(pc.red("WARNING: Disabling certificate pinning"))); + } + + const repository = await createRepository(argv); + const config = await requireClientConfig(repository, argv["client-id"]); + const target = requireTargetConfig( + config, + argv["client-id"], + argv["target-id"], + ); + + if (enabled && !target.delivery?.mtls?.certPinning?.spkiHash) { + throw new Error( + `Target '${argv["target-id"]}' has no SPKI hash stored. Run 'targets-set-certificate' first to configure a certificate hash before enabling pinning.`, + ); + } + + const mtls = target.delivery?.mtls ?? { enabled: false }; + const certPinning = mtls.certPinning ?? { enabled: false }; + target.delivery = { + ...target.delivery, + mtls: { + ...mtls, + certPinning: { + ...certPinning, + enabled, + }, + }, + }; + + const result = await repository.putClientConfig( + argv["client-id"], + config, + argv["dry-run"], + ); + console.log( + `Certificate pinning ${enabled ? "enabled" : "disabled"} for target ${argv["target-id"]}`, + ); + console.log(formatClientConfig(result)); +}; + +export const command: CliCommand = { + command: "targets-set-pinning", + describe: "Enable or disable certificate pinning for a callback target", + builder, + handler, +}; + +export async function main(args: string[] = process.argv) { + await runCommand(command, args); +} diff --git a/tools/client-subscriptions-management/src/repository/client-subscriptions.ts b/tools/client-subscriptions-management/src/repository/client-subscriptions.ts index 4a744fc3..04fc266b 100644 --- a/tools/client-subscriptions-management/src/repository/client-subscriptions.ts +++ b/tools/client-subscriptions-management/src/repository/client-subscriptions.ts @@ -131,7 +131,7 @@ export class ClientSubscriptionRepository { const updated: ClientSubscriptionConfiguration = { ...config, subscriptions: config.subscriptions.map( - // eslint-disable-next-line sonarjs/function-return-type -- false positive: complex conditional spread returns are all SubscriptionConfiguration subtypes + // eslint-disable-next-line sonarjs/function-return-type (sub): SubscriptionConfiguration => { if (sub.subscriptionId !== subscriptionId) return sub; if (sub.subscriptionType === "MessageStatus") { diff --git a/tools/client-subscriptions-management/src/repository/s3-applications-map.ts b/tools/client-subscriptions-management/src/repository/s3-applications-map.ts new file mode 100644 index 00000000..f1465280 --- /dev/null +++ b/tools/client-subscriptions-management/src/repository/s3-applications-map.ts @@ -0,0 +1,69 @@ +import { + GetObjectCommand, + PutObjectCommand, + type S3Client, +} from "@aws-sdk/client-s3"; + +export default class S3ApplicationsMapRepository { + constructor( + private readonly client: S3Client, + private readonly bucket: string, + private readonly key: string, + ) {} + + async getApplication(clientId: string): Promise { + try { + const response = await this.client.send( + new GetObjectCommand({ Bucket: this.bucket, Key: this.key }), + ); + const body = await response.Body?.transformToString(); + if (body) { + const map = JSON.parse(body) as Record; + // eslint-disable-next-line security/detect-object-injection + return map[clientId]; + } + } catch (error) { + if (error instanceof Error && error.name !== "NoSuchKey") { + throw error; + } + } + return undefined; + } + + async addApplication( + clientId: string, + applicationId: string, + dryRun = false, + ): Promise> { + let current: Record = {}; + + try { + const response = await this.client.send( + new GetObjectCommand({ Bucket: this.bucket, Key: this.key }), + ); + const body = await response.Body?.transformToString(); + if (body) { + current = JSON.parse(body) as Record; + } + } catch (error) { + if (error instanceof Error && error.name !== "NoSuchKey") { + throw error; + } + } + + const updated = { ...current, [clientId]: applicationId }; + + if (!dryRun) { + await this.client.send( + new PutObjectCommand({ + Bucket: this.bucket, + Key: this.key, + Body: JSON.stringify(updated), + ContentType: "application/json", + }), + ); + } + + return new Map(Object.entries(updated)); + } +} diff --git a/tools/client-subscriptions-management/src/repository/s3.ts b/tools/client-subscriptions-management/src/repository/s3.ts index 75ffde9c..15eef66b 100644 --- a/tools/client-subscriptions-management/src/repository/s3.ts +++ b/tools/client-subscriptions-management/src/repository/s3.ts @@ -12,12 +12,17 @@ export class S3Repository { constructor( private readonly bucketName: string, private readonly s3Client: S3Client, + private readonly keyPrefix = "", ) {} + private prefixedKey(key: string): string { + return `${this.keyPrefix}${key}`; + } + async getObject(key: string): Promise { const params = { Bucket: this.bucketName, - Key: key, + Key: this.prefixedKey(key), }; try { const { Body } = await this.s3Client.send(new GetObjectCommand(params)); @@ -41,7 +46,7 @@ export class S3Repository { ): Promise { const params = { Bucket: this.bucketName, - Key: key, + Key: this.prefixedKey(key), Body: fileData, }; @@ -51,18 +56,19 @@ export class S3Repository { async listObjectKeys(prefix: string): Promise { const keys: string[] = []; let continuationToken: string | undefined; + const fullPrefix = this.prefixedKey(prefix); do { const { Contents, NextContinuationToken } = await this.s3Client.send( new ListObjectsV2Command({ Bucket: this.bucketName, - Prefix: prefix, + Prefix: fullPrefix, ContinuationToken: continuationToken, }), ); for (const obj of Contents ?? []) { if (obj.Key) { - keys.push(obj.Key); + keys.push(obj.Key.slice(this.keyPrefix.length)); } } continuationToken = NextContinuationToken; diff --git a/tools/client-subscriptions-management/src/repository/ssm-applications-map.ts b/tools/client-subscriptions-management/src/repository/ssm-applications-map.ts deleted file mode 100644 index a7edb3f6..00000000 --- a/tools/client-subscriptions-management/src/repository/ssm-applications-map.ts +++ /dev/null @@ -1,78 +0,0 @@ -import { - GetParameterCommand, - PutParameterCommand, - type SSMClient, -} from "@aws-sdk/client-ssm"; - -export default class SsmApplicationsMapRepository { - constructor( - private readonly client: SSMClient, - private readonly parameterName: string, - ) {} - - async getApplication(clientId: string): Promise { - try { - const response = await this.client.send( - new GetParameterCommand({ - Name: this.parameterName, - WithDecryption: true, - }), - ); - if (response.Parameter?.Value) { - const map = JSON.parse(response.Parameter.Value) as Record< - string, - string - >; - // eslint-disable-next-line security/detect-object-injection - return map[clientId]; - } - } catch (error) { - if (error instanceof Error && error.name !== "ParameterNotFound") { - throw error; - } - } - return undefined; - } - - async addApplication( - clientId: string, - applicationId: string, - dryRun = false, - ): Promise> { - let current: Record = {}; - - try { - const response = await this.client.send( - new GetParameterCommand({ - Name: this.parameterName, - WithDecryption: true, - }), - ); - if (response.Parameter?.Value) { - current = JSON.parse(response.Parameter.Value) as Record< - string, - string - >; - } - } catch (error) { - if (error instanceof Error && error.name !== "ParameterNotFound") { - throw error; - } - } - - const updated = { ...current, [clientId]: applicationId }; - - if (!dryRun) { - await this.client.send( - new PutParameterCommand({ - Name: this.parameterName, - Value: JSON.stringify(updated), - Type: "SecureString", - Overwrite: true, - }), - ); - } - - return new Map(Object.entries(updated)); - } -}