diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml new file mode 100644 index 00000000..17725e24 --- /dev/null +++ b/.github/workflows/integration.yml @@ -0,0 +1,80 @@ +# OPTIONAL live AWS integration tests. +# +# This workflow is intentionally isolated from the offline CI (build/lint/unit/ +# e2e). It costs money and uses real credentials, so it NEVER runs on pushes or +# pull requests (especially not fork PRs). It runs only: +# - manually (workflow_dispatch), or +# - on a schedule (weekly), if you enable the cron below. +# +# Credentials are obtained via GitHub OIDC role assumption (no long-lived keys). +# Configure the repo variables/secrets referenced below, or switch to static +# secrets if you prefer (see the commented block). +name: integration + +on: + workflow_dispatch: + inputs: + caching: + description: 'Run the caching tier (billed hourly)' + type: boolean + default: false + # schedule: + # - cron: '0 6 * * 1' # Mondays 06:00 UTC + +# Required for OIDC role assumption. +permissions: + id-token: write + contents: read + +concurrency: + # Never run two live suites against the same account at once. + group: integration-${{ github.ref }} + cancel-in-progress: false + +jobs: + integration: + # Extra guard: only runs from the canonical repo, never a fork. + if: github.repository == 'sid88in/serverless-appsync-plugin' + runs-on: ubuntu-latest + timeout-minutes: 20 + environment: integration # add manual approval / scoped secrets here + steps: + - uses: actions/checkout@v6 + with: + # OIDC-only job that never pushes git; don't persist GITHUB_TOKEN + # into .git/config, since npm ci + the suite run third-party code. + persist-credentials: false + + - uses: actions/setup-node@v6 + with: + node-version: 22 + cache: npm + + - run: npm ci + + - name: Configure AWS credentials (OIDC) + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ secrets.INTEGRATION_AWS_ROLE_ARN }} + aws-region: ${{ vars.INTEGRATION_AWS_REGION || 'us-west-2' }} + + # Static-secrets alternative (use instead of the OIDC step above): + # - uses: aws-actions/configure-aws-credentials@v4 + # with: + # aws-access-key-id: ${{ secrets.INTEGRATION_AWS_ACCESS_KEY_ID }} + # aws-secret-access-key: ${{ secrets.INTEGRATION_AWS_SECRET_ACCESS_KEY }} + # aws-region: ${{ vars.INTEGRATION_AWS_REGION || 'us-west-2' }} + + - name: Run integration tests + env: + APPSYNC_PLUGIN_INTEGRATION: '1' + APPSYNC_PLUGIN_INTEGRATION_REGION: ${{ vars.INTEGRATION_AWS_REGION || 'us-west-2' }} + APPSYNC_PLUGIN_INTEGRATION_CACHING: ${{ inputs.caching && '1' || '0' }} + run: npm run test:integration + + - name: Sweep leaked resources (always) + if: always() + env: + APPSYNC_PLUGIN_INTEGRATION: '1' + APPSYNC_PLUGIN_INTEGRATION_REGION: ${{ vars.INTEGRATION_AWS_REGION || 'us-west-2' }} + run: npm run test:integration:sweep diff --git a/doc/integration-tests.md b/doc/integration-tests.md new file mode 100644 index 00000000..f66d6b72 --- /dev/null +++ b/doc/integration-tests.md @@ -0,0 +1,283 @@ +# Live AWS integration tests + +This is an **opt-in** test suite that exercises the plugin's live AWS code +paths (the ones the [AWS SDK v3 migration in #686](https://github.com/sid88in/serverless-appsync-plugin/pull/686) +most affects) against a **real AWS account**. It is complementary to: + +- `npm test` — unit tests (`src/__tests__`), no AWS. +- `npm run test:e2e` — offline CloudFormation-synthesis tests (`e2e/`), no AWS. + +Unlike those, the integration suite **costs money and needs credentials**, so it +never runs in the default CI or as part of `npm test` / `npm run test:e2e` / +`npm run test:all`. It is gated behind the `APPSYNC_PLUGIN_INTEGRATION` +environment variable and, when that is unset, every suite resolves to +`describe.skip` and the run exits green. + +## What it proves + +Each scenario maps to a live command and the SDK call it validates: + +| Tier | Command | Live SDK call(s) | Notes | +| ---- | --------------------------------- | ---------------------------------------------------------------------------------- | ------------------------------ | +| A | `appsync evaluate --template` | `EvaluateMappingTemplate` | no deploy | +| A | `appsync evaluate --type/--field` | `EvaluateCode` | no deploy | +| B | `serverless info` | `ListApiKeys`, `GetGraphqlApi`, `DescribeStackResources` | get-api-keys | +| B | `appsync get-introspection` | `GetIntrospectionSchema` | | +| B | `appsync env set` / `env get` | `Put`/`GetGraphqlApiEnvironmentVariables` | | +| B | `appsync logs` | `FilterLogEvents` (CloudWatch Logs) | log group from fixture logging | +| B | **credential/region proof** | deploy + live read | the headline #686 test | +| C | `appsync flush-cache` | `FlushApiCache` | caching billed hourly | +| D | `appsync domain create` | `ListCertificates` (ACM, **us-east-1 pin**) + `CreateDomainName` | | +| D | `appsync domain assoc` | `GetApiAssociation` (+ `NotFoundException` path) + `AssociateApi` | | +| D | `appsync domain create-record` | `ListHostedZonesByName` + `ChangeResourceRecordSets` + `GetChange` (poll → INSYNC) | | + +The **credential/region proof** is the most valuable test: it deploys with an +explicit `--region` (and optional `--aws-profile`), asserts the API actually +landed in that region (by reading the stack's AppSync API ARN), confirms a live +command pointed at that region succeeds, and confirms the same command pointed +at a **different** region fails to find the API — demonstrating that the live +commands honor the region resolved from the Serverless provider, not the bare +default credential chain. + +## Tiers and gating + +| Tier | Switch (in addition to `APPSYNC_PLUGIN_INTEGRATION=1`) | Cost profile | +| ------------------ | --------------------------------------------------------------------------------- | --------------------------------------------- | +| A — evaluate | none | negligible (a few AppSync requests) | +| B — minimal deploy | none | cents (no hourly charge; 1-day log retention) | +| C — caching | `APPSYNC_PLUGIN_INTEGRATION_CACHING=1` | **hourly** caching instance while it exists | +| D — custom domain | `APPSYNC_PLUGIN_INTEGRATION_DOMAIN` + `APPSYNC_PLUGIN_INTEGRATION_HOSTED_ZONE_ID` | minimal (reuses existing zone + cert) | + +Tiers are independently skippable: with only credentials set you get A and B; +caching and domain stay skipped until you opt in. + +## Running it + +```bash +# Tiers A + B (cheapest useful run) +APPSYNC_PLUGIN_INTEGRATION=1 \ +APPSYNC_PLUGIN_INTEGRATION_REGION=us-west-2 \ +AWS_PROFILE=my-sandbox \ +npm run test:integration + +# Add the caching tier +APPSYNC_PLUGIN_INTEGRATION=1 \ +APPSYNC_PLUGIN_INTEGRATION_REGION=us-west-2 \ +APPSYNC_PLUGIN_INTEGRATION_CACHING=1 \ +AWS_PROFILE=my-sandbox \ +npm run test:integration + +# Full credential/region proof with an explicit profile whose default region +# differs from the test region +APPSYNC_PLUGIN_INTEGRATION=1 \ +APPSYNC_PLUGIN_INTEGRATION_REGION=us-west-2 \ +APPSYNC_PLUGIN_INTEGRATION_PROFILE=my-sandbox \ +npm run test:integration +``` + +### Environment variables + +| Variable | Required | Description | +| ------------------------------------------------ | -------- | ----------------------------------------------------------------------------------------------------------------------------------------- | +| `APPSYNC_PLUGIN_INTEGRATION` | yes | Master switch; must be `1`. | +| `APPSYNC_PLUGIN_INTEGRATION_REGION` | no | Test region (default `us-west-2`). Choose something other than `us-east-1` so the ACM us-east-1 pin and the region proof are meaningful. | +| `APPSYNC_PLUGIN_INTEGRATION_PROFILE` | no | Named profile for the deploy/commands; also enables the profile half of the credential proof. Falls back to the default credential chain. | +| `APPSYNC_PLUGIN_INTEGRATION_OTHER_REGION` | no | Region used by the negative half of the region proof (default: the opposite of the test region). | +| `APPSYNC_PLUGIN_INTEGRATION_EXPECTED_ACCOUNT_ID` | no | If set, the profile proof asserts the deployed API's account matches it. | +| `APPSYNC_PLUGIN_INTEGRATION_CACHING` | no | `1` to run the caching tier. | +| `APPSYNC_PLUGIN_INTEGRATION_DOMAIN` | no | Domain name for the custom-domain tier (e.g. `it.example.com`). | +| `APPSYNC_PLUGIN_INTEGRATION_HOSTED_ZONE_ID` | no | Route53 hosted zone id for that domain. | +| `APPSYNC_PLUGIN_INTEGRATION_CERT_ARN` | no | ISSUED ACM cert ARN (us-east-1). If omitted, the plugin discovers a matching cert via `ListCertificates`. | +| `SERVERLESS_BIN` | no | Path to a Serverless binary (e.g. a v4 install). Defaults to the repo's v3. | + +Standard AWS credential variables (`AWS_PROFILE`, `AWS_ACCESS_KEY_ID`, OIDC +`AWS_ROLE_ARN`, …) are honored as usual. + +### Typecheck only + +```bash +npm run test:integration:typecheck # tsc -p tsconfig.integration.json +``` + +## Teardown and leaked-resource recovery + +Reliable teardown is the suite's top priority: + +- Every resource is named with a unique per-run id (`appsync-plugin-it--`) + and tagged `appsync-plugin-integration: `. +- Deploy tiers tear down with `serverless remove` in `afterAll` (runs even on + failure). The custom-domain tier additionally deletes its non-CloudFormation + resources (Route53 record → API association → domain name) in reverse order. + The ACM certificate is **reused, never created, and never deleted**. +- If a run is interrupted, recover leaks with the standalone sweeper: + +```bash +APPSYNC_PLUGIN_INTEGRATION=1 \ +APPSYNC_PLUGIN_INTEGRATION_REGION=us-west-2 \ +AWS_PROFILE=my-sandbox \ +[APPSYNC_PLUGIN_INTEGRATION_DOMAIN=it.example.com] \ +npm run test:integration:sweep +``` + +The sweeper deletes AppSync APIs tagged by the suite, CloudFormation stacks +named with the run-id prefix, and (if a domain is configured) a leaked custom +domain name. It is idempotent and safe to re-run. CloudFormation stack deletes +are asynchronous — verify completion in the console. + +## Least-privilege IAM policy + +The actions actually used by the suite are below. Note that a `serverless +deploy` is itself a CloudFormation operation that creates an AppSync API, an +API key, a CloudWatch log group + logging role, and an S3 deployment bucket; +fully constraining a deploy role is involved, so in a throwaway sandbox account +many teams simply use a broader deploy role. The policy below is the scoped +target. + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "AppSync", + "Effect": "Allow", + "Action": [ + "appsync:CreateGraphqlApi", + "appsync:DeleteGraphqlApi", + "appsync:UpdateGraphqlApi", + "appsync:GetGraphqlApi", + "appsync:ListGraphqlApis", + "appsync:StartSchemaCreation", + "appsync:GetSchemaCreationStatus", + "appsync:GetIntrospectionSchema", + "appsync:CreateApiKey", + "appsync:DeleteApiKey", + "appsync:ListApiKeys", + "appsync:CreateDataSource", + "appsync:DeleteDataSource", + "appsync:UpdateDataSource", + "appsync:CreateResolver", + "appsync:DeleteResolver", + "appsync:CreateFunction", + "appsync:DeleteFunction", + "appsync:FlushApiCache", + "appsync:CreateApiCache", + "appsync:DeleteApiCache", + "appsync:EvaluateCode", + "appsync:EvaluateMappingTemplate", + "appsync:GetGraphqlApiEnvironmentVariables", + "appsync:PutGraphqlApiEnvironmentVariables", + "appsync:TagResource", + "appsync:UntagResource", + "appsync:ListTagsForResource", + "appsync:CreateDomainName", + "appsync:DeleteDomainName", + "appsync:GetDomainName", + "appsync:ListDomainNames", + "appsync:AssociateApi", + "appsync:DisassociateApi", + "appsync:GetApiAssociation" + ], + "Resource": "*" + }, + { + "Sid": "CloudFormation", + "Effect": "Allow", + "Action": [ + "cloudformation:CreateStack", + "cloudformation:UpdateStack", + "cloudformation:DeleteStack", + "cloudformation:DescribeStacks", + "cloudformation:DescribeStackResources", + "cloudformation:DescribeStackEvents", + "cloudformation:GetTemplate", + "cloudformation:ListStacks", + "cloudformation:ValidateTemplate", + "cloudformation:CreateChangeSet", + "cloudformation:DescribeChangeSet", + "cloudformation:ExecuteChangeSet", + "cloudformation:DeleteChangeSet" + ], + "Resource": "*" + }, + { + "Sid": "Logs", + "Effect": "Allow", + "Action": [ + "logs:FilterLogEvents", + "logs:CreateLogGroup", + "logs:DeleteLogGroup", + "logs:PutRetentionPolicy", + "logs:DescribeLogGroups", + "logs:TagResource" + ], + "Resource": "*" + }, + { + "Sid": "IamForDeploy", + "Effect": "Allow", + "Action": [ + "iam:CreateRole", + "iam:DeleteRole", + "iam:GetRole", + "iam:PassRole", + "iam:PutRolePolicy", + "iam:DeleteRolePolicy", + "iam:AttachRolePolicy", + "iam:DetachRolePolicy", + "iam:TagRole" + ], + "Resource": "*" + }, + { + "Sid": "DeploymentBucket", + "Effect": "Allow", + "Action": [ + "s3:CreateBucket", + "s3:DeleteBucket", + "s3:ListBucket", + "s3:GetObject", + "s3:PutObject", + "s3:DeleteObject", + "s3:PutBucketPolicy", + "s3:GetBucketPolicy", + "s3:PutBucketTagging", + "s3:GetEncryptionConfiguration", + "s3:PutEncryptionConfiguration" + ], + "Resource": "*" + }, + { + "Sid": "DomainTierOnly", + "Effect": "Allow", + "Action": [ + "acm:ListCertificates", + "route53:ListHostedZonesByName", + "route53:ChangeResourceRecordSets", + "route53:GetChange" + ], + "Resource": "*" + }, + { + "Sid": "Identity", + "Effect": "Allow", + "Action": ["sts:GetCallerIdentity"], + "Resource": "*" + } + ] +} +``` + +If you skip the custom-domain tier (the default), the `DomainTierOnly` statement +and the domain-related AppSync actions are unnecessary. + +## Serverless v4 note + +The suite spawns the `serverless` binary from `node_modules` (override with +`SERVERLESS_BIN`), so it can be pointed at a Serverless v4 install. Two caveats, +unverified here: + +- v4 may require `SERVERLESS_ACCESS_KEY` / a license and suppression of login + prompts (the wrappers already disable telemetry and interactive setup). +- v4 resolves credentials as SDK v3 objects, so the v2-style `getPromise` / + `expireTime` branch in the plugin's `resolveCredentials` will not fire (it + degrades gracefully). Re-confirm the credential/region proof under v4. diff --git a/integration/caching.integration.test.ts b/integration/caching.integration.test.ts new file mode 100644 index 00000000..8d8fc4ed --- /dev/null +++ b/integration/caching.integration.test.ts @@ -0,0 +1,42 @@ +/** + * Tier C (caching): caching is billed by the hour, so it is gated behind its + * own flag (APPSYNC_PLUGIN_INTEGRATION_CACHING=1) on top of the master switch. + * Deploys an API with FULL_REQUEST_CACHING, flushes the cache, then removes. + * + * - appsync flush-cache -> FlushApiCacheCommand + */ +import { cachingDescribe, integrationConfig } from './helpers/gate'; +import { generateRunId } from './helpers/run-id'; +import { prepareProject, PreparedProject } from './helpers/project'; +import { deploy, remove, appsync } from './helpers/sls'; + +const DEPLOY_TIMEOUT = 1_200_000; // caching clusters take longer to provision + +cachingDescribe('integration / Tier C — caching (flush-cache)', () => { + let project: PreparedProject; + + beforeAll(async () => { + project = prepareProject({ + runId: generateRunId(), + region: integrationConfig.region, + caching: true, + }); + deploy({ cwd: project.dir }); + }, DEPLOY_TIMEOUT); + + afterAll(async () => { + if (project) { + try { + remove({ cwd: project.dir }); + } catch { + // sweeper backstop + } + project.cleanup(); + } + }, DEPLOY_TIMEOUT); + + it('flushes the cache (FlushApiCache)', () => { + const { stdout } = appsync(['flush-cache'], { cwd: project.dir }); + expect(stdout.toLowerCase()).toContain('cache flushed'); + }); +}); diff --git a/integration/deploy.integration.test.ts b/integration/deploy.integration.test.ts new file mode 100644 index 00000000..ddf11692 --- /dev/null +++ b/integration/deploy.integration.test.ts @@ -0,0 +1,142 @@ +/** + * Tier B (minimal deploy): a trivial API_KEY + NONE-datasource API, deployed + * once and torn down with `serverless remove`. Exercises the live commands that + * need a real deployed API, plus the headline credential/region proof for #686. + * + * - serverless info -> ListApiKeys (+ GetGraphqlApi, DescribeStackResources) + * - appsync get-introspection -> GetIntrospectionSchema + * - appsync env set / env get -> Put/GetGraphqlApiEnvironmentVariables + * - appsync logs -> FilterLogEvents (CloudWatch Logs) + * - credential/region proof -> live command honors provider region/profile + * + * Teardown runs in afterAll even if the body throws; everything is also tagged + * with the run id so the standalone sweeper can recover a leak. + */ +import { + integrationDescribe, + integrationConfig, + profileProofEnabled, +} from './helpers/gate'; +import { generateRunId } from './helpers/run-id'; +import { prepareProject, PreparedProject } from './helpers/project'; +import { + deploy, + remove, + info, + appsync, + runServerless, + SlsCommandError, +} from './helpers/sls'; +import { extractApiArn, parseArn } from './helpers/aws'; + +const DEPLOY_TIMEOUT = 900_000; // 15 min: a fresh AppSync API + key + +integrationDescribe('integration / Tier B — deploy + live commands', () => { + let project: PreparedProject; + let runId: string; + + beforeAll(async () => { + runId = generateRunId(); + project = prepareProject({ runId, region: integrationConfig.region }); + deploy({ cwd: project.dir }); + }, DEPLOY_TIMEOUT); + + afterAll(async () => { + // Best-effort teardown; never let cleanup failure mask a test result. + if (project) { + try { + remove({ cwd: project.dir }); + } catch { + // Sweeper will catch anything left behind (resources are tagged). + } + project.cleanup(); + } + }, DEPLOY_TIMEOUT); + + it('lists the API key via `serverless info` (ListApiKeys)', () => { + const { stdout } = info({ cwd: project.dir }); + expect(stdout.toLowerCase()).toContain('appsync api keys'); + }); + + it('fetches the introspection schema (GetIntrospectionSchema)', () => { + const { stdout } = appsync(['get-introspection', '--format', 'SDL'], { + cwd: project.dir, + }); + expect(stdout).toContain('type Query'); + }); + + it('sets and gets an env var (Put/GetGraphqlApiEnvironmentVariables)', () => { + appsync(['env', 'set', '--key', 'FOO', '--value', 'bar'], { + cwd: project.dir, + }); + const { stdout } = appsync(['env', 'get'], { cwd: project.dir }); + expect(stdout).toContain('FOO=bar'); + }); + + it('reads logs without error (FilterLogEvents)', () => { + // The log group exists (logging is enabled in the fixture) but is likely + // empty; the call must still succeed. + expect(() => appsync(['logs'], { cwd: project.dir })).not.toThrow(); + }); + + describe('credential/region proof (#686)', () => { + // Read the deployed API ARN from `serverless info --verbose` (the fixture + // exposes it as the IntegrationApiArn stack output). This runs in the + // serverless child process, so it exercises real credential/region + // resolution without an in-VM SDK client. + const readApiArn = (): string => { + const { stdout } = runServerless(['info', '--verbose'], { + cwd: project.dir, + }); + const arn = extractApiArn(stdout); + if (!arn) { + throw new Error( + `Could not find an AppSync API ARN in info output:\n${stdout}`, + ); + } + return arn; + }; + + it('deployed the API under the configured region, not the default chain', () => { + const arn = readApiArn(); + expect(parseArn(arn).region).toBe(integrationConfig.region); + }); + + it('a live command succeeds when pointed at the configured region', () => { + // Uses the same resolved provider region/profile as the deploy. + expect(() => appsync(['env', 'get'], { cwd: project.dir })).not.toThrow(); + }); + + it('the same live command fails when pointed at a different region', () => { + // Proves the live command's region comes from the resolved provider + // (the --region we pass), not a hardcoded value or the default chain: + // the API does not exist in `otherRegion`, so the command errors. + let thrown: unknown; + try { + appsync(['env', 'get'], { + cwd: project.dir, + region: integrationConfig.otherRegion, + }); + } catch (err) { + thrown = err; + } + expect(thrown).toBeDefined(); + const e = thrown as Partial & { message?: string }; + const text = `${e.stdout ?? ''}\n${e.stderr ?? ''}\n${e.message ?? ''}`; + expect(text).toMatch( + /does not exist|not found|no api|could not|unable|no stack/i, + ); + }); + + (profileProofEnabled ? it : it.skip)( + 'deployed under the account tied to the configured profile', + () => { + const { accountId } = parseArn(readApiArn()); + expect(accountId).toMatch(/^\d{12}$/); + if (integrationConfig.expectedAccountId) { + expect(accountId).toBe(integrationConfig.expectedAccountId); + } + }, + ); + }); +}); diff --git a/integration/domain.integration.test.ts b/integration/domain.integration.test.ts new file mode 100644 index 00000000..2469fc56 --- /dev/null +++ b/integration/domain.integration.test.ts @@ -0,0 +1,105 @@ +/** + * Tier D (custom domain — heaviest): exercises the domain/Route53/ACM code + * paths. Gated behind a provided domain + hosted zone (and, by default, left + * fully skipped). When enabled it requires: + * + * APPSYNC_PLUGIN_INTEGRATION_DOMAIN e.g. it.example.com + * APPSYNC_PLUGIN_INTEGRATION_HOSTED_ZONE_ID the Route53 zone for that domain + * APPSYNC_PLUGIN_INTEGRATION_CERT_ARN (optional) an ISSUED us-east-1 + * ACM cert; if omitted, the plugin + * discovers one via ListCertificates + * + * Covers (in order, with reverse-order teardown): + * - appsync domain create -> ListCertificates (ACM, us-east-1 pin) + CreateDomainName + * - appsync domain assoc -> GetApiAssociation (incl. NotFoundException path) + AssociateApi + * - appsync domain create-record -> GetDomainName + ListHostedZonesByName + ChangeResourceRecordSets + GetChange (poll to INSYNC) + * - appsync domain delete-record -> ChangeResourceRecordSets (DELETE) + GetChange + * - appsync domain disassoc -> DisassociateApi + * - appsync domain delete -> DeleteDomainName + * + * The ACM certificate is reused, never created, so it is never deleted. + */ +import { domainDescribe, integrationConfig } from './helpers/gate'; +import { generateRunId } from './helpers/run-id'; +import { prepareProject, PreparedProject } from './helpers/project'; +import { deploy, remove, appsync } from './helpers/sls'; + +const DEPLOY_TIMEOUT = 900_000; +const DOMAIN_TIMEOUT = 600_000; // create-record polls GetChange to INSYNC + +domainDescribe('integration / Tier D — custom domain + Route53 + ACM', () => { + let project: PreparedProject; + const domainName = integrationConfig.domain.name as string; + + beforeAll(async () => { + project = prepareProject({ + runId: generateRunId(), + region: integrationConfig.region, + domain: { + name: domainName, + hostedZoneId: integrationConfig.domain.hostedZoneId, + certificateArn: integrationConfig.domain.certificateArn, + }, + }); + deploy({ cwd: project.dir }); + }, DEPLOY_TIMEOUT); + + afterAll(async () => { + if (!project) { + return; + } + // Reverse-order teardown of the non-CloudFormation domain resources, then + // the stack. Each step is best-effort; the sweeper is the final backstop. + for (const step of [ + () => appsync(['domain', 'delete-record', '--yes'], { cwd: project.dir }), + () => appsync(['domain', 'disassoc', '--yes'], { cwd: project.dir }), + () => + appsync(['domain', 'delete', '--yes', '--quiet'], { cwd: project.dir }), + ]) { + try { + step(); + } catch { + // continue + } + } + try { + remove({ cwd: project.dir }); + } catch { + // sweeper backstop (resources are tagged; run test:integration:sweep) + } + project.cleanup(); + }, DEPLOY_TIMEOUT); + + it( + 'creates the domain (ListCertificates us-east-1 + CreateDomainName)', + () => { + const { stdout } = appsync(['domain', 'create', '--yes'], { + cwd: project.dir, + }); + expect(stdout.toLowerCase()).toContain('created successfully'); + }, + DOMAIN_TIMEOUT, + ); + + it( + 'associates the API with the domain (AssociateApi)', + () => { + const { stdout } = appsync(['domain', 'assoc', '--yes'], { + cwd: project.dir, + }); + expect(stdout.toLowerCase()).toContain('associated'); + }, + DOMAIN_TIMEOUT, + ); + + it( + 'creates the Route53 alias record (ChangeResourceRecordSets + GetChange poll)', + () => { + const { stdout } = appsync(['domain', 'create-record', '--yes'], { + cwd: project.dir, + }); + expect(stdout.toLowerCase()).toContain('record created'); + }, + DOMAIN_TIMEOUT, + ); +}); diff --git a/integration/eval.integration.test.ts b/integration/eval.integration.test.ts new file mode 100644 index 00000000..360cc760 --- /dev/null +++ b/integration/eval.integration.test.ts @@ -0,0 +1,64 @@ +/** + * Tier A (cheapest, no deploy): pure live AppSync evaluate calls. + * + * - `appsync evaluate --template ...` -> EvaluateMappingTemplateCommand + * - `appsync evaluate --type/--field` -> EvaluateCodeCommand + * + * Neither command needs a deployed API, so this tier creates no AWS resources + * and requires no teardown. It only needs resolvable credentials + a region, + * which also makes it the smallest possible exercise of the v3 client + + * provider credential/region resolution introduced in #686. + */ +import * as fs from 'fs'; +import * as path from 'path'; +import { integrationDescribe, integrationConfig } from './helpers/gate'; +import { generateRunId } from './helpers/run-id'; +import { prepareProject, PreparedProject } from './helpers/project'; +import { appsync } from './helpers/sls'; + +integrationDescribe('integration / Tier A — evaluate (no deploy)', () => { + let project: PreparedProject; + const context = JSON.stringify({ arguments: { name: 'integration' } }); + + beforeAll(() => { + project = prepareProject({ + runId: generateRunId(), + region: integrationConfig.region, + }); + }); + + afterAll(() => { + project?.cleanup(); + }); + + it('evaluates a VTL mapping template (EvaluateMappingTemplate)', () => { + const templatePath = path.join(project.dir, 'template.vtl'); + fs.writeFileSync(templatePath, '$util.toJson($context.arguments)'); + + const { stdout } = appsync( + ['evaluate', '--template', templatePath, '--context', context], + { cwd: project.dir }, + ); + + expect(stdout).toContain('integration'); + }); + + it('evaluates a JS resolver (EvaluateCode)', () => { + const { stdout } = appsync( + [ + 'evaluate', + '--type', + 'Query', + '--field', + 'hello', + '--function', + 'request', + '--context', + context, + ], + { cwd: project.dir }, + ); + + expect(stdout).toContain('integration'); + }); +}); diff --git a/integration/helpers/aws.ts b/integration/helpers/aws.ts new file mode 100644 index 00000000..49c885b0 --- /dev/null +++ b/integration/helpers/aws.ts @@ -0,0 +1,195 @@ +import { + AppSyncClient, + ListGraphqlApisCommand, + ListTagsForResourceCommand, + DeleteGraphqlApiCommand, + ListDomainNamesCommand, + GetApiAssociationCommand, + DisassociateApiCommand, + DeleteDomainNameCommand, +} from '@aws-sdk/client-appsync'; +import { + CloudFormationClient, + ListStacksCommand, + DeleteStackCommand, +} from '@aws-sdk/client-cloudformation'; +import { fromIni, fromNodeProviderChain } from '@aws-sdk/credential-providers'; +import type { AwsCredentialIdentityProvider } from '@aws-sdk/types'; +import { integrationConfig } from './config'; +import { TAG_KEY, RUN_ID_PREFIX } from './run-id'; + +/** + * Resolve credentials the same way the suite asks the plugin to: from the named + * profile when one is configured, otherwise the standard default chain. Used by + * test assertions and the sweeper (NOT by the plugin under test). + */ +export function credentials(): AwsCredentialIdentityProvider { + return integrationConfig.profile + ? fromIni({ profile: integrationConfig.profile }) + : fromNodeProviderChain(); +} + +export function appSyncClient(region: string): AppSyncClient { + return new AppSyncClient({ region, credentials: credentials() }); +} + +export function cfnClient(region: string): CloudFormationClient { + return new CloudFormationClient({ region, credentials: credentials() }); +} + +export type ParsedArn = { + partition: string; + service: string; + region: string; + accountId: string; + resource: string; +}; + +/** Parse a standard ARN into its components. */ +export function parseArn(arn: string): ParsedArn { + const [, partition, service, region, accountId, ...rest] = arn.split(':'); + return { + partition, + service, + region, + accountId, + resource: rest.join(':'), + }; +} + +/** + * Extract the AppSync API ARN from `serverless info --verbose` output. The + * fixture exposes it as the `IntegrationApiArn` stack output, so the + * credential/region proof can read where the API landed without building an + * AWS SDK client inside the Jest VM (the v3 credential provider chain uses a + * dynamic import() that Jest's default VM rejects). + */ +export function extractApiArn(infoOutput: string): string | undefined { + const match = infoOutput.match( + /arn:aws:appsync:[a-z0-9-]+:\d{12}:apis\/[a-zA-Z0-9]+/, + ); + return match?.[0]; +} + +// --------------------------------------------------------------------------- +// Sweeper building blocks (also reused by integration/sweeper.ts) +// --------------------------------------------------------------------------- + +export type LeakedApi = { apiId: string; arn: string; name?: string }; + +/** Find AppSync APIs tagged by this suite (leaked from interrupted runs). */ +export async function findLeakedApis(region: string): Promise { + const client = appSyncClient(region); + const leaked: LeakedApi[] = []; + let nextToken: string | undefined; + + do { + const { graphqlApis, nextToken: next } = await client.send( + new ListGraphqlApisCommand({ nextToken, maxResults: 25 }), + ); + for (const api of graphqlApis ?? []) { + if (!api.arn || !api.apiId) { + continue; + } + const { tags } = await client.send( + new ListTagsForResourceCommand({ resourceArn: api.arn }), + ); + const tagValue = tags?.[TAG_KEY]; + if (tagValue && tagValue.startsWith(RUN_ID_PREFIX)) { + leaked.push({ apiId: api.apiId, arn: api.arn, name: api.name }); + } + } + nextToken = next; + } while (nextToken); + + return leaked; +} + +export async function deleteApi(region: string, apiId: string): Promise { + await appSyncClient(region).send(new DeleteGraphqlApiCommand({ apiId })); +} + +/** Find CloudFormation stacks whose name starts with the run-id prefix. */ +export async function findLeakedStacks(region: string): Promise { + const client = cfnClient(region); + const names: string[] = []; + let nextToken: string | undefined; + + do { + const { StackSummaries, NextToken } = await client.send( + new ListStacksCommand({ + NextToken: nextToken, + StackStatusFilter: [ + 'CREATE_COMPLETE', + 'UPDATE_COMPLETE', + 'ROLLBACK_COMPLETE', + 'UPDATE_ROLLBACK_COMPLETE', + 'CREATE_FAILED', + 'ROLLBACK_FAILED', + ], + }), + ); + for (const s of StackSummaries ?? []) { + if (s.StackName?.startsWith(RUN_ID_PREFIX)) { + names.push(s.StackName); + } + } + nextToken = NextToken; + } while (nextToken); + + return names; +} + +export async function deleteStack( + region: string, + stackName: string, +): Promise { + await cfnClient(region).send( + new DeleteStackCommand({ StackName: stackName }), + ); +} + +/** + * Tear down a custom-domain name created outside CloudFormation (Tier D): + * disassociate any API, then delete the domain name. The ACM certificate is + * reused, never created, so it is intentionally left untouched. + */ +export async function cleanupDomainName( + region: string, + domainName: string, +): Promise { + const client = appSyncClient(region); + + // Paginate: a domain beyond the first page would otherwise be missed, + // silently skipping cleanup and leaking it. + let exists = false; + let nextToken: string | undefined; + do { + const { domainNameConfigs, nextToken: next } = await client.send( + new ListDomainNamesCommand({ maxResults: 25, nextToken }), + ); + if (domainNameConfigs?.some((d) => d.domainName === domainName)) { + exists = true; + break; + } + nextToken = next; + } while (nextToken); + + if (!exists) { + return; + } + try { + await client.send(new GetApiAssociationCommand({ domainName })); + await client.send(new DisassociateApiCommand({ domainName })); + } catch (err) { + // NotFoundException => no association to remove. Re-throw anything else + // (permissions, throttling, ...) so real failures surface rather than + // leaving the domain leaked. + if (!(err instanceof Error) || err.name !== 'NotFoundException') { + throw err; + } + } + await client.send(new DeleteDomainNameCommand({ domainName })); +} + +export const config = integrationConfig; diff --git a/integration/helpers/config.ts b/integration/helpers/config.ts new file mode 100644 index 00000000..654a440f --- /dev/null +++ b/integration/helpers/config.ts @@ -0,0 +1,73 @@ +/** + * Pure, Jest-free configuration for the integration suite. + * + * This module parses the environment only; it references no Jest globals, so it + * is safe to import from the standalone sweeper (which runs under ts-node, not + * Jest). The `describe`/`describe.skip` gating lives in ./gate, which is only + * ever loaded by Jest test files. + */ + +const DEFAULT_REGION = 'us-west-2'; + +const flag = (name: string): boolean => process.env[name] === '1'; + +export const masterEnabled = flag('APPSYNC_PLUGIN_INTEGRATION'); + +const region = + process.env.APPSYNC_PLUGIN_INTEGRATION_REGION || + process.env.AWS_REGION || + process.env.AWS_DEFAULT_REGION || + DEFAULT_REGION; + +const profile = process.env.APPSYNC_PLUGIN_INTEGRATION_PROFILE; + +/** + * Best-effort, side-effect-free check that *some* credential source is + * configured. We deliberately do not make a network call here: the goal is only + * to skip (rather than hard-fail) when the suite is switched on in an + * environment that obviously has no credentials. + */ +const hasCredentialSignal = Boolean( + profile || + process.env.AWS_PROFILE || + process.env.AWS_ACCESS_KEY_ID || + process.env.AWS_SESSION_TOKEN || + process.env.AWS_ROLE_ARN || + process.env.AWS_WEB_IDENTITY_TOKEN_FILE || + process.env.AWS_CONTAINER_CREDENTIALS_RELATIVE_URI || + process.env.AWS_CONTAINER_CREDENTIALS_FULL_URI, +); + +export const integrationConfig = { + enabled: masterEnabled && hasCredentialSignal, + region, + profile, + /** A region intentionally different from `region`, used by the + * credential/region proof to show a live command fails to find the API when + * pointed at the wrong region. */ + otherRegion: + process.env.APPSYNC_PLUGIN_INTEGRATION_OTHER_REGION || + (region === 'us-east-1' ? 'us-west-2' : 'us-east-1'), + expectedAccountId: process.env.APPSYNC_PLUGIN_INTEGRATION_EXPECTED_ACCOUNT_ID, + caching: flag('APPSYNC_PLUGIN_INTEGRATION_CACHING'), + domain: { + name: process.env.APPSYNC_PLUGIN_INTEGRATION_DOMAIN, + hostedZoneId: process.env.APPSYNC_PLUGIN_INTEGRATION_HOSTED_ZONE_ID, + certificateArn: process.env.APPSYNC_PLUGIN_INTEGRATION_CERT_ARN, + }, +}; + +/** Whether the credential signal was missing despite the master switch. */ +export const missingCredentialSignal = masterEnabled && !hasCredentialSignal; + +export const cachingEnabled = + integrationConfig.enabled && integrationConfig.caching; + +export const domainEnabled = + integrationConfig.enabled && + Boolean(integrationConfig.domain.name) && + Boolean(integrationConfig.domain.hostedZoneId); + +/** True only when a named profile is provided, enabling the profile half of + * the credential/region proof. */ +export const profileProofEnabled = Boolean(integrationConfig.profile); diff --git a/integration/helpers/gate.ts b/integration/helpers/gate.ts new file mode 100644 index 00000000..3fe482a7 --- /dev/null +++ b/integration/helpers/gate.ts @@ -0,0 +1,62 @@ +/** + * Opt-in gating for the live AWS integration suite (Jest layer). + * + * These tests cost money and require real AWS credentials, so they must never + * run as part of the default `npm test` / `npm run test:e2e` jobs or in normal + * CI. The master switch is `APPSYNC_PLUGIN_INTEGRATION=1`; without it (and + * without a usable region + credential signal) every suite resolves to + * `describe.skip` and the run exits green with everything pending. + * + * Environment parsing lives in ./config (Jest-free, importable by the sweeper); + * this module adds the `describe`/`describe.skip` wrappers and therefore must + * only be imported from Jest test files. + */ +import { + integrationConfig, + masterEnabled, + missingCredentialSignal, + cachingEnabled, + domainEnabled, + profileProofEnabled, +} from './config'; + +export { integrationConfig, profileProofEnabled }; + +/** + * `describe` when enabled, otherwise `describe.skip`. The reason is logged once + * when the suite was explicitly requested (so a misconfigured environment is + * easy to diagnose), while the default switched-off case stays quiet. + */ +function gatedDescribe( + enabled: boolean, + reason: string, + warn = false, +): jest.Describe { + if (enabled) { + return describe; + } + if (warn && masterEnabled) { + // eslint-disable-next-line no-console + console.warn(`[integration] skipping: ${reason}`); + } + return describe.skip; +} + +export const integrationDescribe: jest.Describe = gatedDescribe( + integrationConfig.enabled, + missingCredentialSignal + ? 'no AWS credential signal found in the environment' + : 'APPSYNC_PLUGIN_INTEGRATION is not set to 1', + true, +); + +export const cachingDescribe: jest.Describe = gatedDescribe( + cachingEnabled, + 'set APPSYNC_PLUGIN_INTEGRATION_CACHING=1 to run the caching tier', +); + +export const domainDescribe: jest.Describe = gatedDescribe( + domainEnabled, + 'set APPSYNC_PLUGIN_INTEGRATION_DOMAIN and ' + + 'APPSYNC_PLUGIN_INTEGRATION_HOSTED_ZONE_ID to run the custom-domain tier', +); diff --git a/integration/helpers/project.ts b/integration/helpers/project.ts new file mode 100644 index 00000000..3c614f5c --- /dev/null +++ b/integration/helpers/project.ts @@ -0,0 +1,189 @@ +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; +import { TAG_KEY } from './run-id'; + +const REPO_ROOT = path.resolve(__dirname, '../..'); + +export type PreparedProject = { + /** Absolute path of the temporary service directory. */ + dir: string; + /** The Serverless service name (== run id). */ + service: string; + /** Remove the temporary directory. Safe to call multiple times. */ + cleanup: () => void; +}; + +export type PrepareOptions = { + runId: string; + region: string; + /** Enable AppSync caching (Tier C). */ + caching?: boolean; + /** Custom-domain config for Tier D (CLI integration, not CloudFormation). */ + domain?: { + name: string; + hostedZoneId?: string; + certificateArn?: string; + }; +}; + +/** + * Build a minimal, deployable Serverless service in a fresh temp directory: + * + * - API_KEY auth with a single default key -> exercises ListApiKeys + * - a NONE data source + one JS (UNIT) resolver -> exercises EvaluateCode and + * keeps the deploy free of extra IAM roles + * - field logging enabled -> gives FilterLogEvents a + * real log group to read + * - tags including TAG_KEY: -> lets the sweeper find it + * + * The source plugin is linked in via a `node_modules/serverless-appsync-plugin` + * symlink pointing at the repo root (the same trick the offline e2e harness + * uses), so the deploy exercises THIS working tree, not an installed version. + */ +export function prepareProject(options: PrepareOptions): PreparedProject { + const { runId, region, caching, domain } = options; + + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'sls-appsync-it-')); + + // Link the plugin source into the project's node_modules. + const nodeModules = path.join(dir, 'node_modules'); + fs.mkdirSync(nodeModules, { recursive: true }); + fs.symlinkSync( + REPO_ROOT, + path.join(nodeModules, 'serverless-appsync-plugin'), + 'dir', + ); + + // GraphQL schema. + fs.writeFileSync( + path.join(dir, 'schema.graphql'), + [ + 'type Query {', + ' hello(name: String): String', + '}', + '', + 'schema {', + ' query: Query', + '}', + '', + ].join('\n'), + ); + + // JS resolver used by the EvaluateCode path. Deterministic so assertions are + // stable; references @aws-appsync/utils to mirror real resolvers (esbuild + // marks it external). + fs.mkdirSync(path.join(dir, 'resolvers'), { recursive: true }); + fs.writeFileSync( + path.join(dir, 'resolvers', 'hello.js'), + [ + "import { util } from '@aws-appsync/utils';", + '', + 'export function request(ctx) {', + ' return { payload: { name: ctx.arguments.name, id: util.autoId() } };', + '}', + '', + 'export function response(ctx) {', + ' return ctx.result;', + '}', + '', + ].join('\n'), + ); + + const yaml = buildServerlessYaml({ runId, region, caching, domain }); + fs.writeFileSync(path.join(dir, 'serverless.yml'), yaml); + + return { + dir, + service: runId, + cleanup: () => { + try { + fs.rmSync(dir, { recursive: true, force: true }); + } catch { + // best-effort + } + }, + }; +} + +function buildServerlessYaml(opts: { + runId: string; + region: string; + caching?: boolean; + domain?: { name: string; hostedZoneId?: string; certificateArn?: string }; +}): string { + const { runId, region, caching, domain } = opts; + + const lines: string[] = [ + `service: ${runId}`, + '', + 'provider:', + ' name: aws', + ' runtime: nodejs22.x', + ` region: ${region}`, + '', + 'plugins:', + ' - serverless-appsync-plugin', + '', + 'appSync:', + ` name: ${runId}`, + ' authentication:', + ' type: API_KEY', + ' apiKeys:', + ' - name: default', + ' logging:', + ' level: ERROR', + ' retentionInDays: 1', + ' tags:', + ` ${TAG_KEY}: ${runId}`, + ]; + + if (caching) { + lines.push( + ' caching:', + ' behavior: FULL_REQUEST_CACHING', + ' type: SMALL', + ' ttl: 60', + ); + } + + if (domain) { + lines.push( + ' domain:', + ` name: ${domain.name}`, + ' useCloudFormation: false', + ); + if (domain.hostedZoneId) { + lines.push(` hostedZoneId: ${domain.hostedZoneId}`); + } + if (domain.certificateArn) { + lines.push(` certificateArn: ${domain.certificateArn}`); + } + } + + lines.push( + ' resolvers:', + ' Query.hello:', + ' kind: UNIT', + ' dataSource: none_ds', + ' code: ./resolvers/hello.js', + ' dataSources:', + ' none_ds:', + ' type: NONE', + '', + // Expose the deployed API ARN as a stack output so the credential/region + // proof can read WHERE the API landed from `serverless info --verbose` + // (a child process) instead of constructing an AWS SDK client inside the + // Jest VM, which the v3 credential provider's dynamic import() breaks. + 'resources:', + ' Outputs:', + ' IntegrationApiArn:', + ' Value:', + ' Fn::GetAtt:', + ' - GraphQlApi', + ' - Arn', + '', + ); + + return lines.join('\n'); +} diff --git a/integration/helpers/run-id.ts b/integration/helpers/run-id.ts new file mode 100644 index 00000000..fd0449b0 --- /dev/null +++ b/integration/helpers/run-id.ts @@ -0,0 +1,33 @@ +import { randomBytes } from 'crypto'; + +/** + * Every resource created by the integration suite is tagged with this key so + * the standalone sweeper can find and delete leaked resources from interrupted + * runs. The value is the per-run id (see {@link generateRunId}). + */ +export const TAG_KEY = 'appsync-plugin-integration'; + +/** + * All run ids (and therefore service names and CloudFormation stack names) + * start with this prefix, so the sweeper can also find resources by name when + * tags are unavailable (e.g. a CloudFormation stack whose AppSync API was + * already deleted). + */ +export const RUN_ID_PREFIX = 'appsync-plugin-it'; + +/** + * Generate a unique, CloudFormation- and DNS-safe run id, e.g. + * `appsync-plugin-it-lq3z9k-1a2b3c4d`. Used as the Serverless service name + * (which drives the stack name), the AppSync API name, and the value of the + * {@link TAG_KEY} tag. + */ +export function generateRunId(): string { + const ts = Date.now().toString(36); + const rand = randomBytes(4).toString('hex'); + return `${RUN_ID_PREFIX}-${ts}-${rand}`; +} + +/** The standard Serverless stack name for a service at a stage. */ +export function stackName(service: string, stage = 'dev'): string { + return `${service}-${stage}`; +} diff --git a/integration/helpers/sls.ts b/integration/helpers/sls.ts new file mode 100644 index 00000000..92bfda9d --- /dev/null +++ b/integration/helpers/sls.ts @@ -0,0 +1,133 @@ +import { execFileSync } from 'child_process'; +import * as fs from 'fs'; +import * as path from 'path'; +import { integrationConfig } from './config'; + +const REPO_ROOT = path.resolve(__dirname, '../..'); + +/** + * The Serverless binary used to run deploys and plugin commands. Override with + * SERVERLESS_BIN to point the suite at a different install (e.g. a Serverless + * v4 binary). Defaults to the v3 install in this repo's node_modules. + * + * NOTE (v4): v4 resolves credentials as SDK v3 objects and may require + * SERVERLESS_ACCESS_KEY / a license and suppression of login prompts. The + * wrappers below already disable telemetry/interactive setup; the + * credential/region proof should be re-confirmed when running under v4. + */ +const SERVERLESS_BIN = + process.env.SERVERLESS_BIN || + path.join(REPO_ROOT, 'node_modules', '.bin', 'serverless'); + +export type SlsResult = { + stdout: string; + stderr: string; +}; + +export class SlsCommandError extends Error { + readonly stdout: string; + readonly stderr: string; + readonly status: number | null; + + constructor( + message: string, + stdout: string, + stderr: string, + status: number | null, + ) { + super(message); + // Restore the prototype chain: subclassing built-in Error under the + // repo's `target: es5` otherwise leaves instances with Error.prototype, + // breaking `instanceof SlsCommandError` and dropping custom fields. + Object.setPrototypeOf(this, SlsCommandError.prototype); + this.name = 'SlsCommandError'; + this.stdout = stdout; + this.stderr = stderr; + this.status = status; + } +} + +export type RunOptions = { + /** Working directory: the prepared service directory. */ + cwd: string; + /** Region passed via --region. Defaults to the configured region. */ + region?: string; + /** Profile passed via --aws-profile. Defaults to the configured profile. */ + profile?: string; + /** Extra environment variables. */ + env?: Record; + /** Per-command timeout in milliseconds. */ + timeout?: number; +}; + +/** + * Run an arbitrary `serverless ...` invocation. Throws {@link SlsCommandError} + * on non-zero exit so tests can assert on failure modes (used by the negative + * half of the credential/region proof). + */ +export function runServerless(args: string[], options: RunOptions): SlsResult { + if (!fs.existsSync(SERVERLESS_BIN)) { + throw new Error( + `Serverless binary not found at ${SERVERLESS_BIN}. Run \`npm ci\` or set SERVERLESS_BIN.`, + ); + } + + const region = options.region ?? integrationConfig.region; + const profile = options.profile ?? integrationConfig.profile; + + const fullArgs = [...args, '--region', region]; + if (profile) { + fullArgs.push('--aws-profile', profile); + } + + try { + const stdout = execFileSync(SERVERLESS_BIN, fullArgs, { + cwd: options.cwd, + timeout: options.timeout ?? 240_000, + // Verbose CloudFormation output from deploy/remove/info can exceed the + // 1 MB default and surface as a spurious ENOBUFS SlsCommandError. + maxBuffer: 64 * 1024 * 1024, + env: { + ...process.env, + SLS_NOTIFICATIONS_MODE: 'off', + SLS_INTERACTIVE_SETUP_ENABLE: '0', + SLS_TELEMETRY_DISABLED: '1', + AWS_REGION: region, + AWS_DEFAULT_REGION: region, + ...options.env, + }, + stdio: ['ignore', 'pipe', 'pipe'], + }); + return { stdout: stdout.toString(), stderr: '' }; + } catch (err) { + const e = err as { + stdout?: Buffer; + stderr?: Buffer; + status?: number | null; + message: string; + }; + const stdout = e.stdout?.toString() ?? ''; + const stderr = e.stderr?.toString() ?? ''; + throw new SlsCommandError( + `serverless ${fullArgs.join(' ')} failed:\n${stdout}\n${stderr}\n${ + e.message + }`, + stdout, + stderr, + e.status ?? null, + ); + } +} + +export const deploy = (options: RunOptions): SlsResult => + runServerless(['deploy'], { timeout: 600_000, ...options }); + +export const remove = (options: RunOptions): SlsResult => + runServerless(['remove'], { timeout: 600_000, ...options }); + +export const info = (options: RunOptions): SlsResult => + runServerless(['info'], options); + +/** Run an `serverless appsync ` plugin command. */ +export const appsync = (subcommand: string[], options: RunOptions): SlsResult => + runServerless(['appsync', ...subcommand], options); diff --git a/integration/sweeper.ts b/integration/sweeper.ts new file mode 100644 index 00000000..7b63e2b3 --- /dev/null +++ b/integration/sweeper.ts @@ -0,0 +1,101 @@ +/** + * Standalone leaked-resource sweeper for the integration suite. + * + * Tests tear themselves down in afterAll, but runs can be interrupted (Ctrl-C, + * CI timeout, crash). This script finds and deletes any resources tagged by the + * suite and any CloudFormation stacks named with the run-id prefix, so leaks + * don't accumulate cost. It is idempotent and safe to re-run. + * + * Usage: + * APPSYNC_PLUGIN_INTEGRATION=1 \ + * APPSYNC_PLUGIN_INTEGRATION_REGION=us-west-2 \ + * [APPSYNC_PLUGIN_INTEGRATION_PROFILE=my-sandbox] \ + * npm run test:integration:sweep + * + * Add APPSYNC_PLUGIN_INTEGRATION_DOMAIN to also clean a leaked custom domain. + */ +import { integrationConfig } from './helpers/config'; +import { + findLeakedApis, + deleteApi, + findLeakedStacks, + deleteStack, + cleanupDomainName, +} from './helpers/aws'; + +function log(message: string): void { + // eslint-disable-next-line no-console + console.log(`[sweeper] ${message}`); +} + +async function main(): Promise { + if (process.env.APPSYNC_PLUGIN_INTEGRATION !== '1') { + log('APPSYNC_PLUGIN_INTEGRATION is not set to 1 — refusing to sweep.'); + process.exitCode = 0; + return; + } + + const { region } = integrationConfig; + let failures = 0; + log( + `sweeping region ${region}` + + (integrationConfig.profile + ? ` (profile ${integrationConfig.profile})` + : ''), + ); + + // 1. Tagged AppSync APIs. + const apis = await findLeakedApis(region); + if (apis.length === 0) { + log('no leaked AppSync APIs found'); + } + for (const api of apis) { + log(`deleting AppSync API ${api.apiId} (${api.name ?? 'unnamed'})`); + try { + await deleteApi(region, api.apiId); + } catch (err) { + failures++; + log(` failed: ${(err as Error).message}`); + } + } + + // 2. Leaked custom domain (if one was configured for this environment). + if (integrationConfig.domain.name) { + log(`cleaning custom domain ${integrationConfig.domain.name}`); + try { + await cleanupDomainName(region, integrationConfig.domain.name); + } catch (err) { + failures++; + log(` failed: ${(err as Error).message}`); + } + } + + // 3. Prefixed CloudFormation stacks. + const stacks = await findLeakedStacks(region); + if (stacks.length === 0) { + log('no leaked CloudFormation stacks found'); + } + for (const name of stacks) { + log(`deleting CloudFormation stack ${name}`); + try { + await deleteStack(region, name); + } catch (err) { + failures++; + log(` failed: ${(err as Error).message}`); + } + } + + log('done (stack deletions are asynchronous; verify in the console)'); + if (failures > 0) { + // Surface delete failures so the CI backstop doesn't report success while + // leaving billable resources behind. + log(`${failures} deletion(s) failed`); + process.exitCode = 1; + } +} + +main().catch((err) => { + // eslint-disable-next-line no-console + console.error(err); + process.exitCode = 1; +}); diff --git a/jest.integration.config.ts b/jest.integration.config.ts new file mode 100644 index 00000000..bbd60c7e --- /dev/null +++ b/jest.integration.config.ts @@ -0,0 +1,30 @@ +import type { Config } from '@jest/types'; + +/** + * Jest config for the OPT-IN live AWS integration suite. + * + * This is intentionally separate from jest.config.ts (unit) and + * jest.e2e.config.ts (offline CFN synthesis). It is only ever run via + * `npm run test:integration` and is never part of `npm test`, + * `npm run test:e2e`, `npm run test:all`, or the default CI. + * + * When APPSYNC_PLUGIN_INTEGRATION is unset, every suite resolves to + * `describe.skip` (see integration/helpers/gate.ts) and this config exits 0. + */ +const config: Config.InitialOptions = { + preset: 'ts-jest', + testEnvironment: 'node', + rootDir: '.', + testMatch: ['/integration/**/*.integration.test.ts'], + // Live AWS calls and eventual consistency: generous default, per-test + // overrides for deploys. + testTimeout: 300_000, + globalSetup: './jest.integration.setup.ts', + // Serialize: deploys are heavy and share an AWS account (and, in Tier D, a + // single domain), so running them in parallel invites throttling/collisions. + maxWorkers: 1, + // Skipped-only runs (no credentials) must still exit green. + passWithNoTests: true, +}; + +export default config; diff --git a/jest.integration.setup.ts b/jest.integration.setup.ts new file mode 100644 index 00000000..c3eaa646 --- /dev/null +++ b/jest.integration.setup.ts @@ -0,0 +1,22 @@ +/** + * Jest globalSetup for the integration suite. + * + * Each test scaffolds its own temporary Serverless service and links the plugin + * source itself, so the only global precondition is that the plugin has been + * compiled to `lib/` (the deployed service resolves the plugin from there). + * `npm run test:integration` runs `npm run build` first, so this is just a + * guard with a helpful message. + */ +import * as fs from 'fs'; +import * as path from 'path'; + +export default async function globalSetup(): Promise { + const libPath = path.resolve(__dirname, 'lib', 'index.js'); + if (!fs.existsSync(libPath)) { + throw new Error( + `Plugin build artifact not found at ${libPath}. ` + + 'Run `npm run build` before the integration suite, or use ' + + '`npm run test:integration` which builds first.', + ); + } +} diff --git a/package.json b/package.json index 7f54e420..e4c789fe 100644 --- a/package.json +++ b/package.json @@ -14,6 +14,9 @@ "test:watch": "jest src/__tests__/.*\\.test\\.ts --watch", "test:e2e": "npm run build && jest --config jest.e2e.config.ts", "test:all": "npm run test && npm run test:e2e", + "test:integration": "npm run build && jest --config jest.integration.config.ts", + "test:integration:typecheck": "tsc -p tsconfig.integration.json", + "test:integration:sweep": "ts-node integration/sweeper.ts", "build": "tsc", "lint": "eslint . && tsc --noEmit", "lint:fix": "eslint . --fix", diff --git a/tsconfig.integration.json b/tsconfig.integration.json new file mode 100644 index 00000000..c7385b33 --- /dev/null +++ b/tsconfig.integration.json @@ -0,0 +1,12 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "noEmit": true + }, + "include": [ + "integration/**/*.ts", + "jest.integration.config.ts", + "jest.integration.setup.ts" + ], + "exclude": ["node_modules", "lib"] +}