diff --git a/README.md b/README.md index 79494cf..87ea800 100644 --- a/README.md +++ b/README.md @@ -243,6 +243,7 @@ This action requires the following minimum set of permissions: "cloudformation:CreateChangeSet", "cloudformation:DescribeChangeSet", "cloudformation:DeleteChangeSet", + "cloudformation:DeleteStack", "cloudformation:ExecuteChangeSet", "cloudformation:DescribeEvents" ], @@ -281,12 +282,14 @@ The action makes the following AWS CloudFormation API calls depending on the ope - `DescribeEvents` - Retrieve detailed error information for validation failures - `DeleteChangeSet` - Clean up failed change sets (unless `no-delete-failed-changeset` is set) +**ROLLBACK_COMPLETE Recovery:** + +- `DeleteStack` - Automatically delete stacks stuck in `ROLLBACK_COMPLETE` state before recreating + **Event Streaming (during stack operations):** - `DescribeEvents` - Monitor real-time CloudFormation events during deployment -> The policy above prevents the stack from being deleted - add `cloudformation:DeleteStack` if deletion is required for your use case - ## Example You want to run your microservices with [Amazon Elastic Kubernetes Services](https://aws.amazon.com/eks/) and leverage the best-practices to run the cluster? Using this GitHub Action you can customize and deploy the [modular and scalable Amazon EKS architecture](https://aws.amazon.com/quickstart/architecture/amazon-eks/) provided in an AWS Quick Start to your AWS Account. The following workflow enables you to create and update a Kubernetes cluster using a manual workflow trigger. diff --git a/__tests__/deploy.test.ts b/__tests__/deploy.test.ts index 4088be8..17f0b96 100644 --- a/__tests__/deploy.test.ts +++ b/__tests__/deploy.test.ts @@ -1,9 +1,11 @@ import { CloudFormationClient, + CloudFormationServiceException, DescribeStacksCommand, DescribeChangeSetCommand, DescribeEventsCommand, CreateChangeSetCommand, + DeleteStackCommand, ExecuteChangeSetCommand, StackStatus, ChangeSetStatus, @@ -13,6 +15,7 @@ import { mockClient } from 'aws-sdk-client-mock' import { waitUntilStackOperationComplete, updateStack, + deployStack, executeExistingChangeSet } from '../src/deploy' import * as core from '@actions/core' @@ -564,4 +567,310 @@ describe('Deploy error scenarios', () => { expect(result.stackId).toBe('test-stack-id') }) }) + + describe('deployStack', () => { + it('deletes and recreates stack in ROLLBACK_COMPLETE state', async () => { + // First call: getStack returns ROLLBACK_COMPLETE + // Second call: waitUntilStackDeleteComplete sees DELETE_COMPLETE + // Third call: waitUntilStackOperationComplete after execute + mockCfnClient + .on(DescribeStacksCommand) + .resolvesOnce({ + Stacks: [ + { + StackName: 'TestStack', + StackId: 'test-stack-id', + StackStatus: StackStatus.ROLLBACK_COMPLETE, + CreationTime: new Date() + } + ] + }) + .resolvesOnce({ + Stacks: [ + { + StackName: 'TestStack', + StackId: 'test-stack-id', + StackStatus: StackStatus.DELETE_COMPLETE, + CreationTime: new Date() + } + ] + }) + .resolves({ + Stacks: [ + { + StackName: 'TestStack', + StackId: 'new-stack-id', + StackStatus: StackStatus.CREATE_COMPLETE, + CreationTime: new Date() + } + ] + }) + + mockCfnClient.on(DeleteStackCommand).resolves({}) + mockCfnClient + .on(CreateChangeSetCommand) + .resolves({ Id: 'test-cs-id', StackId: 'new-stack-id' }) + mockCfnClient.on(DescribeChangeSetCommand).resolves({ + Status: ChangeSetStatus.CREATE_COMPLETE, + Changes: [{ Type: 'Resource' }] + }) + mockCfnClient.on(ExecuteChangeSetCommand).resolves({}) + + const result = await deployStack( + cfn, + { + StackName: 'TestStack', + TemplateBody: '{}', + Capabilities: [] + }, + 'test-cs', + false, + false, + false, + 60 + ) + + expect(mockCfnClient.commandCalls(DeleteStackCommand)).toHaveLength(1) + expect(core.info).toHaveBeenCalledWith( + expect.stringContaining('ROLLBACK_COMPLETE') + ) + expect(result.stackId).toBe('new-stack-id') + }) + + it('does not delete stack when not in ROLLBACK_COMPLETE state', async () => { + mockCfnClient.on(DescribeStacksCommand).resolves({ + Stacks: [ + { + StackName: 'TestStack', + StackId: 'test-stack-id', + StackStatus: StackStatus.CREATE_COMPLETE, + CreationTime: new Date() + } + ] + }) + + mockCfnClient + .on(CreateChangeSetCommand) + .resolves({ Id: 'test-cs-id', StackId: 'test-stack-id' }) + mockCfnClient.on(DescribeChangeSetCommand).resolves({ + Status: ChangeSetStatus.CREATE_COMPLETE, + Changes: [{ Type: 'Resource' }] + }) + mockCfnClient.on(ExecuteChangeSetCommand).resolves({}) + + await deployStack( + cfn, + { + StackName: 'TestStack', + TemplateBody: '{}', + Capabilities: [] + }, + 'test-cs', + false, + false, + false, + 60 + ) + + expect(mockCfnClient.commandCalls(DeleteStackCommand)).toHaveLength(0) + }) + + it('handles delete completing when stack disappears', async () => { + mockCfnClient + .on(DescribeStacksCommand) + .resolvesOnce({ + Stacks: [ + { + StackName: 'TestStack', + StackId: 'test-stack-id', + StackStatus: StackStatus.ROLLBACK_COMPLETE, + CreationTime: new Date() + } + ] + }) + .resolvesOnce({ Stacks: [] }) + .resolves({ + Stacks: [ + { + StackName: 'TestStack', + StackId: 'new-stack-id', + StackStatus: StackStatus.CREATE_COMPLETE, + CreationTime: new Date() + } + ] + }) + + mockCfnClient.on(DeleteStackCommand).resolves({}) + mockCfnClient + .on(CreateChangeSetCommand) + .resolves({ Id: 'test-cs-id', StackId: 'new-stack-id' }) + mockCfnClient.on(DescribeChangeSetCommand).resolves({ + Status: ChangeSetStatus.CREATE_COMPLETE, + Changes: [{ Type: 'Resource' }] + }) + mockCfnClient.on(ExecuteChangeSetCommand).resolves({}) + + const result = await deployStack( + cfn, + { StackName: 'TestStack', TemplateBody: '{}', Capabilities: [] }, + 'test-cs', + false, + false, + false, + 60 + ) + + expect(mockCfnClient.commandCalls(DeleteStackCommand)).toHaveLength(1) + expect(result.stackId).toBe('new-stack-id') + }) + + it('handles delete completing via ValidationError', async () => { + const validationError = new CloudFormationServiceException({ + name: 'ValidationError', + $fault: 'client', + $metadata: { httpStatusCode: 400 }, + message: 'Stack does not exist' + }) + validationError.name = 'ValidationError' + + mockCfnClient + .on(DescribeStacksCommand) + .resolvesOnce({ + Stacks: [ + { + StackName: 'TestStack', + StackId: 'test-stack-id', + StackStatus: StackStatus.ROLLBACK_COMPLETE, + CreationTime: new Date() + } + ] + }) + .rejectsOnce(validationError) + .resolves({ + Stacks: [ + { + StackName: 'TestStack', + StackId: 'new-stack-id', + StackStatus: StackStatus.CREATE_COMPLETE, + CreationTime: new Date() + } + ] + }) + + mockCfnClient.on(DeleteStackCommand).resolves({}) + mockCfnClient + .on(CreateChangeSetCommand) + .resolves({ Id: 'test-cs-id', StackId: 'new-stack-id' }) + mockCfnClient.on(DescribeChangeSetCommand).resolves({ + Status: ChangeSetStatus.CREATE_COMPLETE, + Changes: [{ Type: 'Resource' }] + }) + mockCfnClient.on(ExecuteChangeSetCommand).resolves({}) + + const result = await deployStack( + cfn, + { StackName: 'TestStack', TemplateBody: '{}', Capabilities: [] }, + 'test-cs', + false, + false, + false, + 60 + ) + + expect(result.stackId).toBe('new-stack-id') + }) + + it('throws error when stack deletion fails', async () => { + mockCfnClient + .on(DescribeStacksCommand) + .resolvesOnce({ + Stacks: [ + { + StackName: 'TestStack', + StackId: 'test-stack-id', + StackStatus: StackStatus.ROLLBACK_COMPLETE, + CreationTime: new Date() + } + ] + }) + .resolves({ + Stacks: [ + { + StackName: 'TestStack', + StackId: 'test-stack-id', + StackStatus: StackStatus.DELETE_FAILED, + CreationTime: new Date() + } + ] + }) + + mockCfnClient.on(DeleteStackCommand).resolves({}) + + await expect( + deployStack( + cfn, + { StackName: 'TestStack', TemplateBody: '{}', Capabilities: [] }, + 'test-cs', + false, + false, + false, + 60 + ) + ).rejects.toThrow('Stack deletion failed for TestStack') + }) + + it('times out waiting for stack deletion', async () => { + mockCfnClient + .on(DescribeStacksCommand) + .resolvesOnce({ + Stacks: [ + { + StackName: 'TestStack', + StackId: 'test-stack-id', + StackStatus: StackStatus.ROLLBACK_COMPLETE, + CreationTime: new Date() + } + ] + }) + .resolves({ + Stacks: [ + { + StackName: 'TestStack', + StackId: 'test-stack-id', + StackStatus: StackStatus.DELETE_IN_PROGRESS, + CreationTime: new Date() + } + ] + }) + + mockCfnClient.on(DeleteStackCommand).resolves({}) + + const realDateNow = Date.now + let callCount = 0 + Date.now = jest.fn(() => { + callCount++ + // First call is the startTime capture, subsequent calls exceed maxWaitTime + return callCount <= 1 ? 0 : 2 * 1000 + 1 + }) + + const realSetTimeout = global.setTimeout + global.setTimeout = ((fn: () => void) => + realSetTimeout(fn, 0)) as unknown as typeof setTimeout + + await expect( + deployStack( + cfn, + { StackName: 'TestStack', TemplateBody: '{}', Capabilities: [] }, + 'test-cs', + false, + false, + false, + 2 + ) + ).rejects.toThrow('Timeout waiting for stack deletion after 2 seconds') + + Date.now = realDateNow + global.setTimeout = realSetTimeout + }) + }) }) diff --git a/dist/index.js b/dist/index.js index a352d16..549ad24 100644 --- a/dist/index.js +++ b/dist/index.js @@ -62848,6 +62848,39 @@ function waitUntilStackOperationComplete(params, input) { throw new Error(`Timeout after ${maxWaitTime} seconds`); }); } +function waitUntilStackDeleteComplete(params, input) { + return __awaiter(this, void 0, void 0, function* () { + var _a; + const { client, maxWaitTime, minDelay } = params; + const startTime = Date.now(); + while (Date.now() - startTime < maxWaitTime * 1000) { + try { + const result = yield client.send(new client_cloudformation_1.DescribeStacksCommand(input)); + const stack = (_a = result.Stacks) === null || _a === void 0 ? void 0 : _a[0]; + if (!stack) { + return; + } + if (stack.StackStatus === 'DELETE_COMPLETE') { + return; + } + if (stack.StackStatus === 'DELETE_FAILED') { + throw new Error(`Stack deletion failed for ${input.StackName}`); + } + core.debug(`Stack delete in progress, waiting ${minDelay} seconds...`); + yield new Promise(resolve => setTimeout(resolve, minDelay * 1000)); + } + catch (error) { + if (error instanceof client_cloudformation_1.CloudFormationServiceException && + error.$metadata.httpStatusCode === 400 && + error.name === 'ValidationError') { + return; + } + throw error; + } + } + throw new Error(`Timeout waiting for stack deletion after ${maxWaitTime} seconds`); + }); +} function executeExistingChangeSet(cfn_1, stackName_1, changeSetId_1) { return __awaiter(this, arguments, void 0, function* (cfn, stackName, changeSetId, maxWaitTime = 21000) { core.debug(`Executing existing change set: ${changeSetId}`); @@ -63103,7 +63136,13 @@ function buildUpdateChangeSetParams(params, changeSetName) { } function deployStack(cfn_1, params_1, changeSetName_1, failOnEmptyChangeSet_1, noExecuteChangeSet_1, noDeleteFailedChangeSet_1) { return __awaiter(this, arguments, void 0, function* (cfn, params, changeSetName, failOnEmptyChangeSet, noExecuteChangeSet, noDeleteFailedChangeSet, maxWaitTime = 21000, onChangeSetReady) { - const stack = yield getStack(cfn, params.StackName); + let stack = yield getStack(cfn, params.StackName); + if ((stack === null || stack === void 0 ? void 0 : stack.StackStatus) === 'ROLLBACK_COMPLETE') { + core.info(`Stack ${params.StackName} is in ROLLBACK_COMPLETE state. Deleting stack and recreating.`); + yield cfn.send(new client_cloudformation_1.DeleteStackCommand({ StackName: params.StackName })); + yield waitUntilStackDeleteComplete({ client: cfn, maxWaitTime, minDelay: 5 }, { StackName: params.StackName }); + stack = undefined; + } if (!stack) { core.debug(`Creating CloudFormation Stack via Change Set`); const createParams = buildCreateChangeSetParams(params, changeSetName); diff --git a/src/deploy.ts b/src/deploy.ts index ed5f052..e3737d4 100644 --- a/src/deploy.ts +++ b/src/deploy.ts @@ -4,6 +4,7 @@ import { Stack, DescribeChangeSetCommand, DeleteChangeSetCommand, + DeleteStackCommand, waitUntilChangeSetCreateComplete, CreateChangeSetCommand, ExecuteChangeSetCommand, @@ -85,6 +86,53 @@ export async function waitUntilStackOperationComplete( throw new Error(`Timeout after ${maxWaitTime} seconds`) } +async function waitUntilStackDeleteComplete( + params: { + client: CloudFormationClient + maxWaitTime: number + minDelay: number + }, + input: { StackName: string } +): Promise { + const { client, maxWaitTime, minDelay } = params + const startTime = Date.now() + + while (Date.now() - startTime < maxWaitTime * 1000) { + try { + const result = await client.send(new DescribeStacksCommand(input)) + const stack = result.Stacks?.[0] + + if (!stack) { + return + } + + if (stack.StackStatus === 'DELETE_COMPLETE') { + return + } + + if (stack.StackStatus === 'DELETE_FAILED') { + throw new Error(`Stack deletion failed for ${input.StackName}`) + } + + core.debug(`Stack delete in progress, waiting ${minDelay} seconds...`) + await new Promise(resolve => setTimeout(resolve, minDelay * 1000)) + } catch (error) { + if ( + error instanceof CloudFormationServiceException && + error.$metadata.httpStatusCode === 400 && + error.name === 'ValidationError' + ) { + return + } + throw error + } + } + + throw new Error( + `Timeout waiting for stack deletion after ${maxWaitTime} seconds` + ) +} + export async function executeExistingChangeSet( cfn: CloudFormationClient, stackName: string, @@ -482,7 +530,19 @@ export async function deployStack( maxWaitTime = 21000, onChangeSetReady?: () => void ): Promise<{ stackId?: string; changeSetInfo?: ChangeSetInfo }> { - const stack = await getStack(cfn, params.StackName) + let stack = await getStack(cfn, params.StackName) + + if (stack?.StackStatus === 'ROLLBACK_COMPLETE') { + core.info( + `Stack ${params.StackName} is in ROLLBACK_COMPLETE state. Deleting stack and recreating.` + ) + await cfn.send(new DeleteStackCommand({ StackName: params.StackName })) + await waitUntilStackDeleteComplete( + { client: cfn, maxWaitTime, minDelay: 5 }, + { StackName: params.StackName } + ) + stack = undefined + } if (!stack) { core.debug(`Creating CloudFormation Stack via Change Set`)