CLD-485: only create cloud trail if nesessary

aws/sam-app/README.md

@@ -16,10 +16,18 @@ and records in your AWS account.
 |   Route 53   |  -->  |  CloudTrail  |  -->  |  EventBridge |  -->  |      SQS     |  -->  |    Lambda    |  -->  |      NS1     |
 +--------------+       +--------------+       +--------------+       +--------------+       +--------------+       +--------------+
 
-When zones and records are created, changed, and deleted, events are automatically recorded in CloudTrail. The installed EventBridge rule watches for these events and queues them up 
+When zones and records are created, changed, and deleted, events are automatically recorded in CloudTrail. The installed EventBridge rule watches for these events and queues them up
 for the Lambda to take on the next leg. Processing of the messages for the events is minimal. The message is taken in its entirety, lightly wrapped, and sent
 off across the Internet to the NS1 CloudSync REST API endpoint.  Events are then processed asynchronously in NS1 Connect.
-
+
+## Parameters
+
+| Parameter | Default | Description |
+|---|---|---|
+| `NS1APIKey` | *(required)* | Your NS1 Connect account API key. |
+| `CreateCloudTrail` | `false` | Set to `true` only if your AWS account has no existing active multi-region CloudTrail trail. Most accounts already have a default trail (`management-events`); leave as `false` to avoid creating a duplicate trail and incurring unnecessary charges. |
+| `CloudTrailName` | `NS1CloudSyncTrail` | **Only used when `CreateCloudTrail` is `true`** — leave as default if `CreateCloudTrail` is `false`. Name for the new trail created by this stack. Do not set this to the name of an existing trail. |
+
 ## Security and permissions
 You will need to provide an NS1 API key when installing the stack. A secret in AWS Secrets Manager is created for this key. When the `dns_updates` Lambda initializes it will request the key
 from Secrets Manager and store it in memory for the lifetime of the Lambda execution environment.

aws/sam-app/packaged.yaml

@@ -16,22 +16,52 @@ Metadata:
     Author: NS1
     SpdxLicenseId: Apache-2.0
     LicenseUrl: s3://aws-sam-cli-managed-default-samclisourcebucket-5bpg79c8x6lw/86d3f3a95c324c9479bd8986968f4327
-    ReadmeUrl: s3://aws-sam-cli-managed-default-samclisourcebucket-5bpg79c8x6lw/36a78acacaac7abb146c54ab6b7c3fe2
+    ReadmeUrl: s3://aws-sam-cli-managed-default-samclisourcebucket-5bpg79c8x6lw/88f031fa79b0248f4468d01985457afa
     Labels:
     - serverless
     - NS1
     - Route53
     - DNS
     HomePageUrl: https://github.com/ns1/cloudsync
-    SemanticVersion: '0.4.5'
-    SourceCodeUrl: https://github.com/ns1/cloudsync/tree/0.4.5
+    SemanticVersion: '0.4.6'
+    SourceCodeUrl: https://github.com/ns1/cloudsync/tree/0.4.6
 Parameters:
+  CreateCloudTrail:
+    Type: String
+    Default: 'false'
+    AllowedValues:
+    - 'true'
+    - 'false'
+    Description: 'Controls whether this stack creates a dedicated CloudTrail trail.
+      Set to "false" if your AWS account already has an existing active CloudTrail
+      trail. The trail must be capturing Route 53 management events and delivering
+      them to EventBridge. If DNS sync stops working after setting this to "false",
+      your existing trail may not be configured correctly - set to "true" to have
+      this stack create a dedicated trail with the correct settings. Set to "true"
+      if you are unsure or if your account has no existing trail.
+
+      '
   CloudTrailName:
     Type: String
     Default: NS1CloudSyncTrail
+    AllowedPattern: ^(?!management-events$)(?!aws-controltower-BaselineCloudTrail$)(?!AccountTrail$).+
+    ConstraintDescription: CloudTrailName must not be the name of an existing system
+      or organization trail (e.g. management-events, aws-controltower-BaselineCloudTrail,
+      AccountTrail). Leave as default (NS1CloudSyncTrail) unless you have a specific
+      reason to change it.
+    Description: "ONLY USED when CreateCloudTrail is \"true\" \u2014 ignored otherwise,\
+      \ leave as default. Name for the new CloudTrail trail created by this stack.\
+      \ WARNING: Do not set this to the name of an existing trail such as \"management-events\"\
+      \ \u2014 this stack will delete the trail when it is removed or when CreateCloudTrail\
+      \ is changed to \"false\"."
   NS1APIKey:
     Type: String
     Description: NS1 account API key.
+Conditions:
+  ShouldCreateTrail:
+    Fn::Equals:
+    - Ref: CreateCloudTrail
+    - 'true'
 Resources:
   Route53NSUpdaterRole:
     Type: AWS::IAM::Role
@@ -74,6 +104,7 @@ Resources:
       SamResourceId: DNSUpdateQueue
   TrailKMSKey:
     Type: AWS::KMS::Key
+    Condition: ShouldCreateTrail
     Properties:
       Enabled: true
       KeySpec: SYMMETRIC_DEFAULT
@@ -158,6 +189,7 @@ Resources:
       SamResourceId: TrailKMSKey
   TrailS3Bucket:
     Type: AWS::S3::Bucket
+    Condition: ShouldCreateTrail
     DeletionPolicy: Delete
     Properties:
       BucketName:
@@ -177,6 +209,7 @@ Resources:
       SamResourceId: TrailS3Bucket
   TrailBucketPolicy:
     Type: AWS::S3::BucketPolicy
+    Condition: ShouldCreateTrail
     Properties:
       Bucket:
         Ref: TrailS3Bucket
@@ -213,6 +246,7 @@ Resources:
       SamResourceId: TrailBucketPolicy
   Trail:
     Type: AWS::CloudTrail::Trail
+    Condition: ShouldCreateTrail
     DependsOn: TrailBucketPolicy
     Properties:
       S3BucketName:
@@ -376,7 +410,7 @@ Resources:
     Type: AWS::Serverless::Function
     Properties:
       FunctionName: CloudSyncDNSUpdateFunction
-      CodeUri: s3://aws-sam-cli-managed-default-samclisourcebucket-5bpg79c8x6lw/8c65d817f7b8b21d4e6a63f637184d6c
+      CodeUri: s3://aws-sam-cli-managed-default-samclisourcebucket-5bpg79c8x6lw/7275877880f92da93b2cdbb40b09be4d
       Handler: app.handler
       Timeout: 10
       Role:
@@ -453,32 +487,37 @@ Resources:
             - Fn::Sub: arn:aws:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/lambda/*SnapshotListFunction*
             - Fn::Sub: arn:aws:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/lambda/*TriggerStateMachineFunction*
             - Fn::Sub: arn:aws:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/lambda/*DNSUpdateFunction*
+          - Effect: Allow
+            Action:
+            - s3:ListAllMyBuckets
+            Resource: '*'
           - Effect: Allow
             Action:
             - s3:DeleteObject
             - s3:GetObject
             - s3:ListBucket
             Resource:
-              Fn::GetAtt:
-              - TrailS3Bucket
-              - Arn
+              Fn::Sub: arn:aws:s3:::cloudsync-trail-bucket-*
           - Effect: Allow
             Action:
             - s3:DeleteObject
             - s3:GetObject
             - s3:ListBucket
             Resource:
-              Fn::Sub:
-              - arn:aws:s3:::${Bucket}/*
-              - Bucket:
-                  Ref: TrailS3Bucket
+              Fn::Sub: arn:aws:s3:::cloudsync-trail-bucket-*/*
+          - Effect: Allow
+            Action:
+            - cloudtrail:StopLogging
+            - cloudtrail:DeleteTrail
+            Resource:
+              Fn::Sub: arn:aws:cloudtrail:us-east-1:${AWS::AccountId}:trail/*
     Metadata:
       SamResourceId: ConfigureFunctionRole
   ConfigureFunction:
     Type: AWS::Serverless::Function
     Properties:
       FunctionName: CloudSyncConfigureFunction
-      CodeUri: s3://aws-sam-cli-managed-default-samclisourcebucket-5bpg79c8x6lw/b8986695f01389cb13b2a4b65944bf0e
+      CodeUri: s3://aws-sam-cli-managed-default-samclisourcebucket-5bpg79c8x6lw/6d23588d10c2c64ae0ad67991a609881
       Handler: app.configure_application
       Timeout: 300
       Role:
@@ -504,7 +543,14 @@ Resources:
         - ConfigureFunction
         - Arn
       CloudTrailBucketName:
-        Ref: TrailS3Bucket
+        Fn::If:
+        - ShouldCreateTrail
+        - Ref: TrailS3Bucket
+        - ''
+      CloudTrailName:
+        Ref: CloudTrailName
+      CreateCloudTrail:
+        Ref: CreateCloudTrail
     Metadata:
       SamResourceId: ConfigureCustomResource
   SnapshotFunctionRole:
@@ -554,7 +600,7 @@ Resources:
     Type: AWS::Serverless::Function
     Properties:
       FunctionName: CloudSyncSnapshotFunction
-      CodeUri: s3://aws-sam-cli-managed-default-samclisourcebucket-5bpg79c8x6lw/bca65b343d87c7cb35f21359c065a746
+      CodeUri: s3://aws-sam-cli-managed-default-samclisourcebucket-5bpg79c8x6lw/3973d0b91ac2f038f09a9cdcfd3e6230
       Handler: app.lambda_handler
       Role:
         Fn::GetAtt:
@@ -578,7 +624,7 @@ Resources:
     Type: AWS::Serverless::Function
     Properties:
       FunctionName: CloudSyncSnapshotListFunction
-      CodeUri: s3://aws-sam-cli-managed-default-samclisourcebucket-5bpg79c8x6lw/5cd66b8abe52b88f0d3c8f40fff8f32e
+      CodeUri: s3://aws-sam-cli-managed-default-samclisourcebucket-5bpg79c8x6lw/3eb338e2099b8affed6c70ce13cbdb5b
       Handler: app.lambda_handler
       Policies:
       - Version: '2012-10-17'
@@ -596,7 +642,7 @@ Resources:
     Type: AWS::Serverless::Function
     Properties:
       FunctionName: CloudSyncHCSnapshotFunction
-      CodeUri: s3://aws-sam-cli-managed-default-samclisourcebucket-5bpg79c8x6lw/451b661e63f84cb47cd0f5b6be921c00
+      CodeUri: s3://aws-sam-cli-managed-default-samclisourcebucket-5bpg79c8x6lw/3e6237008c5cd3a303819a0f788594fc
       Handler: app.lambda_handler
       Role:
         Fn::GetAtt:
@@ -739,7 +785,7 @@ Resources:
     Type: AWS::Serverless::Function
     Properties:
       FunctionName: CloudSyncTriggerStateMachineFunction
-      CodeUri: s3://aws-sam-cli-managed-default-samclisourcebucket-5bpg79c8x6lw/504d27b882f65af836f3189e314b7264
+      CodeUri: s3://aws-sam-cli-managed-default-samclisourcebucket-5bpg79c8x6lw/8a180dcc15bae43041f365dbcd76702a
       Handler: app.lambda_handler
       Policies:
       - Version: '2012-10-17'
@@ -768,3 +814,47 @@ Resources:
     DependsOn: SnapshotStateMachine
     Metadata:
       SamResourceId: TriggerSnapshotCustomResource
+Outputs:
+  CloudTrailStatus:
+    Description: 'Whether this stack created a CloudTrail trail. If "Not managed by
+      this stack", ensure your account has an existing active CloudTrail trail that
+      captures Route 53 management events. If DNS sync stops working, your existing
+      trail may not be configured correctly - redeploy with CreateCloudTrail=true.
+      If upgrading from a previous version that created its own trail: the old trail
+      has been stopped and deleted, but the S3 log bucket (cloudsync-trail-bucket-*)
+      has been retained. Check the RetainedCloudTrailBucket output for the bucket
+      name. Once you have verified DNS sync is working, empty and delete the bucket
+      manually to stop incurring storage charges.'
+    Value:
+      Fn::If:
+      - ShouldCreateTrail
+      - Fn::Sub: 'Created and managed by this stack: ${Trail}'
+      - Not managed by this stack - ensure your existing trail captures Route 53 management
+        events
+  EventBridgeRuleArn:
+    Description: ARN of the EventBridge rule that captures Route 53 events
+    Value:
+      Fn::GetAtt:
+      - EventRule
+      - Arn
+  DNSUpdateQueueUrl:
+    Description: URL of the SQS queue receiving Route 53 change events
+    Value:
+      Fn::GetAtt:
+      - DNSUpdateQueue
+      - QueueUrl
+  DNSUpdateFunctionArn:
+    Description: ARN of the Lambda function processing Route 53 change events
+    Value:
+      Fn::GetAtt:
+      - DNSUpdateFunction
+      - Arn
+  RetainedCloudTrailBucket:
+    Description: Name of the CloudTrail log bucket retained after upgrading from a
+      version that created a trail. This bucket is no longer needed and can be safely
+      deleted to stop incurring storage charges. If no bucket was retained during
+      this deployment, this value will be empty.
+    Value:
+      Fn::GetAtt:
+      - ConfigureCustomResource
+      - RetainedCloudTrailBucket

aws/sam-app/src/configure/app.py

@@ -8,6 +8,8 @@
 secrets_manager_client = boto3.client('secretsmanager')
 cloudformation_client = boto3.client('cloudformation')
 logs_client = boto3.client('logs')
+cloudtrail_client = boto3.client('cloudtrail')
+s3_resource = boto3.resource('s3')
 
 # env variables
 ns1_api_key = os.environ['NS1_API_KEY']
@@ -39,6 +41,8 @@ def configure_application(event, context):
     elif cloud_sync_api_key:
         secret_handler.upsert(CS_API_KEY_NAME, cloud_sync_api_key)
 
+    retained_bucket = ""
+
     try:
         request_type = event['RequestType']
         if request_type == 'Create':
@@ -48,17 +52,90 @@ def configure_application(event, context):
             else:
                 secret_handler.upsert(NS1_API_KEY_NAME, ns1_api_key)
 
+        elif request_type == 'Update':
+            # If CreateCloudTrail is transitioning from true → false (or upgrading
+            # from a version <0.4.6 where CreateCloudTrail did not exist and the trail was always
+            # created), stop and delete the CloudSync-managed trail so it stops writing
+            # to the S3 bucket.
+            # The S3 bucket is intentionally NOT emptied or deleted here. Bucket deletion
+            # may take a long time depending on the size of the bucket, and may cause the
+            # CloudFormation stack update to timeout or fail if the deletion takes too long.
+            # The customer is warned via CloudWatch logs to empty and delete the bucket
+            # manually once they have verified DNS sync is working.
+            #
+            # Note: OldResourceProperties will not contain CreateCloudTrail when
+            # upgrading from a version <0.4.6 (the parameter did not exist in those versions and
+            # the trail was always created). Defaulting to 'true' here means the
+            # upgrade path correctly detects the true→false transition and cleans up
+            # the old trail. This is NOT the customer-facing default —
+            # the template parameter default is 'false'.
+            old_create_trail = event['OldResourceProperties'].get('CreateCloudTrail', 'true')
+            # new_create_trail uses 'true' as fallback only for safety — in practice
+            # this key will always be present in ResourceProperties from 0.4.6 onwards.
+            new_create_trail = event['ResourceProperties'].get('CreateCloudTrail', 'true')
+
+            if old_create_trail == 'true' and new_create_trail == 'false':
+                trail_name = event['OldResourceProperties'].get('CloudTrailName', 'NS1CloudSyncTrail')
+
+                # CloudTrailBucketName is not present in OldResourceProperties when
+                # upgrading from a version <0.4.6 — find the bucket by its known name prefix instead.
+                bucket_name = event['OldResourceProperties'].get('CloudTrailBucketName')
+                if not bucket_name:
+                    s3_client = boto3.client('s3')
+                    buckets = s3_client.list_buckets().get('Buckets', [])
+                    for b in buckets:
+                        if b['Name'].startswith('cloudsync-trail-bucket-'):
+                            bucket_name = b['Name']
+                            break
+
+                # Stop and delete the trail so it stops writing to the S3 bucket.
+                # Safety guard: only delete trails with names that match known
+                # CloudSync-created trail names. We never delete trails that were
+                # not created by this stack.
+                # Known CloudSync trail names across all versions:
+                #   - NS1CloudSyncTrail (default from 0.4.x)
+                #   - CloudSyncTrail (used in older deployments)
+                # Any other name (e.g. management-events, org trails) is skipped.
+                cloudsync_trail_names = {'NS1CloudSyncTrail', 'CloudSyncTrail'}
+                if trail_name in cloudsync_trail_names:
+                    try:
+                        cloudtrail_client.stop_logging(Name=trail_name)
+                        cloudtrail_client.delete_trail(Name=trail_name)
+                        print(f"Deleted CloudSync-managed trail: {trail_name}")
+                    except cloudtrail_client.exceptions.TrailNotFoundException:
+                        pass
+                else:
+                    print(f"Skipping deletion of trail not managed by CloudSync: {trail_name}")
+
+                # The S3 bucket is intentionally retained. Emptying it synchronously
+                # risks exceeding the Lambda timeout on large buckets and leaving the
+                # stack in UPDATE_ROLLBACK_FAILED. The trail has been stopped so no
+                # new objects will be written. The customer must empty and delete the
+                # bucket manually.
+                if bucket_name:
+                    retained_bucket = bucket_name
+                    print(
+                        f"WARNING: CloudTrail log bucket '{bucket_name}' has been retained. "
+                        f"The trail has been stopped — no new logs will be written. "
+                        f"Please empty and delete this bucket manually once you have "
+                        f"verified that DNS sync is working correctly. "
+                        f"You can do this from the S3 console or with the AWS CLI: "
+                        f"aws s3 rm s3://{bucket_name} --recursive && "
+                        f"aws s3api delete-bucket --bucket {bucket_name}"
+                    )
+
         elif request_type == 'Delete':
             # clean up secrets stored in Secrets Manager
             secret_handler.delete(NS1_API_KEY_NAME)
             secret_handler.delete(CS_API_KEY_NAME)
             secret_handler.delete(ACCESS_TOKEN_NAME)
             secret_handler.delete(REFRESH_TOKEN_NAME)
 
-            # empty CloudTrail bucket and remove it
-            s3_client = boto3.resource('s3')
-            bucket = s3_client.Bucket(event['ResourceProperties']['CloudTrailBucketName'])
-            bucket.objects.all().delete()
+            # empty CloudTrail bucket and remove it (only if the stack created one)
+            bucket_name = event['ResourceProperties'].get('CloudTrailBucketName')
+            if bucket_name:
+                bucket = s3_resource.Bucket(bucket_name)
+                bucket.objects.all().delete()
 
             # clean up log groups
             delete_log_groups()
@@ -68,7 +145,9 @@ def configure_application(event, context):
         response_data = build_response(event, 'FAILED', {"foo": "bar"}, reason=str(err))
 
     else:
-        response_data = build_response(event, 'SUCCESS', {"foo": "bar"})
+        response_data = build_response(event, 'SUCCESS', {
+            "RetainedCloudTrailBucket": retained_bucket,
+        })
 
     # Respond to Cloudformation to let it know we are done
     response_url = event['ResponseURL']