Fix/enhance alerting for transaction settlement errors

backend/compact-connect/app_clients/bin/create_app_client.py

@@ -125,6 +125,17 @@ def get_user_input():
     """Get user input for app client configuration."""
     print('=== App Client Configuration ===\n')
 
+    # Get environment
+    while True:
+        try:
+            print('Valid environments: test, beta, prod')
+            environment = input('Enter the environment: ').strip().lower()
+            if environment not in ['test', 'beta', 'prod']:
+                raise ValueError('Invalid environment. Must be one of: test, beta, prod')
+            break
+        except ValueError as e:
+            print(f'Error: {e}')
+
     # Get client name
     client_name = input("Enter the app client name (e.g., 'example-ky-app-client-v1'): ").strip()
     if not client_name:
@@ -183,7 +194,13 @@ def get_user_input():
         print('Configuration cancelled.')
         sys.exit(0)
 
-    return {'clientName': client_name, 'compact': compact, 'state': state, 'scopes': deduped_scopes}
+    return {
+        'environment': environment,
+        'clientName': client_name,
+        'compact': compact,
+        'state': state,
+        'scopes': deduped_scopes,
+    }
 
 
 def create_app_client(user_pool_id, config):
@@ -293,20 +310,18 @@ def print_email_template(environment, compact, state):
 
 def main():
     parser = argparse.ArgumentParser(description='Create AWS Cognito app client interactively')
-    parser.add_argument(
-        '-e', '--environment', required=True, choices=['test', 'beta', 'prod'], help='Environment (test, beta, or prod)'
-    )
     parser.add_argument('-u', '--user-pool-id', required=True, help='AWS Cognito User Pool ID')
 
     args = parser.parse_args()
 
     try:
-        print(f'Creating app client for {args.environment} environment...')
         print(f'User Pool ID: {args.user_pool_id}\n')
 
-        # Get configuration from user input
+        # Get configuration from user input (including environment)
         config = get_user_input()
 
+        print(f'\nCreating app client for {config["environment"]} environment...')
+
         # Create the app client
         response = create_app_client(args.user_pool_id, config)
 
@@ -328,7 +343,7 @@ def main():
         print_credentials(client_id, client_secret)
 
         # Print email template
-        print_email_template(args.environment, config['compact'], config['state'])
+        print_email_template(config['environment'], config['compact'], config['state'])
 
         print('\n📝 Remember to add this app client to your external registry!')
 

backend/compact-connect/lambdas/nodejs/email-notification-service/lambda.ts

@@ -79,7 +79,8 @@ export class Lambda implements LambdaInterface {
             await this.emailService.sendTransactionBatchSettlementFailureEmail(
                 event.compact,
                 event.recipientType,
-                event.specificEmails
+                event.specificEmails,
+                event.templateVariables.batchFailureErrorMessage
             );
             break;
         case 'privilegeDeactivationJurisdictionNotification':

backend/compact-connect/lambdas/nodejs/lib/email/email-notification-service.ts

@@ -57,7 +57,8 @@ export class EmailNotificationService extends BaseEmailService {
     public async sendTransactionBatchSettlementFailureEmail(
         compact: string,
         recipientType: RecipientType,
-        specificEmails?: string[]
+        specificEmails?: string[],
+        batchFailureErrorMessage?: string
     ): Promise<void> {
         this.logger.info('Sending transaction batch settlement failure email', { compact: compact });
         const recipients = await this.getCompactRecipients(compact, recipientType, specificEmails);
@@ -69,12 +70,36 @@ export class EmailNotificationService extends BaseEmailService {
         const compactConfig = await this.compactConfigurationClient.getCompactConfiguration(compact);
         const report = this.getNewEmailTemplate();
         const subject = `Transactions Failed to Settle for ${compactConfig.compactName} Payment Processor`;
-        const bodyText = 'A transaction settlement error was detected within the payment processing account for the compact. ' +
-            'Please reach out to your payment processing representative to determine the cause. ' +
-            'Transactions made in the account will not be able to be settled until the issue is addressed.';
+
+        let bodyText = 'A transaction settlement error was detected within the payment processing account for the compact. ' +
+            'Please reach out to your payment processing representative if needed to determine the cause. ';
+
+        // Include detailed error message if provided
+        if (batchFailureErrorMessage) {
+            try {
+                const errorDetails = JSON.parse(batchFailureErrorMessage);
+
+                bodyText += '\n\nDetailed Error Information:\n';
+
+                if (errorDetails.message) {
+                    bodyText += `Error Message: ${errorDetails.message}\n`;
+                }
+
+                if (errorDetails.failedTransactionIds && errorDetails.failedTransactionIds.length > 0) {
+                    bodyText += `Failed Transaction IDs: ${errorDetails.failedTransactionIds.join(', ')}\n`;
+                }
+
+                if (errorDetails.unsettledTransactionIds && errorDetails.unsettledTransactionIds.length > 0) {
+                    bodyText += `Unsettled Transaction IDs (older than 48 hours): ${errorDetails.unsettledTransactionIds.join(', ')}\n`;
+                }
+            } catch (parseError) {
+                // If JSON parsing fails, include the raw message
+                bodyText += `\n\nError Details: ${batchFailureErrorMessage}`;
+            }
+        }
 
         this.insertHeader(report, subject);
-        this.insertBody(report, bodyText);
+        this.insertBody(report, bodyText, 'center', true);
         this.insertFooter(report);
 
         const htmlContent = this.renderTemplate(report);

backend/compact-connect/lambdas/nodejs/tests/email-notification-service.test.ts

@@ -174,6 +174,64 @@ describe('EmailNotificationServiceLambda', () => {
         });
     });
 
+    it('should include detailed error information for failed transactions', async () => {
+        const eventWithFailedTransactions: EmailNotificationEvent = {
+            ...SAMPLE_EVENT,
+            templateVariables: {
+                batchFailureErrorMessage: JSON.stringify({
+                    message: 'Settlement errors detected in one or more transactions.',
+                    failedTransactionIds: ['tx-123', 'tx-456', 'tx-789']
+                })
+            }
+        };
+
+        const response = await lambda.handler(eventWithFailedTransactions, {} as any);
+
+        expect(response).toEqual({
+            message: 'Email message sent'
+        });
+
+        // Get the actual HTML content for detailed validation
+        const emailCall = mockSESClient.commandCalls(SendEmailCommand)[0];
+        const htmlContent = emailCall.args[0].input.Content?.Simple?.Body?.Html?.Data;
+
+        expect(htmlContent).toBeDefined();
+        expect(htmlContent).toContain('A transaction settlement error was detected within the payment processing account for the compact.');
+        expect(htmlContent).toContain('Please reach out to your payment processing representative if needed to determine the cause.');
+        expect(htmlContent).toContain('Detailed Error Information:');
+        expect(htmlContent).toContain('Error Message: Settlement errors detected in one or more transactions.');
+        expect(htmlContent).toContain('Failed Transaction IDs: tx-123, tx-456, tx-789');
+    });
+
+    it('should include detailed error information for unsettled transactions', async () => {
+        const eventWithUnsettledTransactions: EmailNotificationEvent = {
+            ...SAMPLE_EVENT,
+            templateVariables: {
+                batchFailureErrorMessage: JSON.stringify({
+                    message: 'One or more transactions have not settled in over 48 hours.',
+                    unsettledTransactionIds: ['unsettled-tx-001', 'unsettled-tx-002']
+                })
+            }
+        };
+
+        const response = await lambda.handler(eventWithUnsettledTransactions, {} as any);
+
+        expect(response).toEqual({
+            message: 'Email message sent'
+        });
+
+        // Get the actual HTML content for detailed validation
+        const emailCall = mockSESClient.commandCalls(SendEmailCommand)[0];
+        const htmlContent = emailCall.args[0].input.Content?.Simple?.Body?.Html?.Data;
+
+        expect(htmlContent).toBeDefined();
+        expect(htmlContent).toContain('A transaction settlement error was detected within the payment processing account for the compact.');
+        expect(htmlContent).toContain('Please reach out to your payment processing representative if needed to determine the cause.');
+        expect(htmlContent).toContain('Detailed Error Information:');
+        expect(htmlContent).toContain('Error Message: One or more transactions have not settled in over 48 hours.');
+        expect(htmlContent).toContain('Unsettled Transaction IDs (older than 48 hours): unsettled-tx-001, unsettled-tx-002');
+    });
+
     it('should throw error for unsupported template', async () => {
         const event: EmailNotificationEvent = {
             ...SAMPLE_EVENT,

backend/compact-connect/lambdas/python/common/cc_common/data_model/schema/transaction/record.py

@@ -71,3 +71,32 @@ def generate_pk_sk(self, in_data, **kwargs):
             f'#TX#{in_data["transactionId"]}'
         )
         return in_data
+
+
+@BaseRecordSchema.register_schema('unsettled_transaction')
+class UnsettledTransactionRecordSchema(BaseRecordSchema):
+    """
+    Schema for unsettled transaction records in the transaction history table.
+
+    These records track transactions that have been submitted but not yet settled,
+    allowing detection of transactions that fail to settle within the expected timeframe.
+    """
+
+    _record_type = 'unsettled_transaction'
+
+    # Required fields
+    compact = Compact(required=True, allow_none=False)
+    transactionId = String(required=True, allow_none=False)
+    transactionDate = String(required=True, allow_none=False)  # ISO datetime string
+    dateOfUpdate = String(required=True, allow_none=False)  # ISO datetime string
+
+    @pre_dump
+    def generate_pk_sk(self, in_data, **kwargs):
+        """Generate the partition key and sort key for DynamoDB."""
+        transaction_time = datetime.fromisoformat(in_data['transactionDate'])
+        # Convert to epoch timestamp for sorting
+        epoch_timestamp = int(transaction_time.timestamp())
+
+        in_data['pk'] = f'COMPACT#{in_data["compact"]}#UNSETTLED_TRANSACTIONS'
+        in_data['sk'] = f'COMPACT#{in_data["compact"]}#TIME#{epoch_timestamp}#TX#{in_data["transactionId"]}'
+        return in_data

backend/compact-connect/lambdas/python/common/cc_common/data_model/transaction_client.py

@@ -1,9 +1,9 @@
-from datetime import UTC, datetime
+from datetime import UTC, datetime, timedelta
 
 from boto3.dynamodb.conditions import Key
 
 from cc_common.config import _Config, logger
-from cc_common.data_model.schema.transaction.record import TransactionRecordSchema
+from cc_common.data_model.schema.transaction.record import TransactionRecordSchema, UnsettledTransactionRecordSchema
 
 AUTHORIZE_DOT_NET_CLIENT_TYPE = 'authorize.net'
 
@@ -231,3 +231,110 @@ def add_privilege_information_to_transactions(self, compact: str, transactions:
                     )
 
         return transactions
+
+    def store_unsettled_transaction(self, compact: str, transaction_id: str, transaction_date: str) -> None:
+        """
+        Store an unsettled transaction record in DynamoDB.
+
+        :param compact: The compact abbreviation
+        :param transaction_id: The transaction ID from the payment processor
+        :param transaction_date: ISO datetime string of when the transaction was submitted
+        """
+        try:
+            # Create the record data
+            record_data = {
+                'compact': compact,
+                'transactionId': transaction_id,
+                'transactionDate': transaction_date,
+                'dateOfUpdate': datetime.now(UTC).isoformat(),
+            }
+
+            # Validate and serialize using the schema
+            unsettled_schema = UnsettledTransactionRecordSchema()
+            serialized_record = unsettled_schema.dump(record_data)
+
+            self.config.transaction_history_table.put_item(Item=serialized_record)
+            logger.info(
+                'Stored unsettled transaction record',
+                compact=compact,
+                transaction_id=transaction_id,
+            )
+        except Exception as e:  # noqa: BLE001
+            # This record is created for monitoring unsettled transactions, not business critical
+            # If we fail to record it for whatever reason, log error but don't raise an exception
+            logger.error(
+                'Failed to store unsettled transaction record',
+                compact=compact,
+                transaction_id=transaction_id,
+                error=str(e),
+            )
+
+    def reconcile_unsettled_transactions(self, compact: str, settled_transactions: list[dict]) -> list[str]:
+        """
+        Reconcile unsettled transactions with settled transactions and detect old unsettled transactions.
+
+        This method:
+        1. Queries all unsettled transactions for the compact
+        2. Matches them with settled transactions by transaction ID
+        3. Deletes matched unsettled transactions
+        4. Checks for unsettled transactions older than 48 hours
+
+        :param compact: The compact abbreviation
+        :param settled_transactions: List of settled transaction records
+        :return: List of transaction IDs that have not been matched and are older than 48 hours
+        (empty list if none found)
+        """
+        # Query all unsettled transactions for this compact
+        pk = f'COMPACT#{compact}#UNSETTLED_TRANSACTIONS'
+        response = self.config.transaction_history_table.query(
+            KeyConditionExpression=Key('pk').eq(pk),
+        )
+
+        unsettled_transactions = response.get('Items', [])
+
+        if not unsettled_transactions:
+            logger.info('No unsettled transactions found for compact', compact=compact)
+            return []
+
+        # Create a set of settled transaction IDs for efficient lookup
+        settled_transaction_ids = {tx['transactionId'] for tx in settled_transactions}
+
+        # Separate matched and unmatched unsettled transactions
+        matched_unsettled = []
+        unmatched_unsettled = []
+
+        for unsettled_tx in unsettled_transactions:
+            if unsettled_tx['transactionId'] in settled_transaction_ids:
+                matched_unsettled.append(unsettled_tx)
+            else:
+                unmatched_unsettled.append(unsettled_tx)
+
+        # Batch delete matched unsettled transactions
+        if matched_unsettled:
+            logger.info(
+                'Deleting matched unsettled transactions',
+                compact=compact,
+                count=len(matched_unsettled),
+                settled_transaction_ids=settled_transaction_ids
+            )
+            with self.config.transaction_history_table.batch_writer() as batch:
+                for tx in matched_unsettled:
+                    batch.delete_item(Key={'pk': tx['pk'], 'sk': tx['sk']})
+
+        # Check for unsettled transactions older than 48 hours
+        cutoff_time = datetime.now(UTC) - timedelta(hours=48)
+        old_unsettled_transactions = []
+
+        for unsettled_tx in unmatched_unsettled:
+            transaction_date = datetime.fromisoformat(unsettled_tx['transactionDate'])
+            if transaction_date < cutoff_time:
+                old_unsettled_transactions.append(unsettled_tx['transactionId'])
+
+        if old_unsettled_transactions:
+            logger.warning(
+                'Found unsettled transactions older than 48 hours',
+                compact=compact,
+                old_transaction_ids=old_unsettled_transactions,
+            )
+
+        return old_unsettled_transactions