CCM-14320: Set up TraceContext

lambdas/file-scanner-lambda/src/apis/sqs-handler.ts

@@ -55,6 +55,7 @@ async function processRecord(
     messageReference: event.data.messageReference,
     senderId: event.data.senderId,
     createdAt: event.time,
+    traceparent: event.traceparent,
   });
 
   if (result.outcome === 'failed') {

lambdas/file-scanner-lambda/src/app/file-scanner.ts

@@ -24,6 +24,7 @@ export interface ScanFileMetadata {
   messageReference: string;
   senderId: string;
   createdAt: string;
+  traceparent: string;
 }
 
 export type ScanFileResult = {
@@ -158,6 +159,7 @@ export class FileScanner {
         messageReference: metadata.messageReference,
         senderId: metadata.senderId,
         createdAt: metadata.createdAt,
+        traceparent: metadata.traceparent,
       },
     };
 

lambdas/mesh-acknowledge/requirements.txt

@@ -3,6 +3,8 @@ boto3>=1.28.62
 pyopenssl>=24.2.1
 pydantic>=2.0.0
 structlog>=21.5.0
+opentelemetry-api>=1.25.0
+opentelemetry-sdk>=1.25.0
 -e ../../src/digital-letters-events
 -e ../../utils/py-mock-mesh
 -e ../../utils/py-utils

lambdas/mesh-download/mesh_download/processor.py

@@ -27,7 +27,10 @@ def __init__(self, **kwargs):
     def process_sqs_message(self, sqs_record):
         try:
             validated_event = self._parse_and_validate_event(sqs_record)
-            logger = self.__log.bind(mesh_message_id=validated_event.data.meshMessageId)
+            logger = self.__log.bind(
+                mesh_message_id=validated_event.data.meshMessageId,
+                traceparent=validated_event.traceparent,
+            )
 
             logger.info("Processing MESH download request")
             return self._handle_download(validated_event, logger)
@@ -146,6 +149,8 @@ def _store_message_content(self, sender_id, message_reference, mesh_message_id,
     def _publish_downloaded_event(self, incoming_event, message_uri):
         """
         Publishes a MESHInboxMessageDownloaded event.
+        The EventPublisher will derive a child traceparent from the incoming traceparent
+        automatically, preserving the trace-id across this service hop.
         """
         now = datetime.now(timezone.utc).isoformat()
 
@@ -179,7 +184,9 @@ def _publish_downloaded_event(self, incoming_event, message_uri):
             "Published MESHInboxMessageDownloaded event",
             sender_id=incoming_event.data.senderId,
             message_uri=message_uri,
-            message_reference=incoming_event.data.messageReference
+            message_reference=incoming_event.data.messageReference,
+            incoming_traceparent=incoming_event.traceparent,
+            outgoing_traceparent=cloud_event['traceparent'],
         )
 
     def _publish_message_invalid_event(self, incoming_event, failure_code: str):

lambdas/mesh-download/requirements.txt

@@ -3,6 +3,8 @@ structlog>=21.5.0
 pydantic>=2.0.0
 boto3>=1.28.62
 pyopenssl>=24.2.1
+opentelemetry-api>=1.25.0
+opentelemetry-sdk>=1.25.0
 nhs-notify-digital-letters-onboarding @ git+https://github.com/NHSDigital/nhs-notify-digital-letters-onboarding@0.1.0
 -e ../../src/digital-letters-events
 -e ../../utils/py-mock-mesh

lambdas/mesh-poll/mesh_poll/processor.py

@@ -150,6 +150,7 @@ def process_message(self, message):
     def _publish_mesh_inbox_message_received_event(self, event_detail):
         """
         Publishes a MESHInboxMessageReceived event for the retriever component.
+        The EventPublisher will create a root traceparent automatically.
         """
         now = datetime.now(timezone.utc).isoformat()
 
@@ -168,7 +169,6 @@ def _publish_mesh_inbox_message_received_event(self, event_detail):
             'recordedtime': now,
             'severitynumber': 2,
             'severitytext': 'INFO',
-            'traceparent': '00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01',
             'dataschema': (
                 'https://notify.nhs.uk/cloudevents/schemas/digital-letters/'
                 '2025-10-draft/data/digital-letters-mesh-inbox-message-received-data.schema.json'
@@ -186,11 +186,13 @@ def _publish_mesh_inbox_message_received_event(self, event_detail):
 
         self.__log.info("Published MESHInboxMessageReceived event",
                         mesh_message_id=event_detail["data"]["meshMessageId"],
-                        sender_id=event_detail["data"]["senderId"])
+                        sender_id=event_detail["data"]["senderId"],
+                        traceparent=cloud_event['traceparent'])
 
     def _publish_mesh_inbox_message_invalid_event(self, event_detail):
         """
         Publishes a MESHInboxMessageInvalid event when a message fails validation.
+        The EventPublisher will create a root traceparent automatically.
         """
         now = datetime.now(timezone.utc).isoformat()
 
@@ -206,7 +208,6 @@ def _publish_mesh_inbox_message_invalid_event(self, event_detail):
             'recordedtime': now,
             'severitynumber': 3,
             'severitytext': 'WARN',
-            'traceparent': '00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01',
             'dataschema': (
                 'https://notify.nhs.uk/cloudevents/schemas/digital-letters/'
                 '2025-10-draft/data/digital-letters-mesh-inbox-message-invalid-data.schema.json'

lambdas/mesh-poll/requirements.txt

@@ -3,6 +3,8 @@ structlog>=21.5.0
 boto3>=1.28.62
 pyopenssl>=24.2.1
 pydantic>=2.0.0
+opentelemetry-api>=1.25.0
+opentelemetry-sdk>=1.25.0
 -e ../../src/digital-letters-events
 -e ../../utils/py-mock-mesh
 -e ../../utils/py-utils

lambdas/move-scanned-files-lambda/src/app/move-file-handler.ts

@@ -19,6 +19,7 @@ type ObjectMetadata = {
   senderId: string;
   messageReference: string;
   createdAt: string;
+  traceparent?: string;
 };
 
 type ObjectsFromEvent = {
@@ -34,6 +35,7 @@ const SCAN_RESULT_STATUS_NO_THREATS_FOUND = 'NO_THREATS_FOUND';
 const METADATA_CREATED_AT = 'createdat';
 const METADATA_MESSAGE_REFERENCE = 'messagereference';
 const METADATA_SENDER_ID = 'senderid';
+const METADATA_TRACEPARENT = 'traceparent';
 
 /**
  * Utility function to extract a subset of the object key for logging purposes.
@@ -121,6 +123,7 @@ export class MoveFileHandler {
     const createdAt = metadataMap.get(METADATA_CREATED_AT);
     const messageReference = metadataMap.get(METADATA_MESSAGE_REFERENCE);
     const senderId = metadataMap.get(METADATA_SENDER_ID);
+    const traceparent = metadataMap.get(METADATA_TRACEPARENT);
 
     if (!messageReference || !senderId || !createdAt) {
       return null;
@@ -137,6 +140,7 @@ export class MoveFileHandler {
       senderId,
       messageReference,
       createdAt,
+      traceparent,
     };
   }
 
@@ -160,6 +164,7 @@ export class MoveFileHandler {
         metadata.senderId,
         `s3://${destinationBucket}/${objectKey}`,
         metadata.createdAt,
+        metadata.traceparent,
       );
     } else {
       destinationBucket = this.quarantineBucketName;
@@ -176,6 +181,7 @@ export class MoveFileHandler {
         metadata.senderId,
         `s3://${destinationBucket}/${objectKey}`,
         metadata.createdAt,
+        metadata.traceparent,
       );
     }
 

lambdas/move-scanned-files-lambda/src/domain/mapper.ts

@@ -6,6 +6,7 @@ export function createFileSafeEvent(
   senderId: string,
   letterUri: string,
   createdAt: string,
+  traceparent?: string,
 ): FileSafe {
   return {
     specversion: '1.0',
@@ -17,7 +18,7 @@ export function createFileSafeEvent(
     dataschemaversion: '1.0.0',
     datacontenttype: 'application/json',
     source: '/nhs/england/notify/production/primary/digitalletters/print', // Note CCM-13892.
-    traceparent: '00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01', // Note CCM-14255.
+    traceparent: traceparent ?? '',
     type: 'uk.nhs.notify.digital.letters.print.file.safe.v1',
     time: new Date().toISOString(),
     data: {
@@ -36,6 +37,7 @@ export function createFileQuarantinedEvent(
   senderId: string,
   letterUri: string,
   createdAt: string,
+  traceparent?: string,
 ): FileQuarantined {
   return {
     specversion: '1.0',
@@ -47,7 +49,7 @@ export function createFileQuarantinedEvent(
     dataschemaversion: '1.0.0',
     datacontenttype: 'application/json',
     source: '/nhs/england/notify/production/primary/digitalletters/print', // Note CCM-13892.
-    traceparent: '00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01', // Note CCM-14255.
+    traceparent: traceparent ?? '',
     type: 'uk.nhs.notify.digital.letters.print.file.quarantined.v1',
     time: new Date().toISOString(),
     data: {

lambdas/pdm-poll-lambda/src/__tests__/apis/sqs-handler.test.ts

@@ -15,13 +15,24 @@ const pdm = mock<Pdm>();
 
 jest.mock('node:crypto', () => ({
   randomUUID: jest.fn(),
+  randomBytes: jest.fn(),
 }));
 
+// eslint-disable-next-line @typescript-eslint/no-require-imports
+const { randomBytes } = require('node:crypto');
+
 const mockRandomUUID = randomUUID as jest.MockedFunction<typeof randomUUID>;
+const mockRandomBytes = randomBytes as jest.MockedFunction<
+  typeof import('node:crypto').randomBytes
+>;
 const mockDate = jest.spyOn(Date.prototype, 'toISOString');
 mockRandomUUID.mockReturnValue('550e8400-e29b-41d4-a716-446655440001');
+mockRandomBytes.mockImplementation(() => Buffer.from('aabbccdd11223344', 'hex'));
 mockDate.mockReturnValue('2023-06-20T12:00:00.250Z');
 
+const DERIVED_TRACEPARENT =
+  '00-0af7651916cd43dd8448eb211c80319c-aabbccdd11223344-01';
+
 const handler = createHandler({
   eventPublisher,
   logger,
@@ -51,6 +62,7 @@ describe('SQS Handler', () => {
             id: '550e8400-e29b-41d4-a716-446655440001',
             time: '2023-06-20T12:00:00.250Z',
             recordedtime: '2023-06-20T12:00:00.250Z',
+            traceparent: DERIVED_TRACEPARENT,
             dataschema:
               'https://notify.nhs.uk/cloudevents/schemas/digital-letters/2025-10-draft/data/digital-letters-pdm-resource-available-data.schema.json',
             type: 'uk.nhs.notify.digital.letters.pdm.resource.available.v1',
@@ -90,6 +102,7 @@ describe('SQS Handler', () => {
             id: '550e8400-e29b-41d4-a716-446655440001',
             time: '2023-06-20T12:00:00.250Z',
             recordedtime: '2023-06-20T12:00:00.250Z',
+            traceparent: DERIVED_TRACEPARENT,
             dataschema:
               'https://notify.nhs.uk/cloudevents/schemas/digital-letters/2025-10-draft/data/digital-letters-pdm-resource-unavailable-data.schema.json',
             type: 'uk.nhs.notify.digital.letters.pdm.resource.unavailable.v1',
@@ -132,6 +145,7 @@ describe('SQS Handler', () => {
             id: '550e8400-e29b-41d4-a716-446655440001',
             time: '2023-06-20T12:00:00.250Z',
             recordedtime: '2023-06-20T12:00:00.250Z',
+            traceparent: DERIVED_TRACEPARENT,
             dataschema:
               'https://notify.nhs.uk/cloudevents/schemas/digital-letters/2025-10-draft/data/digital-letters-pdm-resource-available-data.schema.json',
             type: 'uk.nhs.notify.digital.letters.pdm.resource.available.v1',
@@ -174,6 +188,7 @@ describe('SQS Handler', () => {
             id: '550e8400-e29b-41d4-a716-446655440001',
             time: '2023-06-20T12:00:00.250Z',
             recordedtime: '2023-06-20T12:00:00.250Z',
+            traceparent: DERIVED_TRACEPARENT,
             type: 'uk.nhs.notify.digital.letters.pdm.resource.unavailable.v1',
             data: {
               messageReference:
@@ -219,6 +234,7 @@ describe('SQS Handler', () => {
             id: '550e8400-e29b-41d4-a716-446655440001',
             time: '2023-06-20T12:00:00.250Z',
             recordedtime: '2023-06-20T12:00:00.250Z',
+            traceparent: DERIVED_TRACEPARENT,
             dataschema:
               'https://notify.nhs.uk/cloudevents/schemas/digital-letters/2025-10-draft/data/digital-letters-pdm-resource-retries-exceeded-data.schema.json',
             type: 'uk.nhs.notify.digital.letters.pdm.resource.retries.exceeded.v1',

lambdas/pdm-poll-lambda/src/apis/sqs-handler.ts

@@ -169,16 +169,30 @@ export const createHandler = ({
 
           if (pdmAvailability === 'unavailable') {
             if (retries >= pollMaxRetries) {
-              retriesExceededEvents.push(
-                generateRetriesExceededEvent(event, retries),
-              );
+              const outgoing = generateRetriesExceededEvent(event, retries);
+              logger.info({
+                description: 'TraceContext hop',
+                incoming_traceparent: event.traceparent,
+                outgoing_traceparent: outgoing.traceparent,
+              });
+              retriesExceededEvents.push(outgoing);
             } else {
-              unavailableEvents.push(generateUnavailableEvent(event, retries));
+              const outgoing = generateUnavailableEvent(event, retries);
+              logger.info({
+                description: 'TraceContext hop',
+                incoming_traceparent: event.traceparent,
+                outgoing_traceparent: outgoing.traceparent,
+              });
+              unavailableEvents.push(outgoing);
             }
           } else {
-            availableEvents.push(
-              generateAvailableEvent(event, nhsNumber, odsCode),
-            );
+            const outgoing = generateAvailableEvent(event, nhsNumber, odsCode);
+            logger.info({
+              description: 'TraceContext hop',
+              incoming_traceparent: event.traceparent,
+              outgoing_traceparent: outgoing.traceparent,
+            });
+            availableEvents.push(outgoing);
           }
         } catch (error: any) {
           logger.warn({

lambdas/report-sender/requirements.txt

@@ -3,6 +3,8 @@ structlog>=21.5.0
 boto3>=1.28.62
 pyopenssl>=24.2.1
 pydantic>=2.0.0
+opentelemetry-api>=1.25.0
+opentelemetry-sdk>=1.25.0
 -e ../../src/digital-letters-events
 -e ../../utils/py-mock-mesh
 -e ../../utils/py-utils