User Timestamp Parsing

examples/demo/demo.ts

@@ -23,6 +23,7 @@
   ParticipantEvent,
   RemoteParticipant,
   RemoteTrackPublication,
+  RemoteVideoTrack,
   Room,
   RoomEvent,
   ScreenSharePresets,
@@ -36,10 +37,12 @@
   isLocalTrack,
   isRemoteParticipant,
   isRemoteTrack,
+  isVideoTrack,
   setLogLevel,
   supportsAV1,
   supportsVP9,
 } from '../../src/index';
+import { TrackEvent } from '../../src/room/events';
 import type { DataTrackFrame } from '../../src/room/data-track/frame';
 import { isSVCCodec, sleep, supportsH265 } from '../../src/room/utils';
 
@@ -243,6 +246,43 @@
         appendLog('subscribed to track', pub.trackSid, participant.identity);
         renderParticipant(participant);
         renderScreenShare(room);
+
+        // Display the user timestamp that matches the frame currently on screen.
+        // Publish & Subscribe timestamps update every frame; latency updates at ~2 Hz
+        // so it's readable (matching the Rust subscriber's approach).
+        if (track instanceof RemoteVideoTrack) {
+          let lastLatencyUpdate = 0;
+          let cachedUserTimestampUs: number | undefined;
+          let cachedLatencyStr = '';
+
+          track.on(TrackEvent.TimeSyncUpdate, ({ rtpTimestamp }) => {
+            const timestampUs = track.lookupUserTimestamp(rtpTimestamp);
+            if (timestampUs !== undefined) {
+              cachedUserTimestampUs = timestampUs;
+            }
+            if (cachedUserTimestampUs === undefined) return;
+
+            const now = Date.now();
+
+            if (now - lastLatencyUpdate >= 500) {
+              const nowUs = now * 1000;
+              const latencyMs = (nowUs - cachedUserTimestampUs) / 1000;
+              cachedLatencyStr = `${latencyMs.toFixed(1)}ms`;
+              lastLatencyUpdate = now;
+            }
+
+            const container = getParticipantsAreaElement();
+            const tsElm = container.querySelector(`#user-ts-${participant.identity}`);
+            if (tsElm) {
+              const pubStr = new Date(cachedUserTimestampUs / 1000).toISOString().substring(11, 23);
+              const subStr = new Date(now).toISOString().substring(11, 23);
+              tsElm.innerHTML =
+                `Publish:&nbsp;&nbsp;&nbsp;${pubStr}<br>` +
+                `Subscribe:&nbsp;${subStr}<br>` +
+                `Latency:&nbsp;&nbsp;&nbsp;${cachedLatencyStr}`;
+            }
+          });
+        }
       })
       .on(RoomEvent.TrackUnsubscribed, (_, pub, participant) => {
         appendLog('unsubscribed from track', pub.trackSid);
@@ -867,6 +907,8 @@
           <span id="e2ee-${identity}" class="e2ee-on"></span>
         </div>
       </div>
+      <div id="user-ts-${identity}" class="user-ts-overlay">
+      </div>
       ${
         !isLocalParticipant(participant)
           ? `<div class="volume-control">

examples/demo/styles.css

@@ -164,3 +164,17 @@
   position: absolute;
   z-index: 4;
 }
+
+.participant .user-ts-overlay {
+  position: absolute;
+  bottom: 28px;
+  left: 0;
+  z-index: 5;
+  font-family: monospace;
+  font-size: 0.65em;
+  color: #eee;
+  background: rgba(0, 0, 0, 0.5);
+  padding: 3px 6px;
+  line-height: 1.4;
+  border-radius: 0 3px 0 0;
+}

rollup.config.worker.js

@@ -3,23 +3,38 @@ import typescript from 'rollup-plugin-typescript2';
 import packageJson from './package.json';
 import { commonPlugins, kebabCaseToPascalCase } from './rollup.config';
 
-export default {
-  input: 'src/e2ee/worker/e2ee.worker.ts',
-  output: [
-    {
-      file: `dist/${packageJson.name}.e2ee.worker.mjs`,
-      format: 'es',
-      strict: true,
-      sourcemap: true,
-    },
-    {
-      file: `dist/${packageJson.name}.e2ee.worker.js`,
-      format: 'umd',
-      strict: true,
-      sourcemap: true,
-      name: kebabCaseToPascalCase(packageJson.name) + '.e2ee.worker',
-      plugins: [terser()],
-    },
-  ],
-  plugins: [typescript({ tsconfig: './src/e2ee/worker/tsconfig.json' }), ...commonPlugins],
-};
+function workerConfig(input, suffix, umdName) {
+  return {
+    input,
+    output: [
+      {
+        file: `dist/${packageJson.name}.${suffix}.mjs`,
+        format: 'es',
+        strict: true,
+        sourcemap: true,
+      },
+      {
+        file: `dist/${packageJson.name}.${suffix}.js`,
+        format: 'umd',
+        strict: true,
+        sourcemap: true,
+        name: umdName,
+        plugins: [terser()],
+      },
+    ],
+    plugins: [typescript({ tsconfig: './src/e2ee/worker/tsconfig.json' }), ...commonPlugins],
+  };
+}
+
+export default [
+  workerConfig(
+    'src/e2ee/worker/e2ee.worker.ts',
+    'e2ee.worker',
+    kebabCaseToPascalCase(packageJson.name) + '.e2ee.worker',
+  ),
+  workerConfig(
+    'src/packet_trailer/packetTrailer.worker.ts',
+    'packet-trailer.worker',
+    kebabCaseToPascalCase(packageJson.name) + '.packetTrailer.worker',
+  ),
+];

src/e2ee/E2eeManager.ts

@@ -8,6 +8,7 @@ import { ConnectionState } from '../room/Room';
 import { DeviceUnsupportedError } from '../room/errors';
 import { EngineEvent, ParticipantEvent, RoomEvent } from '../room/events';
 import type RemoteTrack from '../room/track/RemoteTrack';
+import RemoteVideoTrack from '../room/track/RemoteVideoTrack';
 import type { Track } from '../room/track/Track';
 import type { VideoCodec } from '../room/track/options';
 import { mimeTypeToVideoCodecString } from '../room/track/utils';
@@ -221,6 +222,15 @@ export class E2EEManager
           encryptFuture.resolve(data as EncryptDataResponseMessage['data']);
         }
         break;
+      case 'packetTrailer':
+        this.handleUserTimestamp(
+          data.trackId,
+          data.participantIdentity,
+          data.timestampUs,
+          data.frameId,
+          data.rtpTimestamp,
+        );
+        break;
       default:
         break;
     }
@@ -231,6 +241,32 @@ export class E2EEManager
     this.emit(EncryptionEvent.EncryptionError, ev.error, undefined);
   };
 
+  private handleUserTimestamp(
+    trackId: string,
+    participantIdentity: string,
+    timestampUs: number,
+    frameId?: number,
+    rtpTimestamp?: number,
+  ) {
+    if (!this.room) {
+      return;
+    }
+    const participant = this.room.getParticipantByIdentity(participantIdentity);
+    if (!participant) {
+      return;
+    }
+    for (const pub of participant.trackPublications.values()) {
+      if (
+        pub.track &&
+        pub.track.mediaStreamID === trackId &&
+        pub.track instanceof RemoteVideoTrack
+      ) {
+        pub.track.setUserTimestamp(timestampUs, rtpTimestamp, frameId);
+        return;
+      }
+    }
+  }
+
   public setupEngine(engine: RTCEngine) {
     engine.on(EngineEvent.RTPVideoMapUpdate, (rtpMap) => {
       this.postRTPMap(rtpMap);

src/e2ee/types.ts

@@ -150,6 +150,17 @@ export interface EncryptDataResponseMessage extends BaseMessage {
   };
 }
 
+export interface PacketTrailerMessage extends BaseMessage {
+  kind: 'packetTrailer';
+  data: {
+    trackId: string;
+    participantIdentity: string;
+    timestampUs: number;
+    frameId?: number;
+    rtpTimestamp?: number;
+  };
+}
+
 export type E2EEWorkerMessage =
   | InitMessage
   | SetKeyMessage
@@ -166,7 +177,8 @@ export type E2EEWorkerMessage =
   | DecryptDataRequestMessage
   | DecryptDataResponseMessage
   | EncryptDataRequestMessage
-  | EncryptDataResponseMessage;
+  | EncryptDataResponseMessage
+  | PacketTrailerMessage;
 
 export type KeySet = { material: CryptoKey; encryptionKey: CryptoKey };
 

src/e2ee/worker/FrameCryptor.ts

@@ -4,10 +4,17 @@ import { EventEmitter } from 'events';
 import type TypedEventEmitter from 'typed-emitter';
 import { workerLogger } from '../../logger';
 import type { VideoCodec } from '../../room/track/options';
+import { stripPacketTrailerFromEncodedFrame } from '../../packet_trailer/PacketTrailerTransformer';
 import { ENCRYPTION_ALGORITHM, IV_LENGTH, UNENCRYPTED_BYTES } from '../constants';
 import { CryptorError, CryptorErrorReason } from '../errors';
 import { type CryptorCallbacks, CryptorEvent } from '../events';
-import type { DecodeRatchetOptions, KeyProviderOptions, KeySet, RatchetResult } from '../types';
+import type {
+  DecodeRatchetOptions,
+  KeyProviderOptions,
+  KeySet,
+  RatchetResult,
+  PacketTrailerMessage,
+} from '../types';
 import { deriveKeys, isVideoFrame, needsRbspUnescaping, parseRbsp, writeRbsp } from '../utils';
 import type { ParticipantKeyHandler } from './ParticipantKeyHandler';
 import { processNALUsForEncryption } from './naluUtils';
@@ -454,6 +461,32 @@ export class FrameCryptor extends BaseFrameCryptor {
     encodedFrame: RTCEncodedVideoFrame | RTCEncodedAudioFrame,
     controller: TransformStreamDefaultController,
   ) {
+    // Always attempt to strip LKTS packet trailer before any e2ee
+    // processing. On the send side, the trailer is appended *after* encryption,
+    // so it must be removed *before* decryption.
+    if (isVideoFrame(encodedFrame) && encodedFrame.data.byteLength > 0) {
+      try {
+        const packetTrailerResult = stripPacketTrailerFromEncodedFrame(
+          encodedFrame as RTCEncodedVideoFrame,
+        );
+        if (packetTrailerResult !== undefined && this.trackId && this.participantIdentity) {
+          const msg: PacketTrailerMessage = {
+            kind: 'packetTrailer',
+            data: {
+              trackId: this.trackId,
+              participantIdentity: this.participantIdentity,
+              timestampUs: packetTrailerResult.timestampUs,
+              frameId: packetTrailerResult.frameId,
+              rtpTimestamp: packetTrailerResult.rtpTimestamp,
+            },
+          };
+          postMessage(msg);
+        }
+      } catch {
+        // Best-effort: never break media pipeline if timestamp parsing fails.
+      }
+    }
+
     if (
       !this.isEnabled() ||
       // skip for decryption for empty dtx frames

src/index.ts

@@ -169,3 +169,13 @@ export {
 export { type DataChannelKind } from './room/RTCEngine';
 
 export { LocalTrackRecorder } from './room/track/record';
+
+export {
+  type UserFrameMetadata,
+  type PacketTrailerInfo,
+  type PacketTrailerWithRtp,
+  PACKET_TRAILER_MAGIC,
+  PACKET_TRAILER_SIZE,
+  extractPacketTrailer,
+  stripPacketTrailerFromEncodedFrame,
+} from './packet_trailer';