Support newer Ray-Ban Meta glasses: migrate DAT SDK 0.4.0 → 0.7.0

samples/CameraAccess/CameraAccess.xcodeproj/project.pbxproj

@@ -726,7 +726,7 @@
 			repositoryURL = "https://github.com/facebook/meta-wearables-dat-ios";
 			requirement = {
 				kind = exactVersion;
-				version = 0.4.0;
+				version = 0.7.0;
 			};
 		};
 		9DD6CAFC2F3C62DA00ED7098 /* XCRemoteSwiftPackageReference "WebRTC" */ = {

samples/CameraAccess/CameraAccess/Info.plist

@@ -53,6 +53,8 @@
 	<key>UIBackgroundModes</key>
 	<array>
 		<string>audio</string>
+		<string>processing</string>
+		<string>bluetooth-central</string>
 		<string>bluetooth-peripheral</string>
 		<string>external-accessory</string>
 	</array>
@@ -71,6 +73,12 @@
 	<string>This app uses the microphone to have voice conversations with the AI assistant while streaming from your glasses.</string>
 	<key>NSPhotoLibraryAddUsageDescription</key>
 	<string>This app needs access to save photos captured from your glasses.</string>
+	<key>NSLocalNetworkUsageDescription</key>
+	<string>This allows your phone to find and connect to your glasses over Wi-Fi.</string>
+	<key>NSBonjourServices</key>
+	<array>
+		<string>_bonjour._tcp</string>
+	</array>
 	<key>NSAppTransportSecurity</key>
 	<dict>
 		<key>NSAllowsLocalNetworking</key>

samples/CameraAccess/CameraAccess/ViewModels/StreamSessionViewModel.swift

@@ -67,8 +67,12 @@ class StreamSessionViewModel: ObservableObject {
   // WebRTC Live streaming integration
   var webrtcSessionVM: WebRTCSessionViewModel?
 
-  // The core DAT SDK StreamSession - handles all streaming operations
-  private var streamSession: StreamSession
+  // DAT SDK 0.7.0 session-based model: a DeviceSession owns the device connection, and a
+  // camera Stream (added to a started session) produces video frames + photos. Both are
+  // created on demand in startSession() because addStream() requires a started DeviceSession.
+  private var deviceSession: DeviceSession?
+  private var stream: MWDATCamera.Stream?
+  private var sessionErrorListenerToken: AnyListenerToken?
   // Listener tokens are used to manage DAT SDK event subscriptions
   private var stateListenerToken: AnyListenerToken?
   private var videoFrameListenerToken: AnyListenerToken?
@@ -90,11 +94,6 @@ class StreamSessionViewModel: ObservableObject {
     self.wearables = wearables
     // Let the SDK auto-select from available devices
     self.deviceSelector = AutoDeviceSelector(wearables: wearables)
-    let config = StreamSessionConfig(
-      videoCodec: VideoCodec.raw,
-      resolution: StreamingResolution.low,
-      frameRate: 24)
-    streamSession = StreamSession(streamSessionConfig: config, deviceSelector: deviceSelector)
 
     // Monitor device availability
     deviceMonitorTask = Task { @MainActor in
@@ -104,7 +103,8 @@ class StreamSessionViewModel: ObservableObject {
     }
 
     setupVideoDecoder()
-    attachListeners()
+    // Session + camera Stream (and their listeners) are created on start, not here —
+    // addStream() requires a started DeviceSession in the 0.7.0 model.
   }
 
   private func setupVideoDecoder() {
@@ -134,18 +134,13 @@ class StreamSessionViewModel: ObservableObject {
   func updateResolution(_ resolution: StreamingResolution) {
     guard !isStreaming else { return }
     selectedResolution = resolution
-    let config = StreamSessionConfig(
-      videoCodec: VideoCodec.raw,
-      resolution: resolution,
-      frameRate: 24)
-    streamSession = StreamSession(streamSessionConfig: config, deviceSelector: deviceSelector)
-    attachListeners()
+    // The StreamConfiguration is applied when the camera Stream is created on the next start.
     NSLog("[Stream] Resolution changed to %@", resolutionLabel)
   }
 
-  private func attachListeners() {
+  private func attachStreamListeners(to stream: MWDATCamera.Stream) {
     // Subscribe to session state changes using the DAT SDK listener pattern
-    stateListenerToken = streamSession.statePublisher.listen { [weak self] state in
+    stateListenerToken = stream.statePublisher.listen { [weak self] state in
       Task { @MainActor [weak self] in
         self?.updateStatusFromState(state)
       }
@@ -154,7 +149,7 @@ class StreamSessionViewModel: ObservableObject {
     // Subscribe to video frames from the device camera
     // This callback fires whether the app is in the foreground or background,
     // enabling continuous streaming even when the screen is locked.
-    videoFrameListenerToken = streamSession.videoFramePublisher.listen { [weak self] videoFrame in
+    videoFrameListenerToken = stream.videoFramePublisher.listen { [weak self] videoFrame in
       Task { @MainActor [weak self] in
         guard let self else { return }
 
@@ -208,7 +203,7 @@ class StreamSessionViewModel: ObservableObject {
     }
 
     // Subscribe to streaming errors
-    errorListenerToken = streamSession.errorPublisher.listen { [weak self] error in
+    errorListenerToken = stream.errorPublisher.listen { [weak self] error in
       Task { @MainActor [weak self] in
         guard let self else { return }
         // Suppress device-not-found errors when user hasn't started streaming yet
@@ -223,10 +218,10 @@ class StreamSessionViewModel: ObservableObject {
       }
     }
 
-    updateStatusFromState(streamSession.state)
+    updateStatusFromState(stream.state)
 
     // Subscribe to photo capture events
-    photoDataListenerToken = streamSession.photoDataPublisher.listen { [weak self] photoData in
+    photoDataListenerToken = stream.photoDataPublisher.listen { [weak self] photoData in
       Task { @MainActor [weak self] in
         guard let self else { return }
         if let uiImage = UIImage(data: photoData.data) {
@@ -257,7 +252,63 @@ class StreamSessionViewModel: ObservableObject {
   }
 
   func startSession() async {
-    await streamSession.start()
+    do {
+      // 1) Create a device session bound to the auto-selected wearable.
+      let session = try wearables.createSession(deviceSelector: deviceSelector)
+      self.deviceSession = session
+      attachSessionListeners(to: session)
+
+      // 2) Get the state stream BEFORE start() (it doesn't buffer past events), start the
+      //    session, then wait until it reaches .started. addStream() returns nil if the
+      //    device session hasn't finished connecting yet (Meta's own sample does this).
+      let stateStream = session.stateStream()
+      try session.start()
+      if session.state != .started {
+        for await state in stateStream {
+          if state == .started || state == .stopped { break }
+        }
+      }
+      guard session.state == .started else {
+        showError("The glasses session didn't finish connecting. Please try again.")
+        cleanupSession()
+        return
+      }
+
+      // 3) Add a camera stream and wire its publishers, then start it.
+      let config = StreamConfiguration(
+        videoCodec: VideoCodec.raw,
+        resolution: selectedResolution,
+        frameRate: 24)
+      guard let stream = try session.addStream(config: config) else {
+        showError("Failed to create a camera stream from the glasses.")
+        cleanupSession()
+        return
+      }
+      self.stream = stream
+      attachStreamListeners(to: stream)
+      await stream.start()
+    } catch {
+      showError(formatDeviceSessionError(error))
+      cleanupSession()
+    }
+  }
+
+  /// Subscribe to device-session-level state + error (connection lifecycle), separate from the
+  /// camera Stream's own state/frames.
+  private func attachSessionListeners(to session: DeviceSession) {
+    sessionErrorListenerToken = session.errorPublisher.listen { [weak self] error in
+      Task { @MainActor [weak self] in
+        guard let self else { return }
+        if self.streamingStatus != .stopped {
+          self.showError(self.formatDeviceSessionError(error))
+        }
+      }
+    }
+  }
+
+  private func cleanupSession() {
+    stream = nil
+    deviceSession = nil
   }
 
   private func showError(_ message: String) {
@@ -270,7 +321,9 @@ class StreamSessionViewModel: ObservableObject {
       stopIPhoneSession()
       return
     }
-    await streamSession.stop()
+    await stream?.stop()
+    deviceSession?.stop()
+    cleanupSession()
   }
 
   // MARK: - iPhone Camera Mode
@@ -320,15 +373,15 @@ class StreamSessionViewModel: ObservableObject {
   }
 
   func capturePhoto() {
-    streamSession.capturePhoto(format: .jpeg)
+    _ = stream?.capturePhoto(format: .jpeg)
   }
 
   func dismissPhotoPreview() {
     showPhotoPreview = false
     capturedPhoto = nil
   }
 
-  private func updateStatusFromState(_ state: StreamSessionState) {
+  private func updateStatusFromState(_ state: StreamState) {
     switch state {
     case .stopped:
       currentVideoFrame = nil
@@ -340,7 +393,7 @@ class StreamSessionViewModel: ObservableObject {
     }
   }
 
-  private func formatStreamingError(_ error: StreamSessionError) -> String {
+  private func formatStreamingError(_ error: StreamError) -> String {
     switch error {
     case .internalError:
       return "An internal error occurred. Please try again."
@@ -352,8 +405,6 @@ class StreamSessionViewModel: ObservableObject {
       return "The operation timed out. Please try again."
     case .videoStreamingError:
       return "Video streaming failed. Please try again."
-    case .audioStreamingError:
-      return "Audio streaming failed. Please try again."
     case .permissionDenied:
       return "Camera permission denied. Please grant permission in Settings."
     case .hingesClosed:
@@ -362,4 +413,25 @@ class StreamSessionViewModel: ObservableObject {
       return "An unknown streaming error occurred."
     }
   }
+
+  /// Map 0.7.0 device-session errors to a user-facing message. noEligibleDevice /
+  /// datAppOnTheGlassesUpdateRequired are the common "glasses won't attach" cases.
+  private func formatDeviceSessionError(_ error: DeviceSessionError) -> String {
+    switch error {
+    case .noEligibleDevice:
+      return "No compatible glasses were found. Make sure your glasses are connected in the Meta AI app and camera access is granted."
+    case .datAppOnTheGlassesUpdateRequired:
+      return "Your glasses need a software update. Open the Meta AI app and update your glasses, then try again."
+    case .sessionAlreadyExists, .capabilityAlreadyActive:
+      return "A session is already active. Stop streaming and try again."
+    case .thermalCritical, .thermalEmergency:
+      return "The glasses are too warm to stream right now. Let them cool down and try again."
+    case .batteryCritical:
+      return "The glasses' battery is too low to stream. Charge them and try again."
+    case .unexpectedError(let description):
+      return "Failed to start session: \(description)"
+    @unknown default:
+      return "Failed to start the glasses session. Please try again."
+    }
+  }
 }

samples/CameraAccess/CameraAccess/ViewModels/WearablesViewModel.swift

@@ -33,6 +33,7 @@ class WearablesViewModel: ObservableObject {
 
   private var registrationTask: Task<Void, Never>?
   private var deviceStreamTask: Task<Void, Never>?
+  private var didRequestCameraPermission = false
   private var setupDeviceStreamTask: Task<Void, Never>?
   private let wearables: WearablesInterface
   private var compatibilityListenerTokens: [DeviceIdentifier: AnyListenerToken] = [:]
@@ -55,6 +56,13 @@ class WearablesViewModel: ObservableObject {
         if self.showGettingStartedSheet == false && registrationState == .registered && previousState == .registering {
           self.showGettingStartedSheet = true
         }
+        // Per Meta DAT docs: a wearable will NOT appear in devicesStream until at least one
+        // permission (camera) is granted via the Meta AI app. The stock app only requests it
+        // inside handleStartStreaming(), which is gated behind a button disabled until a device
+        // appears — a deadlock. Request it as soon as we're registered so the glasses show up.
+        if registrationState == .registered {
+          requestCameraPermissionIfNeeded()
+        }
       }
     }
   }
@@ -73,6 +81,10 @@ class WearablesViewModel: ObservableObject {
     deviceStreamTask = Task {
       for await devices in wearables.devicesStream() {
         self.devices = devices
+        // Already-registered launch with no devices yet: still need the camera permission grant.
+        if devices.isEmpty && self.registrationState == .registered {
+          requestCameraPermissionIfNeeded()
+        }
         #if canImport(MWDATMockDevice)
         self.hasMockDevice = !MockDeviceKit.shared.pairedDevices.isEmpty
         #endif
@@ -106,6 +118,23 @@ class WearablesViewModel: ObservableObject {
     }
   }
 
+  /// Request glasses camera permission via the Meta AI app. Required for the wearable to appear
+  /// in devicesStream at all (per Meta DAT docs). Guarded so it only fires once per session.
+  func requestCameraPermissionIfNeeded() {
+    guard !didRequestCameraPermission else { return }
+    didRequestCameraPermission = true
+    Task { @MainActor in
+      do {
+        let status = try await wearables.checkPermissionStatus(Permission.camera)
+        if status != .granted {
+          _ = try await wearables.requestPermission(Permission.camera)
+        }
+      } catch {
+        self.didRequestCameraPermission = false  // allow a retry on error
+      }
+    }
+  }
+
   func connectGlasses() {
     guard registrationState != .registering else { return }
     Task { @MainActor in