From 06b2e8bb2048d4562df7f5bdbed80edc9a5722a7 Mon Sep 17 00:00:00 2001
From: Timo <16718859+toger5@users.noreply.github.com>
Date: Thu, 19 Jun 2025 16:42:19 +0200
Subject: [PATCH] Fixes from ios debugging session: (#3342)

- dont use preferred vs selected concept in controlled media. Its not needed since we dont use the id for actual browser media devices (the id's are not even actual browser media devices)
  - add more logging
  - add more conditions to not accidently set a deviceId that is not a browser deviceId but one provided via controlled.
---
 src/controls.ts           |  6 ++++++
 src/livekit/useLivekit.ts | 13 +++++++++----
 src/state/MediaDevices.ts | 17 +++++++----------
 3 files changed, 22 insertions(+), 14 deletions(-)

diff --git a/src/controls.ts b/src/controls.ts
index ff0ccefc..ba816348 100644
--- a/src/controls.ts
+++ b/src/controls.ts
@@ -6,6 +6,9 @@ Please see LICENSE in the repository root for full details.
 */
 
 import { Subject } from "rxjs";
+import { logger as rootLogger } from "matrix-js-sdk/lib/logger";
+
+const logger = rootLogger.getChild("[controlled-output]");
 
 export interface Controls {
   canEnterPip(): boolean;
@@ -74,12 +77,15 @@ window.controls = {
     setPipEnabled$.next(false);
   },
   setAvailableAudioDevices(devices: OutputDevice[]): void {
+    logger.info("setAvailableAudioDevices called from native:", devices);
     availableOutputDevices$.next(devices);
   },
   setAudioDevice(id: string): void {
+    logger.info("setAudioDevice called from native", id);
     outputDevice$.next(id);
   },
   setAudioEnabled(enabled: boolean): void {
+    logger.info("setAudioEnabled called from native:", enabled);
     if (!setAudioEnabled$.observed)
       throw new Error(
         "Output controls are disabled. No setAudioEnabled$ observer",
diff --git a/src/livekit/useLivekit.ts b/src/livekit/useLivekit.ts
index 8f2ed4ac..58f088f6 100644
--- a/src/livekit/useLivekit.ts
+++ b/src/livekit/useLivekit.ts
@@ -64,7 +64,7 @@ export function useLivekit(
   const initialMuteStates = useInitial(() => muteStates);
 
   const devices = useMediaDevices();
-  const initialAudioInput = useInitial(
+  const initialAudioInputId = useInitial(
     () => getValue(devices.audioInput.selected$)?.id,
   );
 
@@ -109,10 +109,15 @@ export function useLivekit(
       },
       audioCaptureDefaults: {
         ...defaultLiveKitOptions.audioCaptureDefaults,
-        deviceId: initialAudioInput,
+        deviceId: initialAudioInputId,
       },
       audioOutput: {
-        deviceId: getValue(devices.audioOutput.selected$)?.id,
+        // When using controlled audio devices, we don't want to set the
+        // deviceId here, because it will be set by the native app.
+        // (also the id does not need to match a browser device id)
+        deviceId: controlledAudioDevices
+          ? undefined
+          : getValue(devices.audioOutput.selected$)?.id,
       },
       e2ee,
     };
@@ -167,7 +172,7 @@ export function useLivekit(
   );
 
   const connectionState = useECConnectionState(
-    initialAudioInput,
+    initialAudioInputId,
     initialMuteStates.audio.enabled,
     room,
     sfuConfig,
diff --git a/src/state/MediaDevices.ts b/src/state/MediaDevices.ts
index 8aa279f3..873cc1fc 100644
--- a/src/state/MediaDevices.ts
+++ b/src/state/MediaDevices.ts
@@ -28,8 +28,8 @@ import {
 } from "../settings/settings";
 import { type ObservableScope } from "./ObservableScope";
 import {
-  outputDevice$ as externalDeviceSelection$,
-  availableOutputDevices$,
+  outputDevice$ as controlledOutputSelection$,
+  availableOutputDevices$ as controlledAvailableOutputDevices$,
 } from "../controls";
 import { getUrlParams } from "../UrlParams";
 
@@ -239,7 +239,7 @@ class ControlledAudioOutput
   implements MediaDevice<AudioOutputDeviceLabel, SelectedAudioOutputDevice>
 {
   public readonly available$ = combineLatest(
-    [availableOutputDevices$.pipe(startWith([])), iosDeviceMenu$],
+    [controlledAvailableOutputDevices$.pipe(startWith([])), iosDeviceMenu$],
     (availableRaw, iosDeviceMenu) => {
       const available = new Map<string, AudioOutputDeviceLabel>(
         availableRaw.map(
@@ -269,15 +269,11 @@ class ControlledAudioOutput
     this.deviceSelection$.next(id);
   }
 
-  private readonly preferredDevice$ = merge(
+  public readonly selected$ = merge(
     this.deviceSelection$,
-    externalDeviceSelection$,
-  ).pipe(startWith<string | undefined>(undefined), this.scope.state());
-
-  public readonly selected$ = selectDevice$(
-    this.available$,
-    this.preferredDevice$,
+    controlledOutputSelection$,
   ).pipe(
+    startWith<string | undefined>(undefined),
     map((id) =>
       id === undefined
         ? undefined
@@ -293,6 +289,7 @@ class ControlledAudioOutput
       // been selected - for example, Element X iOS listens to this to determine
       // whether it should enable the proximity sensor.
       if (device !== undefined) {
+        logger.info("[controlled-output] setAudioDeviceSelect called:", device);
         window.controls.onAudioDeviceSelect?.(device.id);
         // Also invoke the deprecated callback for backward compatibility
         window.controls.onOutputDeviceSelect?.(device.id);