make audio work

Signed-off-by: Timo K <toger5@hotmail.de>

src/livekit/MatrixAudioRenderer.test.tsx

@@ -18,7 +18,7 @@ import { useTracks } from "@livekit/components-react";
 
 import { testAudioContext } from "../useAudioContext.test";
 import * as MediaDevicesContext from "../MediaDevicesContext";
-import { MatrixAudioRenderer } from "./MatrixAudioRenderer";
+import { LivekitRoomAudioRenderer } from "./MatrixAudioRenderer";
 import { mockMediaDevices, mockTrack } from "../utils/test";
 
 export const TestAudioContextConstructor = vi.fn(() => testAudioContext);
@@ -54,7 +54,7 @@ vi.mocked(useTracks).mockReturnValue(tracks);
 it("should render for member", () => {
   const { container, queryAllByTestId } = render(
     <MediaDevicesProvider value={mockMediaDevices({})}>
-      <MatrixAudioRenderer
+      <LivekitRoomAudioRenderer
         members={[{ sender: "test", deviceId: "123" }] as CallMembership[]}
       />
     </MediaDevicesProvider>,
@@ -69,7 +69,7 @@ it("should not render without member", () => {
   ] as CallMembership[];
   const { container, queryAllByTestId } = render(
     <MediaDevicesProvider value={mockMediaDevices({})}>
-      <MatrixAudioRenderer members={memberships} />
+      <LivekitRoomAudioRenderer members={memberships} />
     </MediaDevicesProvider>,
   );
   expect(container).toBeTruthy();
@@ -79,7 +79,7 @@ it("should not render without member", () => {
 it("should not setup audioContext gain and pan if there is no need to.", () => {
   render(
     <MediaDevicesProvider value={mockMediaDevices({})}>
-      <MatrixAudioRenderer
+      <LivekitRoomAudioRenderer
         members={[{ sender: "test", deviceId: "123" }] as CallMembership[]}
       />
     </MediaDevicesProvider>,
@@ -102,7 +102,7 @@ it("should setup audioContext gain and pan", () => {
   });
   render(
     <MediaDevicesProvider value={mockMediaDevices({})}>
-      <MatrixAudioRenderer
+      <LivekitRoomAudioRenderer
         members={[{ sender: "test", deviceId: "123" }] as CallMembership[]}
       />
     </MediaDevicesProvider>,

src/livekit/MatrixAudioRenderer.tsx

@@ -6,6 +6,7 @@ Please see LICENSE in the repository root for full details.
 */
 
 import { getTrackReferenceId } from "@livekit/components-core";
+import { type Room as LivekitRoom } from "livekit-client";
 import { type RemoteAudioTrack, Track } from "livekit-client";
 import { useEffect, useMemo, useRef, useState, type ReactNode } from "react";
 import {
@@ -19,7 +20,7 @@ import { logger } from "matrix-js-sdk/lib/logger";
 import { useEarpieceAudioConfig } from "../MediaDevicesContext";
 import { useReactiveState } from "../useReactiveState";
 import * as controls from "../controls";
-
+import {} from "@livekit/components-core";
 export interface MatrixAudioRendererProps {
   /**
    * The list of participants to render audio for.
@@ -27,6 +28,7 @@ export interface MatrixAudioRendererProps {
    * that are not expected to be in the rtc session.
    */
   members: CallMembership[];
+  livekitRoom: LivekitRoom;
   /**
    * If set to `true`, mutes all audio tracks rendered by the component.
    * @remarks
@@ -49,9 +51,10 @@ export interface MatrixAudioRendererProps {
  * ```
  * @public
  */
-export function MatrixAudioRenderer({
+export function LivekitRoomAudioRenderer({
   members,
   muted,
+  livekitRoom,
 }: MatrixAudioRendererProps): ReactNode {
   const validIdentities = useMemo(
     () =>
@@ -78,6 +81,8 @@ export function MatrixAudioRenderer({
     loggedInvalidIdentities.current.add(identity);
   };
 
+  // TODO-MULTI-SFU this uses the livekit room form the context. We need to change it so it uses the
+  // livekit room explicitly so we can pass a list of rooms into the audio renderer and call useTracks for each room.
   const tracks = useTracks(
     [
       Track.Source.Microphone,
@@ -87,6 +92,7 @@ export function MatrixAudioRenderer({
     {
       updateOnlyOn: [],
       onlySubscribed: true,
+      room: livekitRoom,
     },
   ).filter((ref) => {
     const isValid = validIdentities?.has(ref.participant.identity);

src/livekit/livekitSubscriptionRoom.ts

@@ -0,0 +1,123 @@
+/*
+Copyright 2023, 2024 New Vector Ltd.
+
+SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
+Please see LICENSE in the repository root for full details.
+*/
+
+import {
+  ConnectionState,
+  type E2EEManagerOptions,
+  ExternalE2EEKeyProvider,
+  LocalVideoTrack,
+  Room,
+  type RoomOptions,
+} from "livekit-client";
+import { useEffect, useRef } from "react";
+import E2EEWorker from "livekit-client/e2ee-worker?worker";
+import { logger } from "matrix-js-sdk/lib/logger";
+import { type MatrixRTCSession } from "matrix-js-sdk/lib/matrixrtc";
+
+import { defaultLiveKitOptions } from "./options";
+import { type SFUConfig } from "./openIDSFU";
+import { type MuteStates } from "../room/MuteStates";
+import { useMediaDevices } from "../MediaDevicesContext";
+import {
+  type ECConnectionState,
+  useECConnectionState,
+} from "./useECConnectionState";
+import { MatrixKeyProvider } from "../e2ee/matrixKeyProvider";
+import { E2eeType } from "../e2ee/e2eeType";
+import { type EncryptionSystem } from "../e2ee/sharedKeyManagement";
+import {
+  useTrackProcessor,
+  useTrackProcessorSync,
+} from "./TrackProcessorContext";
+import { observeTrackReference$ } from "../state/MediaViewModel";
+import { useUrlParams } from "../UrlParams";
+import { useInitial } from "../useInitial";
+import { getValue } from "../utils/observable";
+import { type SelectedDevice } from "../state/MediaDevices";
+
+interface UseLivekitResult {
+  livekitPublicationRoom?: Room;
+  connState: ECConnectionState;
+}
+
+// TODO-MULTI-SFU This is all the logic we need in the subscription connection logic (sync output devices)
+// This is not used! (but summarizes what we need)
+export function livekitSubscriptionRoom(
+  rtcSession: MatrixRTCSession,
+  muteStates: MuteStates,
+  sfuConfig: SFUConfig | undefined,
+  e2eeSystem: EncryptionSystem,
+): UseLivekitResult {
+  // Only ever create the room once via useInitial.
+  // The call can end up with multiple livekit rooms. This is the particular room in
+  // which this participant publishes their media.
+  const publicationRoom = useInitial(() => {
+    logger.info("[LivekitRoom] Create LiveKit room");
+
+    let e2ee: E2EEManagerOptions | undefined;
+    if (e2eeSystem.kind === E2eeType.PER_PARTICIPANT) {
+      logger.info("Created MatrixKeyProvider (per participant)");
+      e2ee = {
+        keyProvider: new MatrixKeyProvider(),
+        worker: new E2EEWorker(),
+      };
+    } else if (e2eeSystem.kind === E2eeType.SHARED_KEY && e2eeSystem.secret) {
+      logger.info("Created ExternalE2EEKeyProvider (shared key)");
+      e2ee = {
+        keyProvider: new ExternalE2EEKeyProvider(),
+        worker: new E2EEWorker(),
+      };
+    }
+
+    const roomOptions: RoomOptions = {
+      ...defaultLiveKitOptions,
+      audioOutput: {
+        // When using controlled audio devices, we don't want to set the
+        // deviceId here, because it will be set by the native app.
+        // (also the id does not need to match a browser device id)
+        deviceId: controlledAudioDevices
+          ? undefined
+          : getValue(devices.audioOutput.selected$)?.id,
+      },
+      e2ee,
+    };
+    // We have to create the room manually here due to a bug inside
+    // @livekit/components-react. JSON.stringify() is used in deps of a
+    // useEffect() with an argument that references itself, if E2EE is enabled
+    const room = new Room(roomOptions);
+    room.setE2EEEnabled(e2eeSystem.kind !== E2eeType.NONE).catch((e) => {
+      logger.error("Failed to set E2EE enabled on room", e);
+    });
+
+    return room;
+  });
+
+  // Setup and update the keyProvider which was create by `createRoom`
+  useEffect(() => {
+    const e2eeOptions = publicationRoom.options.e2ee;
+    if (
+      e2eeSystem.kind === E2eeType.NONE ||
+      !(e2eeOptions && "keyProvider" in e2eeOptions)
+    )
+      return;
+
+    if (e2eeSystem.kind === E2eeType.PER_PARTICIPANT) {
+      (e2eeOptions.keyProvider as MatrixKeyProvider).setRTCSession(rtcSession);
+    } else if (e2eeSystem.kind === E2eeType.SHARED_KEY && e2eeSystem.secret) {
+      (e2eeOptions.keyProvider as ExternalE2EEKeyProvider)
+        .setKey(e2eeSystem.secret)
+        .catch((e) => {
+          logger.error("Failed to set shared key for E2EE", e);
+        });
+    }
+  }, [publicationRoom.options.e2ee, e2eeSystem, rtcSession]);
+
+  return {
+    connState: connectionState,
+    livekitPublicationRoom: publicationRoom,
+  };
+}

src/livekit/openIDSFU.ts

@@ -7,12 +7,7 @@ Please see LICENSE in the repository root for full details.
 
 import { type IOpenIDToken, type MatrixClient } from "matrix-js-sdk";
 import { logger } from "matrix-js-sdk/lib/logger";
-import { type MatrixRTCSession } from "matrix-js-sdk/lib/matrixrtc";
-import { useEffect, useState } from "react";
-import { type LivekitFocus } from "matrix-js-sdk/lib/matrixrtc";
 
-import { useActiveLivekitFocus } from "../room/useActiveFocus";
-import { useErrorBoundary } from "../useErrorBoundary";
 import { FailToGetOpenIdToken } from "../utils/errors";
 import { doNetworkOperationWithRetry } from "../utils/matrix";
 
@@ -34,38 +29,11 @@ export type OpenIDClientParts = Pick<
   "getOpenIdToken" | "getDeviceId"
 >;
 
-export function useOpenIDSFU(
-  client: OpenIDClientParts,
-  rtcSession: MatrixRTCSession,
-): SFUConfig | undefined {
-  const [sfuConfig, setSFUConfig] = useState<SFUConfig | undefined>(undefined);
-
-  const activeFocus = useActiveLivekitFocus(rtcSession);
-  const { showErrorBoundary } = useErrorBoundary();
-
-  useEffect(() => {
-    if (activeFocus) {
-      getSFUConfigWithOpenID(client, activeFocus).then(
-        (sfuConfig) => {
-          setSFUConfig(sfuConfig);
-        },
-        (e) => {
-          showErrorBoundary(new FailToGetOpenIdToken(e));
-          logger.error("Failed to get SFU config", e);
-        },
-      );
-    } else {
-      setSFUConfig(undefined);
-    }
-  }, [client, activeFocus, showErrorBoundary]);
-
-  return sfuConfig;
-}
-
 export async function getSFUConfigWithOpenID(
   client: OpenIDClientParts,
-  activeFocus: LivekitFocus,
-): Promise<SFUConfig | undefined> {
+  serviceUrl: string,
+  livekitAlias: string,
+): Promise<SFUConfig> {
   let openIdToken: IOpenIDToken;
   try {
     openIdToken = await doNetworkOperationWithRetry(async () =>
@@ -78,26 +46,16 @@ export async function getSFUConfigWithOpenID(
   }
   logger.debug("Got openID token", openIdToken);
 
-  try {
-    logger.info(
-      `Trying to get JWT from call's active focus URL of ${activeFocus.livekit_service_url}...`,
-    );
-    const sfuConfig = await getLiveKitJWT(
-      client,
-      activeFocus.livekit_service_url,
-      activeFocus.livekit_alias,
-      openIdToken,
-    );
-    logger.info(`Got JWT from call's active focus URL.`);
+  logger.info(`Trying to get JWT for focus ${serviceUrl}...`);
+  const sfuConfig = await getLiveKitJWT(
+    client,
+    serviceUrl,
+    livekitAlias,
+    openIdToken,
+  );
+  logger.info(`Got JWT from call's active focus URL.`);
 
-    return sfuConfig;
-  } catch (e) {
-    logger.warn(
-      `Failed to get JWT from RTC session's active focus URL of ${activeFocus.livekit_service_url}.`,
-      e,
-    );
-    return undefined;
-  }
+  return sfuConfig;
 }
 
 async function getLiveKitJWT(

src/livekit/useECConnectionState.ts

@@ -1,3 +1,4 @@
+// TODO not used anymore - remove
 /*
 Copyright 2023, 2024 New Vector Ltd.
 

src/livekit/useLivekit.ts

@@ -49,11 +49,12 @@ import { getValue } from "../utils/observable";
 import { type SelectedDevice } from "../state/MediaDevices";
 
 interface UseLivekitResult {
-  livekitRoom?: Room;
+  livekitPublicationRoom?: Room;
   connState: ECConnectionState;
 }
 
-export function useLivekit(
+// TODO-MULTI-SFU This is not used anymore but the device syncing logic needs to be moved into the connection object.
+export function useLivekitPublicationRoom(
   rtcSession: MatrixRTCSession,
   muteStates: MuteStates,
   sfuConfig: SFUConfig | undefined,
@@ -82,7 +83,9 @@ export function useLivekit(
   const { processor } = useTrackProcessor();
 
   // Only ever create the room once via useInitial.
-  const room = useInitial(() => {
+  // The call can end up with multiple livekit rooms. This is the particular room in
+  // which this participant publishes their media.
+  const publicationRoom = useInitial(() => {
     logger.info("[LivekitRoom] Create LiveKit room");
 
     let e2ee: E2EEManagerOptions | undefined;
@@ -134,7 +137,7 @@ export function useLivekit(
 
   // Setup and update the keyProvider which was create by `createRoom`
   useEffect(() => {
-    const e2eeOptions = room.options.e2ee;
+    const e2eeOptions = publicationRoom.options.e2ee;
     if (
       e2eeSystem.kind === E2eeType.NONE ||
       !(e2eeOptions && "keyProvider" in e2eeOptions)
@@ -150,7 +153,7 @@ export function useLivekit(
           logger.error("Failed to set shared key for E2EE", e);
         });
     }
-  }, [room.options.e2ee, e2eeSystem, rtcSession]);
+  }, [publicationRoom.options.e2ee, e2eeSystem, rtcSession]);
 
   // Sync the requested track processors with LiveKit
   useTrackProcessorSync(
@@ -169,7 +172,7 @@ export function useLivekit(
               return track instanceof LocalVideoTrack ? track : null;
             }),
           ),
-        [room],
+        [publicationRoom],
       ),
     ),
   );
@@ -177,7 +180,7 @@ export function useLivekit(
   const connectionState = useECConnectionState(
     initialAudioInputId,
     initialMuteStates.audio.enabled,
-    room,
+    publicationRoom,
     sfuConfig,
   );
 
@@ -188,8 +191,11 @@ export function useLivekit(
     // It's important that we only do this in the connected state, because
     // LiveKit's internal mute states aren't consistent during connection setup,
     // and setting tracks to be enabled during this time causes errors.
-    if (room !== undefined && connectionState === ConnectionState.Connected) {
-      const participant = room.localParticipant;
+    if (
+      publicationRoom !== undefined &&
+      connectionState === ConnectionState.Connected
+    ) {
+      const participant = publicationRoom.localParticipant;
       // Always update the muteButtonState Ref so that we can read the current
       // state in awaited blocks.
       buttonEnabled.current = {
@@ -247,7 +253,7 @@ export function useLivekit(
                 audioMuteUpdating.current = true;
                 trackPublication = await participant.setMicrophoneEnabled(
                   buttonEnabled.current.audio,
-                  room.options.audioCaptureDefaults,
+                  publicationRoom.options.audioCaptureDefaults,
                 );
                 audioMuteUpdating.current = false;
                 break;
@@ -255,7 +261,7 @@ export function useLivekit(
                 videoMuteUpdating.current = true;
                 trackPublication = await participant.setCameraEnabled(
                   buttonEnabled.current.video,
-                  room.options.videoCaptureDefaults,
+                  publicationRoom.options.videoCaptureDefaults,
                 );
                 videoMuteUpdating.current = false;
                 break;
@@ -319,11 +325,14 @@ export function useLivekit(
         logger.error("Failed to sync video mute state with LiveKit", e);
       });
     }
-  }, [room, muteStates, connectionState]);
+  }, [publicationRoom, muteStates, connectionState]);
 
   useEffect(() => {
     // Sync the requested devices with LiveKit's devices
-    if (room !== undefined && connectionState === ConnectionState.Connected) {
+    if (
+      publicationRoom !== undefined &&
+      connectionState === ConnectionState.Connected
+    ) {
       const syncDevice = (
         kind: MediaDeviceKind,
         selected$: Observable<SelectedDevice | undefined>,
@@ -331,15 +340,15 @@ export function useLivekit(
         selected$.subscribe((device) => {
           logger.info(
             "[LivekitRoom] syncDevice room.getActiveDevice(kind) !== d.id :",
-            room.getActiveDevice(kind),
+            publicationRoom.getActiveDevice(kind),
             " !== ",
             device?.id,
           );
           if (
             device !== undefined &&
-            room.getActiveDevice(kind) !== device.id
+            publicationRoom.getActiveDevice(kind) !== device.id
           ) {
-            room
+            publicationRoom
               .switchActiveDevice(kind, device.id)
               .catch((e) =>
                 logger.error(`Failed to sync ${kind} device with LiveKit`, e),
@@ -365,7 +374,7 @@ export function useLivekit(
           .pipe(switchMap((device) => device?.hardwareDeviceChange$ ?? NEVER))
           .subscribe(() => {
             const activeMicTrack = Array.from(
-              room.localParticipant.audioTrackPublications.values(),
+              publicationRoom.localParticipant.audioTrackPublications.values(),
             ).find((d) => d.source === Track.Source.Microphone)?.track;
 
             if (
@@ -380,7 +389,7 @@ export function useLivekit(
               // getUserMedia() call with deviceId: default to get the *new* default device.
               // Note that room.switchActiveDevice() won't work: Livekit will ignore it because
               // the deviceId hasn't changed (was & still is default).
-              room.localParticipant
+              publicationRoom.localParticipant
                 .getTrackPublication(Track.Source.Microphone)
                 ?.audioTrack?.restartTrack()
                 .catch((e) => {
@@ -394,10 +403,10 @@ export function useLivekit(
         for (const s of subscriptions) s?.unsubscribe();
       };
     }
-  }, [room, devices, connectionState, controlledAudioDevices]);
+  }, [publicationRoom, devices, connectionState, controlledAudioDevices]);
 
   return {
     connState: connectionState,
-    livekitRoom: room,
+    livekitPublicationRoom: publicationRoom,
   };
 }

src/main.tsx

@@ -60,9 +60,9 @@ if (fatalError !== null) {
 Initializer.initBeforeReact()
   .then(() => {
     root.render(
-      <StrictMode>
-        <App vm={new AppViewModel()} />
-      </StrictMode>,
+      // <StrictMode>
+      <App vm={new AppViewModel()} />,
+      // </StrictMode>,
     );
   })
   .catch((e) => {

src/room/GroupCallView.tsx

@@ -38,10 +38,10 @@ import { PosthogAnalytics } from "../analytics/PosthogAnalytics";
 import { useProfile } from "../profile/useProfile";
 import { findDeviceByName } from "../utils/media";
 import { ActiveCall } from "./InCallView";
-import { MUTE_PARTICIPANT_COUNT, type MuteStates } from "./MuteStates";
+import { type MuteStates } from "../state/MuteStates";
 import { useMediaDevices } from "../MediaDevicesContext";
 import { useMatrixRTCSessionMemberships } from "../useMatrixRTCSessionMemberships";
-import { enterRTCSession, leaveRTCSession } from "../rtcSessionHelpers";
+import { leaveRTCSession } from "../rtcSessionHelpers";
 import {
   saveKeyForRoom,
   useRoomEncryptionSystem,
@@ -73,6 +73,12 @@ import { muteAllAudio$ } from "../state/MuteAllAudioModel.ts";
 import { useAppBarTitle } from "../AppBar.tsx";
 import { useBehavior } from "../useBehavior.ts";
 
+/**
+ * If there already are this many participants in the call, we automatically mute
+ * the user.
+ */
+export const MUTE_PARTICIPANT_COUNT = 8;
+
 declare global {
   interface Window {
     rtcSession?: MatrixRTCSession;
@@ -87,7 +93,8 @@ interface Props {
   skipLobby: boolean;
   header: HeaderStyle;
   rtcSession: MatrixRTCSession;
-  isJoined: boolean;
+  joined: boolean;
+  setJoined: (value: boolean) => void;
   muteStates: MuteStates;
   widget: WidgetHelpers | null;
 }
@@ -100,7 +107,8 @@ export const GroupCallView: FC<Props> = ({
   skipLobby,
   header,
   rtcSession,
-  isJoined,
+  joined,
+  setJoined,
   muteStates,
   widget,
 }) => {
@@ -121,7 +129,7 @@ export const GroupCallView: FC<Props> = ({
   // This should use `useEffectEvent` (only available in experimental versions)
   useEffect(() => {
     if (memberships.length >= MUTE_PARTICIPANT_COUNT)
-      muteStates.audio.setEnabled?.(false);
+      muteStates.audio.setEnabled$.value?.(false);
     // eslint-disable-next-line react-hooks/exhaustive-deps
   }, []);
 
@@ -210,12 +218,14 @@ export const GroupCallView: FC<Props> = ({
   const enterRTCSessionOrError = useCallback(
     async (rtcSession: MatrixRTCSession): Promise<void> => {
       try {
-        await enterRTCSession(
-          rtcSession,
-          perParticipantE2EE,
-          useNewMembershipManager,
-          useExperimentalToDeviceTransport,
-        );
+        setJoined(true);
+        // TODO-MULTI-SFU what to do with error handling now that we don't use this function?
+        // await enterRTCSession(
+        //   rtcSession,
+        //   perParticipantE2EE,
+        //   useNewMembershipManager,
+        //   useExperimentalToDeviceTransport,
+        // );
       } catch (e) {
         if (e instanceof ElementCallError) {
           setExternalError(e);
@@ -227,12 +237,9 @@ export const GroupCallView: FC<Props> = ({
           setExternalError(error);
         }
       }
+      return Promise.resolve();
     },
-    [
-      perParticipantE2EE,
-      useExperimentalToDeviceTransport,
-      useNewMembershipManager,
-    ],
+    [setJoined],
   );
 
   useEffect(() => {
@@ -251,7 +258,7 @@ export const GroupCallView: FC<Props> = ({
         if (!deviceId) {
           logger.warn("Unknown audio input: " + audioInput);
           // override the default mute state
-          latestMuteStates.current!.audio.setEnabled?.(false);
+          latestMuteStates.current!.audio.setEnabled$.value?.(false);
         } else {
           logger.debug(
             `Found audio input ID ${deviceId} for name ${audioInput}`,
@@ -265,7 +272,7 @@ export const GroupCallView: FC<Props> = ({
         if (!deviceId) {
           logger.warn("Unknown video input: " + videoInput);
           // override the default mute state
-          latestMuteStates.current!.video.setEnabled?.(false);
+          latestMuteStates.current!.video.setEnabled$.value?.(false);
         } else {
           logger.debug(
             `Found video input ID ${deviceId} for name ${videoInput}`,
@@ -284,7 +291,7 @@ export const GroupCallView: FC<Props> = ({
               await defaultDeviceSetup(
                 ev.detail.data as unknown as JoinCallData,
               );
-              await enterRTCSessionOrError(rtcSession);
+              setJoined(true);
               widget.api.transport.reply(ev.detail, {});
             })().catch((e) => {
               logger.error("Error joining RTC session", e);
@@ -297,13 +304,14 @@ export const GroupCallView: FC<Props> = ({
         } else {
           // No lobby and no preload: we enter the rtc session right away
           (async (): Promise<void> => {
-            await enterRTCSessionOrError(rtcSession);
+            setJoined(true);
+            return Promise.resolve();
           })().catch((e) => {
             logger.error("Error joining RTC session", e);
           });
         }
       } else {
-        void enterRTCSessionOrError(rtcSession);
+        setJoined(true);
       }
     }
   }, [
@@ -314,7 +322,7 @@ export const GroupCallView: FC<Props> = ({
     perParticipantE2EE,
     mediaDevices,
     latestMuteStates,
-    enterRTCSessionOrError,
+    setJoined,
     useNewMembershipManager,
   ]);
 
@@ -341,6 +349,7 @@ export const GroupCallView: FC<Props> = ({
         window.setTimeout(resolve, 10);
       });
 
+      // TODO-MULTI-SFU find a solution if this is supposed to happen here or in the view model.
       leaveRTCSession(
         rtcSession,
         cause,
@@ -373,7 +382,7 @@ export const GroupCallView: FC<Props> = ({
   );
 
   useEffect(() => {
-    if (widget && isJoined) {
+    if (widget && joined) {
       // set widget to sticky once joined.
       widget.api.setAlwaysOnScreen(true).catch((e) => {
         logger.error("Error calling setAlwaysOnScreen(true)", e);
@@ -391,7 +400,7 @@ export const GroupCallView: FC<Props> = ({
         widget.lazyActions.off(ElementWidgetActions.HangupCall, onHangup);
       };
     }
-  }, [widget, isJoined, rtcSession]);
+  }, [widget, joined, rtcSession]);
 
   const joinRule = useJoinRule(room);
 
@@ -426,7 +435,7 @@ export const GroupCallView: FC<Props> = ({
         client={client}
         matrixInfo={matrixInfo}
         muteStates={muteStates}
-        onEnter={() => void enterRTCSessionOrError(rtcSession)}
+        onEnter={() => setJoined(true)}
         confineToRoom={confineToRoom}
         hideHeader={header === HeaderStyle.None}
         participantCount={participantCount}
@@ -444,7 +453,7 @@ export const GroupCallView: FC<Props> = ({
       throw externalError;
     };
     body = <ErrorComponent />;
-  } else if (isJoined) {
+  } else if (joined) {
     body = (
       <>
         {shareModal}

src/room/InCallView.test.tsx

@@ -45,7 +45,7 @@ import {
 } from "../settings/settings";
 import { ReactionsSenderProvider } from "../reactions/useReactionsSender";
 import { useRoomEncryptionSystem } from "../e2ee/sharedKeyManagement";
-import { MatrixAudioRenderer } from "../livekit/MatrixAudioRenderer";
+import { LivekitRoomAudioRenderer } from "../livekit/MatrixAudioRenderer";
 import { MediaDevicesContext } from "../MediaDevicesContext";
 import { HeaderStyle } from "../UrlParams";
 
@@ -88,7 +88,7 @@ beforeEach(() => {
 
   // MatrixAudioRenderer is tested separately.
   (
-    MatrixAudioRenderer as MockedFunction<typeof MatrixAudioRenderer>
+    LivekitRoomAudioRenderer as MockedFunction<typeof LivekitRoomAudioRenderer>
   ).mockImplementation((_props) => {
     return <div>mocked: MatrixAudioRenderer</div>;
   });

src/room/InCallView.tsx

@@ -5,9 +5,7 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
 Please see LICENSE in the repository root for full details.
 */
 
-import { RoomContext, useLocalParticipant } from "@livekit/components-react";
 import { IconButton, Text, Tooltip } from "@vector-im/compound-web";
-import { ConnectionState, type Room as LivekitRoom } from "livekit-client";
 import { type MatrixClient, type Room as MatrixRoom } from "matrix-js-sdk";
 import {
   type FC,
@@ -55,15 +53,12 @@ import { type OTelGroupCallMembership } from "../otel/OTelGroupCallMembership";
 import { SettingsModal, defaultSettingsTab } from "../settings/SettingsModal";
 import { useRageshakeRequestModal } from "../settings/submit-rageshake";
 import { RageshakeRequestModal } from "./RageshakeRequestModal";
-import { useLivekit } from "../livekit/useLivekit.ts";
 import { useWakeLock } from "../useWakeLock";
 import { useMergedRefs } from "../useMergedRefs";
-import { type MuteStates } from "./MuteStates";
+import { type MuteStates } from "../state/MuteStates";
 import { type MatrixInfo } from "./VideoPreview";
 import { InviteButton } from "../button/InviteButton";
 import { LayoutToggle } from "./LayoutToggle";
-import { type ECConnectionState } from "../livekit/useECConnectionState";
-import { useOpenIDSFU } from "../livekit/openIDSFU";
 import {
   CallViewModel,
   type GridMode,
@@ -102,11 +97,9 @@ import {
   useSetting,
 } from "../settings/settings";
 import { ReactionsReader } from "../reactions/ReactionsReader";
-import { ConnectionLostError } from "../utils/errors.ts";
 import { useTypedEventEmitter } from "../useEvents.ts";
-import { MatrixAudioRenderer } from "../livekit/MatrixAudioRenderer.tsx";
+import { LivekitRoomAudioRenderer } from "../livekit/MatrixAudioRenderer.tsx";
 import { muteAllAudio$ } from "../state/MuteAllAudioModel.ts";
-import { useMatrixRTCSessionMemberships } from "../useMatrixRTCSessionMemberships.ts";
 import { useMediaDevices } from "../MediaDevicesContext.ts";
 import { EarpieceOverlay } from "./EarpieceOverlay.tsx";
 import { useAppBarHidden, useAppBarSecondaryButton } from "../AppBar.tsx";
@@ -124,87 +117,44 @@ export interface ActiveCallProps
 
 export const ActiveCall: FC<ActiveCallProps> = (props) => {
   const mediaDevices = useMediaDevices();
-  const sfuConfig = useOpenIDSFU(props.client, props.rtcSession);
-  const { livekitRoom, connState } = useLivekit(
-    props.rtcSession,
-    props.muteStates,
-    sfuConfig,
-    props.e2eeSystem,
-  );
-  const connStateObservable$ = useObservable(
-    (inputs$) => inputs$.pipe(map(([connState]) => connState)),
-    [connState],
-  );
   const [vm, setVm] = useState<CallViewModel | null>(null);
 
-  useEffect(() => {
-    logger.info(
-      `[Lifecycle] InCallView Component mounted, livekit room state ${livekitRoom?.state}`,
-    );
-    return (): void => {
-      logger.info(
-        `[Lifecycle] InCallView Component unmounted, livekit room state ${livekitRoom?.state}`,
-      );
-      livekitRoom
-        ?.disconnect()
-        .then(() => {
-          logger.info(
-            `[Lifecycle] Disconnected from livekit room, state:${livekitRoom?.state}`,
-          );
-        })
-        .catch((e) => {
-          logger.error("[Lifecycle] Failed to disconnect from livekit room", e);
-        });
-    };
-  }, [livekitRoom]);
-
   const { autoLeaveWhenOthersLeft } = useUrlParams();
 
   useEffect(() => {
-    if (livekitRoom !== undefined) {
-      const reactionsReader = new ReactionsReader(props.rtcSession);
-      const vm = new CallViewModel(
-        props.rtcSession,
-        props.matrixRoom,
-        livekitRoom,
-        mediaDevices,
-        {
-          encryptionSystem: props.e2eeSystem,
-          autoLeaveWhenOthersLeft,
-        },
-        connStateObservable$,
-        reactionsReader.raisedHands$,
-        reactionsReader.reactions$,
-      );
-      setVm(vm);
-      return (): void => {
-        vm.destroy();
-        reactionsReader.destroy();
-      };
-    }
+    const reactionsReader = new ReactionsReader(props.rtcSession);
+    const vm = new CallViewModel(
+      props.rtcSession,
+      props.matrixRoom,
+      mediaDevices,
+      props.muteStates,
+      {
+        encryptionSystem: props.e2eeSystem,
+        autoLeaveWhenOthersLeft,
+      },
+      reactionsReader.raisedHands$,
+      reactionsReader.reactions$,
+    );
+    setVm(vm);
+    return (): void => {
+      vm.destroy();
+      reactionsReader.destroy();
+    };
   }, [
     props.rtcSession,
     props.matrixRoom,
-    livekitRoom,
     mediaDevices,
+    props.muteStates,
     props.e2eeSystem,
-    connStateObservable$,
     autoLeaveWhenOthersLeft,
   ]);
 
-  if (livekitRoom === undefined || vm === null) return null;
+  if (vm === null) return null;
 
   return (
-    <RoomContext value={livekitRoom}>
-      <ReactionsSenderProvider vm={vm} rtcSession={props.rtcSession}>
-        <InCallView
-          {...props}
-          vm={vm}
-          livekitRoom={livekitRoom}
-          connState={connState}
-        />
-      </ReactionsSenderProvider>
-    </RoomContext>
+    <ReactionsSenderProvider vm={vm} rtcSession={props.rtcSession}>
+      <InCallView {...props} vm={vm} />
+    </ReactionsSenderProvider>
   );
 };
 
@@ -214,14 +164,12 @@ export interface InCallViewProps {
   matrixInfo: MatrixInfo;
   rtcSession: MatrixRTCSession;
   matrixRoom: MatrixRoom;
-  livekitRoom: LivekitRoom;
   muteStates: MuteStates;
   participantCount: number;
   /** Function to call when the user explicitly ends the call */
   onLeave: () => void;
   header: HeaderStyle;
   otelGroupCallMembership?: OTelGroupCallMembership;
-  connState: ECConnectionState;
   onShareClick: (() => void) | null;
 }
 
@@ -231,12 +179,10 @@ export const InCallView: FC<InCallViewProps> = ({
   matrixInfo,
   rtcSession,
   matrixRoom,
-  livekitRoom,
   muteStates,
   participantCount,
   onLeave,
   header: headerStyle,
-  connState,
   onShareClick,
 }) => {
   const { t } = useTranslation();
@@ -245,11 +191,6 @@ export const InCallView: FC<InCallViewProps> = ({
 
   useWakeLock();
 
-  // annoyingly we don't get the disconnection reason this way,
-  // only by listening for the emitted event
-  if (connState === ConnectionState.Disconnected)
-    throw new ConnectionLostError();
-
   const containerRef1 = useRef<HTMLDivElement | null>(null);
   const [containerRef2, bounds] = useMeasure();
   // Merge the refs so they can attach to the same element
@@ -257,10 +198,6 @@ export const InCallView: FC<InCallViewProps> = ({
 
   const { hideScreensharing, showControls } = useUrlParams();
 
-  const { isScreenShareEnabled, localParticipant } = useLocalParticipant({
-    room: livekitRoom,
-  });
-
   const muteAllAudio = useBehavior(muteAllAudio$);
 
   // This seems like it might be enough logic to use move it into the call view model?
@@ -276,7 +213,6 @@ export const InCallView: FC<InCallViewProps> = ({
     useExperimentalToDeviceTransportSetting,
   );
   const encryptionSystem = useRoomEncryptionSystem(matrixRoom.roomId);
-  const memberships = useMatrixRTCSessionMemberships(rtcSession);
 
   const showToDeviceEncryption = useMemo(
     () =>
@@ -292,22 +228,19 @@ export const InCallView: FC<InCallViewProps> = ({
     ],
   );
 
-  const toggleMicrophone = useCallback(
-    () => muteStates.audio.setEnabled?.((e) => !e),
-    [muteStates],
-  );
-  const toggleCamera = useCallback(
-    () => muteStates.video.setEnabled?.((e) => !e),
-    [muteStates],
-  );
+  const audioEnabled = useBehavior(muteStates.audio.enabled$);
+  const videoEnabled = useBehavior(muteStates.video.enabled$);
+  const toggleAudio = useBehavior(muteStates.audio.toggle$);
+  const toggleVideo = useBehavior(muteStates.video.toggle$);
+  const setAudioEnabled = useBehavior(muteStates.audio.setEnabled$);
 
   // This function incorrectly assumes that there is a camera and microphone, which is not always the case.
   // TODO: Make sure that this module is resilient when it comes to camera/microphone availability!
   useCallViewKeyboardShortcuts(
     containerRef1,
-    toggleMicrophone,
-    toggleCamera,
-    (muted) => muteStates.audio.setEnabled?.(!muted),
+    toggleAudio,
+    toggleVideo,
+    setAudioEnabled,
     (reaction) => void sendReaction(reaction),
     () => void toggleRaisedHand(),
   );
@@ -652,34 +585,37 @@ export const InCallView: FC<InCallViewProps> = ({
     matrixRoom.roomId,
   );
 
+  const allLivekitRooms = useBehavior(vm.allLivekitRooms$);
+  const memberships = useBehavior(vm.memberships$);
   const toggleScreensharing = useCallback(() => {
-    localParticipant
-      .setScreenShareEnabled(!isScreenShareEnabled, {
-        audio: true,
-        selfBrowserSurface: "include",
-        surfaceSwitching: "include",
-        systemAudio: "include",
-      })
-      .catch(logger.error);
-  }, [localParticipant, isScreenShareEnabled]);
+    throw new Error("TODO-MULTI-SFU");
+    // localParticipant
+    //   .setScreenShareEnabled(!isScreenShareEnabled, {
+    //     audio: true,
+    //     selfBrowserSurface: "include",
+    //     surfaceSwitching: "include",
+    //     systemAudio: "include",
+    //   })
+    //   .catch(logger.error);
+  }, []);
 
   const buttons: JSX.Element[] = [];
 
   buttons.push(
     <MicButton
       key="audio"
-      muted={!muteStates.audio.enabled}
-      onClick={toggleMicrophone}
+      muted={!audioEnabled}
+      onClick={toggleAudio ?? undefined}
       onTouchEnd={onControlsTouchEnd}
-      disabled={muteStates.audio.setEnabled === null}
+      disabled={toggleAudio === null}
       data-testid="incall_mute"
     />,
     <VideoButton
       key="video"
-      muted={!muteStates.video.enabled}
-      onClick={toggleCamera}
+      muted={!videoEnabled}
+      onClick={toggleVideo ?? undefined}
       onTouchEnd={onControlsTouchEnd}
-      disabled={muteStates.video.setEnabled === null}
+      disabled={toggleVideo === null}
       data-testid="incall_videomute"
     />,
   );
@@ -688,7 +624,7 @@ export const InCallView: FC<InCallViewProps> = ({
       <ShareScreenButton
         key="share_screen"
         className={styles.shareScreen}
-        enabled={isScreenShareEnabled}
+        enabled={false} // TODO-MULTI-SFU
         onClick={toggleScreensharing}
         onTouchEnd={onControlsTouchEnd}
         data-testid="incall_screenshare"
@@ -786,7 +722,14 @@ export const InCallView: FC<InCallViewProps> = ({
           </Text>
         )
       }
-      <MatrixAudioRenderer members={memberships} muted={muteAllAudio} />
+      {allLivekitRooms.map((roomItem) => (
+        <LivekitRoomAudioRenderer
+          key={roomItem.url}
+          livekitRoom={roomItem.room}
+          members={memberships}
+          muted={muteAllAudio}
+        />
+      ))}
       {renderContent()}
       <CallEventAudioRenderer vm={vm} muted={muteAllAudio} />
       <ReactionsAudioRenderer vm={vm} muted={muteAllAudio} />
@@ -813,7 +756,7 @@ export const InCallView: FC<InCallViewProps> = ({
             onDismiss={closeSettings}
             tab={settingsTab}
             onTabChange={setSettingsTab}
-            livekitRoom={livekitRoom}
+            livekitRoom={undefined} // TODO-MULTI-SFU
           />
         </>
       )}

src/room/LobbyView.tsx

@@ -31,7 +31,7 @@ import inCallStyles from "./InCallView.module.css";
 import styles from "./LobbyView.module.css";
 import { Header, LeftNav, RightNav, RoomHeaderInfo } from "../Header";
 import { type MatrixInfo, VideoPreview } from "./VideoPreview";
-import { type MuteStates } from "./MuteStates";
+import { type MuteStates } from "../state/MuteStates";
 import { InviteButton } from "../button/InviteButton";
 import {
   EndCallButton,
@@ -50,8 +50,8 @@ import {
   useTrackProcessorSync,
 } from "../livekit/TrackProcessorContext";
 import { usePageTitle } from "../usePageTitle";
-import { useLatest } from "../useLatest";
 import { getValue } from "../utils/observable";
+import { useBehavior } from "../useBehavior";
 
 interface Props {
   client: MatrixClient;
@@ -88,14 +88,10 @@ export const LobbyView: FC<Props> = ({
   const { t } = useTranslation();
   usePageTitle(matrixInfo.roomName);
 
-  const onAudioPress = useCallback(
-    () => muteStates.audio.setEnabled?.((e) => !e),
-    [muteStates],
-  );
-  const onVideoPress = useCallback(
-    () => muteStates.video.setEnabled?.((e) => !e),
-    [muteStates],
-  );
+  const audioEnabled = useBehavior(muteStates.audio.enabled$);
+  const videoEnabled = useBehavior(muteStates.video.enabled$);
+  const toggleAudio = useBehavior(muteStates.audio.toggle$);
+  const toggleVideo = useBehavior(muteStates.video.toggle$);
 
   const [settingsModalOpen, setSettingsModalOpen] = useState(false);
   const [settingsTab, setSettingsTab] = useState(defaultSettingsTab);
@@ -133,7 +129,7 @@ export const LobbyView: FC<Props> = ({
   // re-open the devices when they change (see below).
   const initialAudioOptions = useInitial(
     () =>
-      muteStates.audio.enabled && {
+      audioEnabled && {
         deviceId: getValue(devices.audioInput.selected$)?.id,
       },
   );
@@ -150,27 +146,21 @@ export const LobbyView: FC<Props> = ({
       // We also pass in a clone because livekit mutates the object passed in,
       // which would cause the devices to be re-opened on the next render.
       audio: Object.assign({}, initialAudioOptions),
-      video: muteStates.video.enabled && {
+      video: videoEnabled && {
         deviceId: videoInputId,
         processor: initialProcessor,
       },
     }),
-    [
-      initialAudioOptions,
-      muteStates.video.enabled,
-      videoInputId,
-      initialProcessor,
-    ],
+    [initialAudioOptions, videoEnabled, videoInputId, initialProcessor],
   );
 
-  const latestMuteStates = useLatest(muteStates);
   const onError = useCallback(
     (error: Error) => {
       logger.error("Error while creating preview Tracks:", error);
-      latestMuteStates.current.audio.setEnabled?.(false);
-      latestMuteStates.current.video.setEnabled?.(false);
+      muteStates.audio.setEnabled$.value?.(false);
+      muteStates.video.setEnabled$.value?.(false);
     },
-    [latestMuteStates],
+    [muteStates],
   );
 
   const tracks = usePreviewTracks(localTrackOptions, onError);
@@ -217,7 +207,7 @@ export const LobbyView: FC<Props> = ({
         <div className={styles.content}>
           <VideoPreview
             matrixInfo={matrixInfo}
-            muteStates={muteStates}
+            videoEnabled={videoEnabled}
             videoTrack={videoTrack}
           >
             <Button
@@ -239,14 +229,14 @@ export const LobbyView: FC<Props> = ({
           {recentsButtonInFooter && recentsButton}
           <div className={inCallStyles.buttons}>
             <MicButton
-              muted={!muteStates.audio.enabled}
-              onClick={onAudioPress}
-              disabled={muteStates.audio.setEnabled === null}
+              muted={!audioEnabled}
+              onClick={toggleAudio ?? undefined}
+              disabled={toggleAudio === null}
             />
             <VideoButton
-              muted={!muteStates.video.enabled}
-              onClick={onVideoPress}
-              disabled={muteStates.video.setEnabled === null}
+              muted={!videoEnabled}
+              onClick={toggleVideo ?? undefined}
+              disabled={toggleVideo === null}
             />
             <SettingsButton onClick={openSettings} />
             {!confineToRoom && <EndCallButton onClick={onLeaveClick} />}

src/room/RoomPage.tsx

@@ -20,6 +20,8 @@ import {
   CheckIcon,
   UnknownSolidIcon,
 } from "@vector-im/compound-design-tokens/assets/web/icons";
+import { useObservable } from "observable-hooks";
+import { map } from "rxjs";
 
 import { useClientLegacy } from "../ClientContext";
 import { ErrorPage, FullScreenView, LoadingPage } from "../FullScreenView";
@@ -35,12 +37,13 @@ import { CallTerminatedMessage, useLoadGroupCall } from "./useLoadGroupCall";
 import { LobbyView } from "./LobbyView";
 import { E2eeType } from "../e2ee/e2eeType";
 import { useProfile } from "../profile/useProfile";
-import { useMuteStates } from "./MuteStates";
 import { useOptInAnalytics } from "../settings/settings";
 import { Config } from "../config/Config";
 import { Link } from "../button/Link";
 import { ErrorView } from "../ErrorView";
-import { useMatrixRTCSessionJoinState } from "../useMatrixRTCSessionJoinState";
+import { useMediaDevices } from "../MediaDevicesContext";
+import { MuteStates } from "../state/MuteStates";
+import { ObservableScope } from "../state/ObservableScope";
 
 export const RoomPage: FC = () => {
   const { confineToRoom, appPrompt, preload, header, displayName, skipLobby } =
@@ -61,10 +64,19 @@ export const RoomPage: FC = () => {
   const { avatarUrl, displayName: userDisplayName } = useProfile(client);
 
   const groupCallState = useLoadGroupCall(client, roomIdOrAlias, viaServers);
-  const isJoined = useMatrixRTCSessionJoinState(
-    groupCallState.kind === "loaded" ? groupCallState.rtcSession : undefined,
+  const [joined, setJoined] = useState(false);
+
+  const devices = useMediaDevices();
+  const [muteStates, setMuteStates] = useState<MuteStates | null>(null);
+  const joined$ = useObservable(
+    (inputs$) => inputs$.pipe(map(([joined]) => joined)),
+    [joined],
   );
-  const muteStates = useMuteStates(isJoined);
+  useEffect(() => {
+    const scope = new ObservableScope();
+    setMuteStates(new MuteStates(scope, devices, joined$));
+    return (): void => scope.end();
+  }, [devices, joined$]);
 
   useEffect(() => {
     // If we've finished loading, are not already authed and we've been given a display name as
@@ -101,15 +113,16 @@ export const RoomPage: FC = () => {
     }
   }, [groupCallState.kind]);
 
-  const groupCallView = (): JSX.Element => {
+  const groupCallView = (): ReactNode => {
     switch (groupCallState.kind) {
       case "loaded":
-        return (
+        return muteStates && (
           <GroupCallView
             widget={widget}
             client={client!}
             rtcSession={groupCallState.rtcSession}
-            isJoined={isJoined}
+            joined={joined}
+            setJoined={setJoined}
             isPasswordlessUser={passwordlessUser}
             confineToRoom={confineToRoom}
             preload={preload}
@@ -135,7 +148,7 @@ export const RoomPage: FC = () => {
             </>
           );
         return (
-          <LobbyView
+          muteStates && <LobbyView
             client={client!}
             matrixInfo={{
               userId: client!.getUserId() ?? "",

src/room/VideoPreview.tsx

@@ -13,7 +13,6 @@ import { useTranslation } from "react-i18next";
 
 import { TileAvatar } from "../tile/TileAvatar";
 import styles from "./VideoPreview.module.css";
-import { type MuteStates } from "./MuteStates";
 import { type EncryptionSystem } from "../e2ee/sharedKeyManagement";
 
 export type MatrixInfo = {
@@ -29,14 +28,14 @@ export type MatrixInfo = {
 
 interface Props {
   matrixInfo: MatrixInfo;
-  muteStates: MuteStates;
+  videoEnabled: boolean;
   videoTrack: LocalVideoTrack | null;
   children: ReactNode;
 }
 
 export const VideoPreview: FC<Props> = ({
   matrixInfo,
-  muteStates,
+  videoEnabled,
   videoTrack,
   children,
 }) => {
@@ -56,8 +55,8 @@ export const VideoPreview: FC<Props> = ({
   }, [videoTrack]);
 
   const cameraIsStarting = useMemo(
-    () => muteStates.video.enabled && !videoTrack,
-    [muteStates.video.enabled, videoTrack],
+    () => videoEnabled && !videoTrack,
+    [videoEnabled, videoTrack],
   );
 
   return (
@@ -76,7 +75,7 @@ export const VideoPreview: FC<Props> = ({
         tabIndex={-1}
         disablePictureInPicture
       />
-      {(!muteStates.video.enabled || cameraIsStarting) && (
+      {(!videoEnabled || cameraIsStarting) && (
         <>
           <div className={styles.avatarContainer}>
             {cameraIsStarting && (

src/rtcSessionHelpers.ts

@@ -6,10 +6,10 @@ Please see LICENSE in the repository root for full details.
 */
 
 import {
-  isLivekitFocus,
   isLivekitFocusConfig,
+  type LivekitFocusConfig,
   type LivekitFocus,
-  type LivekitFocusActive,
+  type LivekitFocusSelection,
   type MatrixRTCSession,
 } from "matrix-js-sdk/lib/matrixrtc";
 import { logger } from "matrix-js-sdk/lib/logger";
@@ -24,34 +24,47 @@ import { getSFUConfigWithOpenID } from "./livekit/openIDSFU.ts";
 
 const FOCI_WK_KEY = "org.matrix.msc4143.rtc_foci";
 
-export function makeActiveFocus(): LivekitFocusActive {
+export function makeActiveFocus(): LivekitFocusSelection {
   return {
     type: "livekit",
     focus_selection: "oldest_membership",
   };
 }
 
-async function makePreferredLivekitFoci(
+export function getLivekitAlias(rtcSession: MatrixRTCSession): string {
+  // For now we assume everything is a room-scoped call
+  return rtcSession.room.roomId;
+}
+
+async function makeFocusInternal(
   rtcSession: MatrixRTCSession,
-  livekitAlias: string,
-): Promise<LivekitFocus[]> {
-  logger.log("Start building foci_preferred list: ", rtcSession.room.roomId);
+): Promise<LivekitFocus> {
+  logger.log("Searching for a preferred focus");
+  const livekitAlias = getLivekitAlias(rtcSession);
 
-  const preferredFoci: LivekitFocus[] = [];
-
-  // Make the Focus from the running rtc session the highest priority one
-  // This minimizes how often we need to switch foci during a call.
-  const focusInUse = rtcSession.getFocusInUse();
-  if (focusInUse && isLivekitFocus(focusInUse)) {
-    logger.log("Adding livekit focus from oldest member: ", focusInUse);
-    preferredFoci.push(focusInUse);
+  const urlFromStorage = localStorage.getItem("robin-matrixrtc-auth");
+  if (urlFromStorage !== null) {
+    const focusFromStorage: LivekitFocus = {
+      type: "livekit",
+      livekit_service_url: urlFromStorage,
+      livekit_alias: livekitAlias,
+    };
+    logger.log("Using LiveKit focus from local storage: ", focusFromStorage);
+    return focusFromStorage;
   }
 
-  // Warm up the first focus we owned, to ensure livekit room is created before any state event sent.
-  let toWarmUp: LivekitFocus | undefined;
-
   // Prioritize the .well-known/matrix/client, if available, over the configured SFU
   const domain = rtcSession.room.client.getDomain();
+  if (localStorage.getItem("timo-focus-url")) {
+    const timoFocusUrl = localStorage.getItem("timo-focus-url")!;
+    const focusFromUrl: LivekitFocus = {
+      type: "livekit",
+      livekit_service_url: timoFocusUrl,
+      livekit_alias: livekitAlias,
+    };
+    logger.log("Using LiveKit focus from localStorage: ", timoFocusUrl);
+    return focusFromUrl;
+  }
   if (domain) {
     // we use AutoDiscovery instead of relying on the MatrixClient having already
     // been fully configured and started
@@ -59,51 +72,46 @@ async function makePreferredLivekitFoci(
       FOCI_WK_KEY
     ];
     if (Array.isArray(wellKnownFoci)) {
-      const validWellKnownFoci = wellKnownFoci
-        .filter((f) => !!f)
-        .filter(isLivekitFocusConfig)
-        .map((wellKnownFocus) => {
-          logger.log("Adding livekit focus from well known: ", wellKnownFocus);
-          return { ...wellKnownFocus, livekit_alias: livekitAlias };
-        });
-      if (validWellKnownFoci.length > 0) {
-        toWarmUp = validWellKnownFoci[0];
+      const focus: LivekitFocusConfig | undefined = wellKnownFoci.find(
+        (f) => f && isLivekitFocusConfig(f),
+      );
+      if (focus !== undefined) {
+        logger.log("Using LiveKit focus from .well-known: ", focus);
+        return { ...focus, livekit_alias: livekitAlias };
       }
-      preferredFoci.push(...validWellKnownFoci);
     }
   }
 
   const urlFromConf = Config.get().livekit?.livekit_service_url;
   if (urlFromConf) {
-    const focusFormConf: LivekitFocus = {
+    const focusFromConf: LivekitFocus = {
       type: "livekit",
       livekit_service_url: urlFromConf,
       livekit_alias: livekitAlias,
     };
-    toWarmUp = toWarmUp ?? focusFormConf;
-    logger.log("Adding livekit focus from config: ", focusFormConf);
-    preferredFoci.push(focusFormConf);
+    logger.log("Using LiveKit focus from config: ", focusFromConf);
+    return focusFromConf;
   }
 
-  if (toWarmUp) {
-    // this will call the jwt/sfu/get endpoint to pre create the livekit room.
-    await getSFUConfigWithOpenID(rtcSession.room.client, toWarmUp);
-  }
-  if (preferredFoci.length === 0)
-    throw new MatrixRTCFocusMissingError(domain ?? "");
-  return Promise.resolve(preferredFoci);
+  throw new MatrixRTCFocusMissingError(domain ?? "");
+}
 
-  // TODO: we want to do something like this:
-  //
-  // const focusOtherMembers = await focusFromOtherMembers(
-  //   rtcSession,
-  //   livekitAlias,
-  // );
-  // if (focusOtherMembers) preferredFoci.push(focusOtherMembers);
+export async function makeFocus(
+  rtcSession: MatrixRTCSession,
+): Promise<LivekitFocus> {
+  const focus = await makeFocusInternal(rtcSession);
+  // this will call the jwt/sfu/get endpoint to pre create the livekit room.
+  await getSFUConfigWithOpenID(
+    rtcSession.room.client,
+    focus.livekit_service_url,
+    focus.livekit_alias,
+  );
+  return focus;
 }
 
 export async function enterRTCSession(
   rtcSession: MatrixRTCSession,
+  focus: LivekitFocus,
   encryptMedia: boolean,
   useNewMembershipManager = true,
   useExperimentalToDeviceTransport = false,
@@ -115,34 +123,27 @@ export async function enterRTCSession(
   // have started tracking by the time calls start getting created.
   // groupCallOTelMembership?.onJoinCall();
 
-  // right now we assume everything is a room-scoped call
-  const livekitAlias = rtcSession.room.roomId;
   const { features, matrix_rtc_session: matrixRtcSessionConfig } = Config.get();
   const useDeviceSessionMemberEvents =
     features?.feature_use_device_session_member_events;
-  rtcSession.joinRoomSession(
-    await makePreferredLivekitFoci(rtcSession, livekitAlias),
-    makeActiveFocus(),
-    {
-      notificationType: getUrlParams().sendNotificationType,
-      useNewMembershipManager,
-      manageMediaKeys: encryptMedia,
-      ...(useDeviceSessionMemberEvents !== undefined && {
-        useLegacyMemberEvents: !useDeviceSessionMemberEvents,
-      }),
-      delayedLeaveEventRestartMs:
-        matrixRtcSessionConfig?.delayed_leave_event_restart_ms,
-      delayedLeaveEventDelayMs:
-        matrixRtcSessionConfig?.delayed_leave_event_delay_ms,
-      delayedLeaveEventRestartLocalTimeoutMs:
-        matrixRtcSessionConfig?.delayed_leave_event_restart_local_timeout_ms,
-      networkErrorRetryMs: matrixRtcSessionConfig?.network_error_retry_ms,
-      makeKeyDelay: matrixRtcSessionConfig?.wait_for_key_rotation_ms,
-      membershipEventExpiryMs:
-        matrixRtcSessionConfig?.membership_event_expiry_ms,
-      useExperimentalToDeviceTransport,
-    },
-  );
+  rtcSession.joinRoomSession([focus], focus, {
+    notificationType: getUrlParams().sendNotificationType,
+    useNewMembershipManager,
+    manageMediaKeys: encryptMedia,
+    ...(useDeviceSessionMemberEvents !== undefined && {
+      useLegacyMemberEvents: !useDeviceSessionMemberEvents,
+    }),
+    delayedLeaveEventRestartMs:
+      matrixRtcSessionConfig?.delayed_leave_event_restart_ms,
+    delayedLeaveEventDelayMs:
+      matrixRtcSessionConfig?.delayed_leave_event_delay_ms,
+    delayedLeaveEventRestartLocalTimeoutMs:
+      matrixRtcSessionConfig?.delayed_leave_event_restart_local_timeout_ms,
+    networkErrorRetryMs: matrixRtcSessionConfig?.network_error_retry_ms,
+    makeKeyDelay: matrixRtcSessionConfig?.wait_for_key_rotation_ms,
+    membershipEventExpiryMs: matrixRtcSessionConfig?.membership_event_expiry_ms,
+    useExperimentalToDeviceTransport,
+  });
   if (widget) {
     try {
       await widget.api.transport.send(ElementWidgetActions.JoinCall, {});

src/state/CallViewModel.ts

@@ -5,24 +5,23 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
 Please see LICENSE in the repository root for full details.
 */
 
+import { observeParticipantEvents } from "@livekit/components-core";
 import {
-  connectedParticipantsObserver,
-  observeParticipantEvents,
-  observeParticipantMedia,
-} from "@livekit/components-core";
-import {
-  ConnectionState,
+  type BaseKeyProvider,
+  type E2EEOptions,
+  ExternalE2EEKeyProvider,
   type Room as LivekitRoom,
   type LocalParticipant,
   ParticipantEvent,
   type RemoteParticipant,
 } from "livekit-client";
+import E2EEWorker from "livekit-client/e2ee-worker?worker";
 import {
   ClientEvent,
+  type RoomMember,
   RoomStateEvent,
   SyncState,
   type Room as MatrixRoom,
-  type RoomMember,
 } from "matrix-js-sdk";
 import {
   BehaviorSubject,
@@ -30,15 +29,15 @@ import {
   type Observable,
   Subject,
   combineLatest,
-  concat,
+  concatMap,
   distinctUntilChanged,
   filter,
-  forkJoin,
+  from,
   fromEvent,
   map,
   merge,
-  mergeMap,
   of,
+  pairwise,
   race,
   scan,
   skip,
@@ -48,11 +47,11 @@ import {
   switchScan,
   take,
   timer,
-  withLatestFrom,
 } from "rxjs";
 import { logger } from "matrix-js-sdk/lib/logger";
 import {
   type CallMembership,
+  isLivekitFocus,
   type MatrixRTCSession,
   MatrixRTCSessionEvent,
   MembershipManagerEvent,
@@ -60,10 +59,6 @@ import {
 } from "matrix-js-sdk/lib/matrixrtc";
 
 import { ViewModel } from "./ViewModel";
-import {
-  ECAddonConnectionState,
-  type ECConnectionState,
-} from "../livekit/useECConnectionState";
 import {
   LocalUserMediaViewModel,
   type MediaViewModel,
@@ -106,14 +101,20 @@ import { shallowEquals } from "../utils/array";
 import { calculateDisplayName, shouldDisambiguate } from "../utils/displayname";
 import { type MediaDevices } from "./MediaDevices";
 import { type Behavior } from "./Behavior";
+import {
+  enterRTCSession,
+  getLivekitAlias,
+  makeFocus,
+} from "../rtcSessionHelpers";
+import { E2eeType } from "../e2ee/e2eeType";
+import { MatrixKeyProvider } from "../e2ee/matrixKeyProvider";
+import { Connection, PublishConnection } from "./Connection";
+import { type MuteStates } from "./MuteStates";
 
 export interface CallViewModelOptions {
   encryptionSystem: EncryptionSystem;
   autoLeaveWhenOthersLeft?: boolean;
 }
-// How long we wait after a focus switch before showing the real participant
-// list again
-const POST_FOCUS_PARTICIPANT_UPDATE_DELAY_MS = 3000;
 
 // This is the number of participants that we think constitutes a "small" call
 // on mobile. No spotlight tile should be shown below this threshold.
@@ -341,6 +342,7 @@ class UserMedia {
 
   public destroy(): void {
     this.scope.end();
+
     this.vm.destroy();
   }
 }
@@ -405,8 +407,165 @@ function getRoomMemberFromRtcMember(
   return { id, member };
 }
 
-// TODO: Move wayyyy more business logic from the call and lobby views into here
 export class CallViewModel extends ViewModel {
+  private readonly livekitAlias = getLivekitAlias(this.matrixRTCSession);
+
+  private readonly livekitE2EEKeyProvider = getE2eeKeyProvider(
+    this.options.encryptionSystem,
+    this.matrixRTCSession,
+  );
+  private readonly e2eeLivekitOptions = (): E2EEOptions | undefined =>
+    this.livekitE2EEKeyProvider
+      ? {
+          keyProvider: this.livekitE2EEKeyProvider,
+          worker: new E2EEWorker(),
+        }
+      : undefined;
+
+  private readonly localFocus = makeFocus(this.matrixRTCSession);
+
+  private readonly localConnection = this.localFocus.then(
+    (focus) =>
+      new PublishConnection(
+        focus,
+        this.livekitAlias,
+        this.matrixRTCSession.room.client,
+        this.scope,
+        this.membershipsAndFocusMap$,
+        this.mediaDevices,
+        this.muteStates,
+        this.e2eeLivekitOptions(),
+      ),
+  );
+
+  // TODO-MULTI-SFU make sure that we consider the room memberships here as well (so that here we only have valid memberships)
+  // this also makes it possible to use this memberships$ list in all observables based on it.
+  // there should be no other call to: this.matrixRTCSession.memberships!
+  public readonly memberships$ = this.scope.behavior(
+    fromEvent(
+      this.matrixRTCSession,
+      MatrixRTCSessionEvent.MembershipsChanged,
+    ).pipe(
+      startWith(null),
+      map(() => this.matrixRTCSession.memberships),
+    ),
+  );
+
+  private readonly membershipsAndFocusMap$ = this.scope.behavior(
+    this.memberships$.pipe(
+      map((memberships) =>
+        memberships.flatMap((m) => {
+          const f = this.matrixRTCSession.resolveActiveFocus(m);
+          return f && isLivekitFocus(f) ? [{ membership: m, focus: f }] : [];
+        }),
+      ),
+    ),
+  );
+
+  private readonly livekitServiceUrls$ = this.membershipsAndFocusMap$.pipe(
+    map((v) => new Set(v.map(({ focus }) => focus.livekit_service_url))),
+  );
+
+  private readonly remoteConnections$ = this.scope.behavior(
+    combineLatest([this.localFocus, this.livekitServiceUrls$]).pipe(
+      accumulate(
+        new Map<string, Connection>(),
+        (prev, [localFocus, focusUrls]) => {
+          const stopped = new Map(prev);
+          const next = new Map<string, Connection>();
+          for (const focusUrl of focusUrls) {
+            if (focusUrl !== localFocus.livekit_service_url) {
+              stopped.delete(focusUrl);
+
+              let nextConnection = prev.get(focusUrl);
+              if (!nextConnection) {
+                logger.log(
+                  "SFU remoteConnections$ construct new connection: ",
+                  focusUrl,
+                );
+                nextConnection = new Connection(
+                  {
+                    livekit_service_url: focusUrl,
+                    livekit_alias: this.livekitAlias,
+                    type: "livekit",
+                  },
+                  this.livekitAlias,
+                  this.matrixRTCSession.room.client,
+                  this.scope,
+                  this.membershipsAndFocusMap$,
+                  this.e2eeLivekitOptions(),
+                );
+              } else {
+                logger.log(
+                  "SFU remoteConnections$ use prev connection: ",
+                  focusUrl,
+                );
+              }
+              next.set(focusUrl, nextConnection);
+            }
+          }
+
+          for (const connection of stopped.values()) connection.stop();
+          return next;
+        },
+      ),
+    ),
+  );
+
+  private readonly join$ = new Subject<void>();
+
+  public join(): void {
+    this.join$.next();
+  }
+
+  private readonly leave$ = new Subject<void>();
+
+  public leave(): void {
+    this.leave$.next();
+  }
+
+  private readonly connectionInstructions$ = this.join$.pipe(
+    switchMap(() => this.remoteConnections$),
+    startWith(new Map<string, Connection>()),
+    pairwise(),
+    map(([prev, next]) => {
+      const start = new Set(next.values());
+      for (const connection of prev.values()) start.delete(connection);
+      const stop = new Set(prev.values());
+      for (const connection of next.values()) stop.delete(connection);
+
+      return { start, stop };
+    }),
+    this.scope.share,
+  );
+
+  private readonly startConnection$ = this.connectionInstructions$.pipe(
+    concatMap(({ start }) => start),
+  );
+  private readonly stopConnection$ = this.connectionInstructions$.pipe(
+    concatMap(({ stop }) => stop),
+  );
+
+  public readonly allLivekitRooms$ = this.scope.behavior(
+    combineLatest([
+      this.remoteConnections$,
+      this.localConnection,
+      this.localFocus,
+    ]).pipe(
+      map(([remoteConnections, localConnection, localFocus]) =>
+        Array.from(remoteConnections.entries())
+          .map(([index, c]) => ({ room: c.livekitRoom, url: index }))
+          .concat([
+            {
+              room: localConnection.livekitRoom,
+              url: localFocus.livekit_service_url,
+            },
+          ]),
+      ),
+      startWith([]),
+    ),
+  );
+
   private readonly userId = this.matrixRoom.client.getUserId();
 
   private readonly matrixConnected$ = this.scope.behavior(
@@ -446,9 +605,10 @@ export class CallViewModel extends ViewModel {
   private readonly connected$ = this.scope.behavior(
     and$(
       this.matrixConnected$,
-      this.livekitConnectionState$.pipe(
-        map((state) => state === ConnectionState.Connected),
-      ),
+      // TODO-MULTI-SFU
+      // this.livekitConnectionState$.pipe(
+      //   map((state) => state === ConnectionState.Connected),
+      // ),
     ),
   );
 
@@ -480,81 +640,55 @@ export class CallViewModel extends ViewModel {
   // in a split-brained state.
   private readonly pretendToBeDisconnected$ = this.reconnecting$;
 
-  /**
-   * The raw list of RemoteParticipants as reported by LiveKit
-   */
-  private readonly rawRemoteParticipants$ = this.scope.behavior<
-    RemoteParticipant[]
-  >(connectedParticipantsObserver(this.livekitRoom), []);
-
-  /**
-   * Lists of RemoteParticipants to "hold" on display, even if LiveKit claims that
-   * they've left
-   */
-  private readonly remoteParticipantHolds$ = this.scope.behavior<
-    RemoteParticipant[][]
+  private readonly participants$ = this.scope.behavior<
+    {
+      participant: LocalParticipant | RemoteParticipant;
+      member: RoomMember;
+      livekitRoom: LivekitRoom;
+    }[]
   >(
-    this.livekitConnectionState$.pipe(
-      withLatestFrom(this.rawRemoteParticipants$),
-      mergeMap(([s, ps]) => {
-        // Whenever we switch focuses, we should retain all the previous
-        // participants for at least POST_FOCUS_PARTICIPANT_UPDATE_DELAY_MS ms to
-        // give their clients time to switch over and avoid jarring layout shifts
-        if (s === ECAddonConnectionState.ECSwitchingFocus) {
-          return concat(
-            // Hold these participants
-            of({ hold: ps }),
-            // Wait for time to pass and the connection state to have changed
-            forkJoin([
-              timer(POST_FOCUS_PARTICIPANT_UPDATE_DELAY_MS),
-              this.livekitConnectionState$.pipe(
-                filter((s) => s !== ECAddonConnectionState.ECSwitchingFocus),
-                take(1),
+    from(this.localConnection)
+      .pipe(
+        switchMap((localConnection) => {
+          const memberError = (): never => {
+            throw new Error("No room member for call membership");
+          };
+          const localParticipant = {
+            participant: localConnection.livekitRoom.localParticipant,
+            member:
+              this.matrixRoom.getMember(this.userId ?? "") ?? memberError(),
+            livekitRoom: localConnection.livekitRoom,
+          };
+          return this.remoteConnections$.pipe(
+            switchMap((connections) =>
+              combineLatest(
+                [localConnection, ...connections.values()].map((c) =>
+                  c.publishingParticipants$.pipe(
+                    map((ps) =>
+                      ps.map(({ participant, membership }) => ({
+                        participant,
+                        member:
+                          getRoomMemberFromRtcMember(
+                            membership,
+                            this.matrixRoom,
+                          )?.member ?? memberError(),
+                        livekitRoom: c.livekitRoom,
+                      })),
+                    ),
+                  ),
+                ),
               ),
-              // Then unhold them
-            ]).pipe(map(() => ({ unhold: ps }))),
+            ),
+            map((remoteParticipants) => [
+              localParticipant,
+              ...remoteParticipants.flat(1),
+            ]),
           );
-        } else {
-          return EMPTY;
-        }
-      }),
-      // Accumulate the hold instructions into a single list showing which
-      // participants are being held
-      accumulate([] as RemoteParticipant[][], (holds, instruction) =>
-        "hold" in instruction
-          ? [instruction.hold, ...holds]
-          : holds.filter((h) => h !== instruction.unhold),
-      ),
-    ),
+        }),
+      )
+      .pipe(startWith([]), pauseWhen(this.pretendToBeDisconnected$)),
   );
 
-  /**
-   * The RemoteParticipants including those that are being "held" on the screen
-   */
-  private readonly remoteParticipants$ = this.scope
-    .behavior<RemoteParticipant[]>(
-      combineLatest(
-        [this.rawRemoteParticipants$, this.remoteParticipantHolds$],
-        (raw, holds) => {
-          const result = [...raw];
-          const resultIds = new Set(result.map((p) => p.identity));
-
-          // Incorporate the held participants into the list
-          for (const hold of holds) {
-            for (const p of hold) {
-              if (!resultIds.has(p.identity)) {
-                result.push(p);
-                resultIds.add(p.identity);
-              }
-            }
-          }
-
-          return result;
-        },
-      ),
-    )
-    .pipe(pauseWhen(this.pretendToBeDisconnected$));
-
   /**
    * Displaynames for each member of the call. This will disambiguate
    * any displaynames that clashes with another member. Only members
@@ -576,7 +710,9 @@ export class CallViewModel extends ViewModel {
       startWith(null),
       map(() => {
         const memberships = this.matrixRTCSession.memberships;
-        const displaynameMap = new Map<string, string>();
+        const displaynameMap = new Map<string, string>([
+          ["local", this.matrixRoom.getMember(this.userId!)!.rawDisplayName],
+        ]);
         const room = this.matrixRoom;
 
         // We only consider RTC members for disambiguation as they are the only visible members.
@@ -625,8 +761,7 @@ export class CallViewModel extends ViewModel {
    */
   private readonly mediaItems$ = this.scope.behavior<MediaItem[]>(
     combineLatest([
-      this.remoteParticipants$,
-      observeParticipantMedia(this.livekitRoom.localParticipant),
+      this.participants$,
       duplicateTiles.value$,
       // Also react to changes in the MatrixRTC session list.
       // The session list will also be update if a room membership changes.
@@ -641,43 +776,21 @@ export class CallViewModel extends ViewModel {
         (
           prevItems,
           [
-            remoteParticipants,
-            { participant: localParticipant },
+            participants,
             duplicateTiles,
             _membershipsChanged,
             showNonMemberTiles,
           ],
         ) => {
-          const newItems = new Map(
+          const newItems: Map<string, UserMedia | ScreenShare> = new Map(
             function* (this: CallViewModel): Iterable<[string, MediaItem]> {
-              const room = this.matrixRoom;
-              // m.rtc.members are the basis for calculating what is visible in the call
-              for (const rtcMember of this.matrixRTCSession.memberships) {
-                const { member, id: livekitParticipantId } =
-                  getRoomMemberFromRtcMember(rtcMember, room);
-                const matrixIdentifier = `${rtcMember.sender}:${rtcMember.deviceId}`;
-
-                let participant:
-                  | LocalParticipant
-                  | RemoteParticipant
-                  | undefined = undefined;
-                if (livekitParticipantId === "local") {
-                  participant = localParticipant;
-                } else {
-                  participant = remoteParticipants.find(
-                    (p) => p.identity === livekitParticipantId,
-                  );
-                }
-
-                if (!member) {
-                  logger.error(
-                    "Could not find member for media id: ",
-                    livekitParticipantId,
-                  );
-                }
+              for (const { participant, member, livekitRoom } of participants) {
+                const matrixId = participant.isLocal
+                  ? "local"
+                  : participant.identity;
                 for (let i = 0; i < 1 + duplicateTiles; i++) {
-                  const indexedMediaId = `${livekitParticipantId}:${i}`;
-                  let prevMedia = prevItems.get(indexedMediaId);
+                  const mediaId = `${matrixId}:${i}`;
+                  let prevMedia = prevItems.get(mediaId);
                   if (prevMedia && prevMedia instanceof UserMedia) {
                     prevMedia.updateParticipant(participant);
                     if (prevMedia.vm.member === undefined) {
@@ -690,33 +803,33 @@ export class CallViewModel extends ViewModel {
                     }
                   }
                   yield [
-                    indexedMediaId,
+                    mediaId,
                     // We create UserMedia with or without a participant.
                     // This will be the initial value of a BehaviourSubject.
                     // Once a participant appears we will update the BehaviourSubject. (see above)
                     prevMedia ??
                       new UserMedia(
-                        indexedMediaId,
+                        mediaId,
                         member,
                         participant,
                         this.options.encryptionSystem,
-                        this.livekitRoom,
+                        livekitRoom,
                         this.mediaDevices,
                         this.pretendToBeDisconnected$,
                         this.memberDisplaynames$.pipe(
-                          map((m) => m.get(matrixIdentifier) ?? "[👻]"),
+                          map((m) => m.get(matrixId) ?? "[👻]"),
                         ),
                         this.handsRaised$.pipe(
-                          map((v) => v[matrixIdentifier]?.time ?? null),
+                          map((v) => v[matrixId]?.time ?? null),
                         ),
                         this.reactions$.pipe(
-                          map((v) => v[matrixIdentifier] ?? undefined),
+                          map((v) => v[matrixId] ?? undefined),
                         ),
                       ),
                   ];
 
                   if (participant?.isScreenShareEnabled) {
-                    const screenShareId = `${indexedMediaId}:screen-share`;
+                    const screenShareId = `${mediaId}:screen-share`;
                     yield [
                       screenShareId,
                       prevItems.get(screenShareId) ??
@@ -725,10 +838,10 @@ export class CallViewModel extends ViewModel {
                           member,
                           participant,
                           this.options.encryptionSystem,
-                          this.livekitRoom,
+                          livekitRoom,
                           this.pretendToBeDisconnected$,
                           this.memberDisplaynames$.pipe(
-                            map((m) => m.get(matrixIdentifier) ?? "[👻]"),
+                            map((m) => m.get(matrixId) ?? "[👻]"),
                           ),
                         ),
                     ];
@@ -750,47 +863,51 @@ export class CallViewModel extends ViewModel {
           //  - If one wants to test scalability using the LiveKit CLI.
           //  - If an experimental project does not yet do the MatrixRTC bits.
           //  - If someone wants to debug if the LiveKit connection works but MatrixRTC room state failed to arrive.
-          const newNonMemberItems = showNonMemberTiles
-            ? new Map(
-                function* (this: CallViewModel): Iterable<[string, MediaItem]> {
-                  for (const participant of remoteParticipants) {
-                    for (let i = 0; i < 1 + duplicateTiles; i++) {
-                      const maybeNonMemberParticipantId =
-                        participant.identity + ":" + i;
-                      if (!newItems.has(maybeNonMemberParticipantId)) {
-                        const nonMemberId = maybeNonMemberParticipantId;
-                        yield [
-                          nonMemberId,
-                          prevItems.get(nonMemberId) ??
-                            new UserMedia(
-                              nonMemberId,
-                              undefined,
-                              participant,
-                              this.options.encryptionSystem,
-                              this.livekitRoom,
-                              this.mediaDevices,
-                              this.pretendToBeDisconnected$,
-                              this.memberDisplaynames$.pipe(
-                                map(
-                                  (m) => m.get(participant.identity) ?? "[👻]",
-                                ),
-                              ),
-                              of(null),
-                              of(null),
-                            ),
-                        ];
-                      }
-                    }
-                  }
-                }.bind(this)(),
-              )
-            : new Map();
-          if (newNonMemberItems.size > 0) {
-            logger.debug("Added NonMember items: ", newNonMemberItems);
-          }
+          // TODO-MULTI-SFU
+          // const newNonMemberItems = showNonMemberTiles
+          //   ? new Map(
+          //       function* (
+          //         this: CallViewModel,
+          //       ): Iterable<[string, MediaItem]> {
+          //         for (const participant of remoteParticipants) {
+          //           for (let i = 0; i < 1 + duplicateTiles; i++) {
+          //             const maybeNonMemberParticipantId =
+          //               participant.identity + ":" + i;
+          //             if (!newItems.has(maybeNonMemberParticipantId)) {
+          //               const nonMemberId = maybeNonMemberParticipantId;
+          //               yield [
+          //                 nonMemberId,
+          //                 prevItems.get(nonMemberId) ??
+          //                   new UserMedia(
+          //                     nonMemberId,
+          //                     undefined,
+          //                     participant,
+          //                     this.options.encryptionSystem,
+          //                     localConnection.livekitRoom,
+          //                     this.mediaDevices,
+          //                     this.pretendToBeDisconnected$,
+          //                     this.memberDisplaynames$.pipe(
+          //                       map(
+          //                         (m) =>
+          //                           m.get(participant.identity) ?? "[👻]",
+          //                       ),
+          //                     ),
+          //                     of(null),
+          //                     of(null),
+          //                   ),
+          //               ];
+          //             }
+          //           }
+          //         }
+          //       }.bind(this)(),
+          //     )
+          //   : new Map();
+          // if (newNonMemberItems.size > 0) {
+          //   logger.debug("Added NonMember items: ", newNonMemberItems);
+          // }
 
           const combinedNew = new Map([
-            ...newNonMemberItems.entries(),
+            // ...newNonMemberItems.entries(),
             ...newItems.entries(),
           ]);
 
@@ -1574,13 +1691,12 @@ export class CallViewModel extends ViewModel {
   );
 
   public constructor(
-    // A call is permanently tied to a single Matrix room and LiveKit room
+    // A call is permanently tied to a single Matrix room
     private readonly matrixRTCSession: MatrixRTCSession,
     private readonly matrixRoom: MatrixRoom,
-    private readonly livekitRoom: LivekitRoom,
     private readonly mediaDevices: MediaDevices,
+    private readonly muteStates: MuteStates,
     private readonly options: CallViewModelOptions,
-    private readonly livekitConnectionState$: Observable<ECConnectionState>,
     private readonly handsRaisedSubject$: Observable<
       Record<string, RaisedHandInfo>
     >,
@@ -1590,50 +1706,120 @@ export class CallViewModel extends ViewModel {
   ) {
     super();
 
+    void from(this.localConnection)
+      .pipe(this.scope.bind())
+      .subscribe(
+        (c) =>
+          void c
+            .start()
+            // eslint-disable-next-line no-console
+            .then(() => console.log("successfully started publishing"))
+            // eslint-disable-next-line no-console
+            .catch((e) => console.error("failed to start publishing", e)),
+      );
+
+    this.startConnection$
+      .pipe(this.scope.bind())
+      .subscribe((c) => void c.start());
+    this.stopConnection$.pipe(this.scope.bind()).subscribe((c) => c.stop());
+
+    combineLatest([this.localFocus, this.join$])
+      .pipe(this.scope.bind())
+      .subscribe(([localFocus]) => {
+        void enterRTCSession(
+          this.matrixRTCSession,
+          localFocus,
+          this.options.encryptionSystem.kind !== E2eeType.NONE,
+          true,
+          true,
+        );
+      });
+
+    this.join$.pipe(this.scope.bind()).subscribe(() => {
+      // TODO-MULTI-SFU: this makes no sense what so ever!!!
+      // need to look into this again.
+      // leaveRTCSession(
+      //   this.matrixRTCSession,
+      //   "user", // TODO-MULTI-SFU ?
+      //   // Wait for the sound in widget mode (it's not long)
+      //   Promise.resolve(), // TODO-MULTI-SFU
+      //   //Promise.all([audioPromise, posthogRequest]),
+      // ).catch((e) => {
+      //   logger.error("Error leaving RTC session", e);
+      // });
+    });
+
     // Pause upstream of all local media tracks when we're disconnected from
     // MatrixRTC, because it can be an unpleasant surprise for the app to say
     // 'reconnecting' and yet still be transmitting your media to others.
     // We use matrixConnected$ rather than reconnecting$ because we want to
     // pause tracks during the initial joining sequence too until we're sure
     // that our own media is displayed on screen.
-    this.matrixConnected$.pipe(this.scope.bind()).subscribe((connected) => {
-      const publications =
-        this.livekitRoom.localParticipant.trackPublications.values();
-      if (connected) {
-        for (const p of publications) {
-          if (p.track?.isUpstreamPaused === true) {
-            const kind = p.track.kind;
-            logger.log(
-              `Resumming ${kind} track (MatrixRTC connection present)`,
-            );
-            p.track
-              .resumeUpstream()
-              .catch((e) =>
-                logger.error(
-                  `Failed to resume ${kind} track after MatrixRTC reconnection`,
-                  e,
-                ),
+    void this.localConnection.then((localConnection) =>
+      this.matrixConnected$.pipe(this.scope.bind()).subscribe((connected) => {
+        const publications =
+          localConnection.livekitRoom.localParticipant.trackPublications.values();
+        if (connected) {
+          for (const p of publications) {
+            if (p.track?.isUpstreamPaused === true) {
+              const kind = p.track.kind;
+              logger.log(
+                `Resuming ${kind} track (MatrixRTC connection present)`,
               );
+              p.track
+                .resumeUpstream()
+                .catch((e) =>
+                  logger.error(
+                    `Failed to resume ${kind} track after MatrixRTC reconnection`,
+                    e,
+                  ),
+                );
+            }
+          }
+        } else {
+          for (const p of publications) {
+            if (p.track?.isUpstreamPaused === false) {
+              const kind = p.track.kind;
+              logger.log(
+                `Pausing ${kind} track (uncertain MatrixRTC connection)`,
+              );
+              p.track
+                .pauseUpstream()
+                .catch((e) =>
+                  logger.error(
+                    `Failed to pause ${kind} track after entering uncertain MatrixRTC connection`,
+                    e,
+                  ),
+                );
+            }
           }
         }
-      } else {
-        for (const p of publications) {
-          if (p.track?.isUpstreamPaused === false) {
-            const kind = p.track.kind;
-            logger.log(
-              `Pausing ${kind} track (uncertain MatrixRTC connection)`,
-            );
-            p.track
-              .pauseUpstream()
-              .catch((e) =>
-                logger.error(
-                  `Failed to pause ${kind} track after entering uncertain MatrixRTC connection`,
-                  e,
-                ),
-              );
-          }
-        }
-      }
-    });
+      }),
+    );
+
+    // Join automatically
+    this.join(); // TODO-MULTI-SFU: Use this view model for the lobby as well, and only call this once 'join' is clicked?
+  }
+}
+
+// TODO-MULTI-SFU   // Setup and update the keyProvider which was create by `createRoom` was a thing before. Now we never update if the E2EEsystem changes
+// do we need this?
+
+function getE2eeKeyProvider(
+  e2eeSystem: EncryptionSystem,
+  rtcSession: MatrixRTCSession,
+): BaseKeyProvider | undefined {
+  if (e2eeSystem.kind === E2eeType.NONE) return undefined;
+
+  if (e2eeSystem.kind === E2eeType.PER_PARTICIPANT) {
+    const keyProvider = new MatrixKeyProvider();
+    keyProvider.setRTCSession(rtcSession);
+    return keyProvider;
+  } else if (e2eeSystem.kind === E2eeType.SHARED_KEY && e2eeSystem.secret) {
+    const keyProvider = new ExternalE2EEKeyProvider();
+    keyProvider
+      .setKey(e2eeSystem.secret)
+      .catch((e) => logger.error("Failed to set shared key for E2EE", e));
+    return keyProvider;
   }
 }

src/state/Connection.ts

@@ -0,0 +1,281 @@
+// TODO-MULTI-SFU Add all device syncing logic from useLivekit
+/*
+Copyright 2025 New Vector Ltd.
+
+SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
+Please see LICENSE in the repository root for full details.
+*/
+
+import {
+  connectedParticipantsObserver,
+  connectionStateObserver,
+} from "@livekit/components-core";
+import {
+  ConnectionState,
+  Room as LivekitRoom,
+  type E2EEOptions,
+  Track,
+} from "livekit-client";
+import { type MatrixClient } from "matrix-js-sdk";
+import {
+  type LivekitFocus,
+  type CallMembership,
+} from "matrix-js-sdk/lib/matrixrtc";
+import {
+  combineLatest,
+  map,
+  NEVER,
+  type Observable,
+  type Subscription,
+  switchMap,
+} from "rxjs";
+import { logger } from "matrix-js-sdk/lib/logger";
+
+import { type SelectedDevice, type MediaDevices } from "./MediaDevices";
+import { getSFUConfigWithOpenID } from "../livekit/openIDSFU";
+import { type Behavior } from "./Behavior";
+import { type ObservableScope } from "./ObservableScope";
+import { defaultLiveKitOptions } from "../livekit/options";
+import { getValue } from "../utils/observable";
+import { getUrlParams } from "../UrlParams";
+import { type MuteStates } from "./MuteStates";
+
+export class Connection {
+  protected stopped = false;
+
+  public async start(): Promise<void> {
+    this.stopped = false;
+    const { url, jwt } = await this.sfuConfig;
+    if (!this.stopped) await this.livekitRoom.connect(url, jwt);
+  }
+
+  public stop(): void {
+    void this.livekitRoom.disconnect();
+    this.stopped = true;
+  }
+
+  protected readonly sfuConfig = getSFUConfigWithOpenID(
+    this.client,
+    this.focus.livekit_service_url,
+    this.livekitAlias,
+  );
+
+  public readonly participantsIncludingSubscribers$;
+  public readonly publishingParticipants$;
+  public readonly livekitRoom: LivekitRoom;
+
+  public connectionState$: Behavior<ConnectionState>;
+  public constructor(
+    protected readonly focus: LivekitFocus,
+    protected readonly livekitAlias: string,
+    protected readonly client: MatrixClient,
+    protected readonly scope: ObservableScope,
+    protected readonly membershipsFocusMap$: Behavior<
+      { membership: CallMembership; focus: LivekitFocus }[]
+    >,
+    e2eeLivekitOptions: E2EEOptions | undefined,
+    livekitRoom: LivekitRoom | undefined = undefined,
+  ) {
+    this.livekitRoom =
+      livekitRoom ??
+      new LivekitRoom({
+        ...defaultLiveKitOptions,
+        e2ee: e2eeLivekitOptions,
+      });
+    this.participantsIncludingSubscribers$ = this.scope.behavior(
+      connectedParticipantsObserver(this.livekitRoom),
+      [],
+    );
+
+    this.publishingParticipants$ = this.scope.behavior(
+      combineLatest([
+        this.participantsIncludingSubscribers$,
+        this.membershipsFocusMap$,
+      ]).pipe(
+        map(([participants, membershipsFocusMap]) =>
+          membershipsFocusMap
+            // Find all members that claim to publish on this connection
+            .flatMap(({ membership, focus }) =>
+              focus.livekit_service_url === this.focus.livekit_service_url
+                ? [membership]
+                : [],
+            )
+            // Find all associated publishing livekit participant objects
+            .flatMap((membership) => {
+              const participant = participants.find(
+                (p) =>
+                  p.identity === `${membership.sender}:${membership.deviceId}`,
+              );
+              return participant ? [{ participant, membership }] : [];
+            }),
+        ),
+      ),
+      [],
+    );
+    this.connectionState$ = this.scope.behavior<ConnectionState>(
+      connectionStateObserver(this.livekitRoom),
+    );
+  }
+}
+
+export class PublishConnection extends Connection {
+  public async start(): Promise<void> {
+    this.stopped = false;
+    const { url, jwt } = await this.sfuConfig;
+    if (!this.stopped) await this.livekitRoom.connect(url, jwt);
+
+    if (!this.stopped) {
+      const tracks = await this.livekitRoom.localParticipant.createTracks({
+        audio: this.muteStates.audio.enabled$.value,
+        video: this.muteStates.video.enabled$.value,
+      });
+      for (const track of tracks) {
+        await this.livekitRoom.localParticipant.publishTrack(track);
+      }
+    }
+  }
+
+  public stop(): void {
+    void this.livekitRoom.disconnect();
+    this.stopped = true;
+  }
+
+  public constructor(
+    focus: LivekitFocus,
+    livekitAlias: string,
+    client: MatrixClient,
+    scope: ObservableScope,
+    membershipsFocusMap$: Behavior<
+      { membership: CallMembership; focus: LivekitFocus }[]
+    >,
+    devices: MediaDevices,
+    private readonly muteStates: MuteStates,
+    e2eeLivekitOptions: E2EEOptions | undefined,
+  ) {
+    logger.info("[LivekitRoom] Create LiveKit room");
+    const { controlledAudioDevices } = getUrlParams();
+
+    const room = new LivekitRoom({
+      ...defaultLiveKitOptions,
+      videoCaptureDefaults: {
+        ...defaultLiveKitOptions.videoCaptureDefaults,
+        deviceId: devices.videoInput.selected$.value?.id,
+        // TODO-MULTI-SFU add processor support back
+        // processor,
+      },
+      audioCaptureDefaults: {
+        ...defaultLiveKitOptions.audioCaptureDefaults,
+        deviceId: devices.audioInput.selected$.value?.id,
+      },
+      audioOutput: {
+        // When using controlled audio devices, we don't want to set the
+        // deviceId here, because it will be set by the native app.
+        // (also the id does not need to match a browser device id)
+        deviceId: controlledAudioDevices
+          ? undefined
+          : getValue(devices.audioOutput.selected$)?.id,
+      },
+      e2ee: e2eeLivekitOptions,
+    });
+    room.setE2EEEnabled(e2eeLivekitOptions !== undefined).catch((e) => {
+      logger.error("Failed to set E2EE enabled on room", e);
+    });
+
+    super(
+      focus,
+      livekitAlias,
+      client,
+      scope,
+      membershipsFocusMap$,
+      e2eeLivekitOptions,
+      room,
+    );
+
+    this.muteStates.audio.setHandler(async (desired) => {
+      try {
+        await this.livekitRoom.localParticipant.setMicrophoneEnabled(desired);
+      } catch (e) {
+        logger.error("Failed to update LiveKit audio input mute state", e);
+      }
+      return this.livekitRoom.localParticipant.isMicrophoneEnabled;
+    });
+    this.muteStates.video.setHandler(async (desired) => {
+      try {
+        await this.livekitRoom.localParticipant.setCameraEnabled(desired);
+      } catch (e) {
+        logger.error("Failed to update LiveKit video input mute state", e);
+      }
+      return this.livekitRoom.localParticipant.isCameraEnabled;
+    });
+    // TODO-MULTI-SFU: Unset mute state handlers on destroy
+
+    const syncDevice = (
+      kind: MediaDeviceKind,
+      selected$: Observable<SelectedDevice | undefined>,
+    ): Subscription =>
+      selected$.pipe(this.scope.bind()).subscribe((device) => {
+        if (this.connectionState$.value !== ConnectionState.Connected) return;
+        logger.info(
+          "[LivekitRoom] syncDevice room.getActiveDevice(kind) !== d.id :",
+          this.livekitRoom.getActiveDevice(kind),
+          " !== ",
+          device?.id,
+        );
+        if (
+          device !== undefined &&
+          this.livekitRoom.getActiveDevice(kind) !== device.id
+        ) {
+          this.livekitRoom
+            .switchActiveDevice(kind, device.id)
+            .catch((e) =>
+              logger.error(`Failed to sync ${kind} device with LiveKit`, e),
+            );
+        }
+      });
+
+    syncDevice("audioinput", devices.audioInput.selected$);
+    if (!controlledAudioDevices)
+      syncDevice("audiooutput", devices.audioOutput.selected$);
+    syncDevice("videoinput", devices.videoInput.selected$);
+    // Restart the audio input track whenever we detect that the active media
+    // device has changed to refer to a different hardware device. We do this
+    // for the sake of Chrome, which provides a "default" device that is meant
+    // to match the system's default audio input, whatever that may be.
+    // This is special-cased for only audio inputs because we need to dig around
+    // in the LocalParticipant object for the track object and there's not a nice
+    // way to do that generically. There is usually no OS-level default video capture
+    // device anyway, and audio outputs work differently.
+    devices.audioInput.selected$
+      .pipe(
+        switchMap((device) => device?.hardwareDeviceChange$ ?? NEVER),
+        this.scope.bind(),
+      )
+      .subscribe(() => {
+        if (this.connectionState$.value !== ConnectionState.Connected) return;
+        const activeMicTrack = Array.from(
+          this.livekitRoom.localParticipant.audioTrackPublications.values(),
+        ).find((d) => d.source === Track.Source.Microphone)?.track;
+
+        if (
+          activeMicTrack &&
+          // only restart if the stream is still running: LiveKit will detect
+          // when a track stops & restart appropriately, so this is not our job.
+          // Plus, we need to avoid restarting again if the track is already in
+          // the process of being restarted.
+          activeMicTrack.mediaStreamTrack.readyState !== "ended"
+        ) {
+          // Restart the track, which will cause Livekit to do another
+          // getUserMedia() call with deviceId: default to get the *new* default device.
+          // Note that room.switchActiveDevice() won't work: Livekit will ignore it because
+          // the deviceId hasn't changed (was & still is default).
+          this.livekitRoom.localParticipant
+            .getTrackPublication(Track.Source.Microphone)
+            ?.audioTrack?.restartTrack()
+            .catch((e) => {
+              logger.error(`Failed to restart audio device track`, e);
+            });
+        }
+      });
+  }
+  // TODO-MULTI-SFU Sync the requested track processors with LiveKit
+}

src/state/MuteStates.ts

@@ -0,0 +1,207 @@
+/*
+Copyright 2023-2025 New Vector Ltd.
+
+SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
+Please see LICENSE in the repository root for full details.
+*/
+
+import { type IWidgetApiRequest } from "matrix-widget-api";
+import { logger } from "matrix-js-sdk/lib/logger";
+import {
+  BehaviorSubject,
+  combineLatest,
+  distinctUntilChanged,
+  firstValueFrom,
+  fromEvent,
+  map,
+  merge,
+  Observable,
+  of,
+  Subject,
+  switchMap,
+  withLatestFrom,
+} from "rxjs";
+
+import { type MediaDevices, type MediaDevice } from "../state/MediaDevices";
+import { ElementWidgetActions, widget } from "../widget";
+import { Config } from "../config/Config";
+import { getUrlParams } from "../UrlParams";
+import { type ObservableScope } from "./ObservableScope";
+import { type Behavior } from "./Behavior";
+
+interface MuteStateData {
+  enabled$: Observable<boolean>;
+  set: ((enabled: boolean) => void) | null;
+  toggle: (() => void) | null;
+}
+
+export type Handler = (desired: boolean) => Promise<boolean>;
+const defaultHandler: Handler = async (desired) => Promise.resolve(desired);
+
+class MuteState<Label, Selected> {
+  private readonly enabledByDefault$ =
+    this.enabledByConfig && !getUrlParams().skipLobby
+      ? this.joined$.pipe(map((isJoined) => !isJoined))
+      : of(false);
+
+  private readonly handler$ = new BehaviorSubject(defaultHandler);
+  public setHandler(handler: Handler): void {
+    if (this.handler$.value !== defaultHandler)
+      throw new Error("Multiple mute state handlers are not supported");
+    this.handler$.next(handler);
+  }
+  public unsetHandler(): void {
+    this.handler$.next(defaultHandler);
+  }
+
+  private readonly data$ = this.scope.behavior<MuteStateData>(
+    this.device.available$.pipe(
+      map((available) => available.size > 0),
+      distinctUntilChanged(),
+      withLatestFrom(
+        this.enabledByDefault$,
+        (devicesConnected, enabledByDefault) => {
+          if (!devicesConnected)
+            return { enabled$: of(false), set: null, toggle: null };
+
+          // Assume the default value only once devices are actually connected
+          let enabled = enabledByDefault;
+          const set$ = new Subject<boolean>();
+          const toggle$ = new Subject<void>();
+          const desired$ = merge(set$, toggle$.pipe(map(() => !enabled)));
+          const enabled$ = new Observable<boolean>((subscriber) => {
+            subscriber.next(enabled);
+            let latestDesired = enabledByDefault;
+            let syncing = false;
+
+            const sync = async (): Promise<void> => {
+              if (enabled === latestDesired) syncing = false;
+              else {
+                const previouslyEnabled = enabled;
+                enabled = await firstValueFrom(
+                  this.handler$.pipe(
+                    switchMap(async (handler) => handler(latestDesired)),
+                  ),
+                );
+                if (enabled === previouslyEnabled) {
+                  syncing = false;
+                } else {
+                  subscriber.next(enabled);
+                  syncing = true;
+                  sync();
+                }
+              }
+            };
+
+            const s = desired$.subscribe((desired) => {
+              latestDesired = desired;
+              if (syncing === false) {
+                syncing = true;
+                sync();
+              }
+            });
+            return (): void => s.unsubscribe();
+          });
+
+          return {
+            set: (enabled: boolean): void => set$.next(enabled),
+            toggle: (): void => toggle$.next(),
+            enabled$,
+          };
+        },
+      ),
+    ),
+  );
+
+  public readonly enabled$: Behavior<boolean> = this.scope.behavior(
+    this.data$.pipe(switchMap(({ enabled$ }) => enabled$)),
+  );
+
+  public readonly setEnabled$: Behavior<((enabled: boolean) => void) | null> =
+    this.scope.behavior(this.data$.pipe(map(({ set }) => set)));
+
+  public readonly toggle$: Behavior<(() => void) | null> = this.scope.behavior(
+    this.data$.pipe(map(({ toggle }) => toggle)),
+  );
+
+  public constructor(
+    private readonly scope: ObservableScope,
+    private readonly device: MediaDevice<Label, Selected>,
+    private readonly joined$: Observable<boolean>,
+    private readonly enabledByConfig: boolean,
+  ) {}
+}
+
+export class MuteStates {
+  public readonly audio = new MuteState(
+    this.scope,
+    this.mediaDevices.audioInput,
+    this.joined$,
+    Config.get().media_devices.enable_video,
+  );
+  public readonly video = new MuteState(
+    this.scope,
+    this.mediaDevices.videoInput,
+    this.joined$,
+    Config.get().media_devices.enable_video,
+  );
+
+  public constructor(
+    private readonly scope: ObservableScope,
+    private readonly mediaDevices: MediaDevices,
+    private readonly joined$: Observable<boolean>,
+  ) {
+    if (widget !== null) {
+      // Sync our mute states with the hosting client
+      const widgetApiState$ = combineLatest(
+        [this.audio.enabled$, this.video.enabled$],
+        (audio, video) => ({ audio_enabled: audio, video_enabled: video }),
+      );
+      widgetApiState$.pipe(this.scope.bind()).subscribe((state) => {
+        widget!.api.transport
+          .send(ElementWidgetActions.DeviceMute, state)
+          .catch((e) =>
+            logger.warn("Could not send DeviceMute action to widget", e),
+          );
+      });
+
+      // Also sync the hosting client's mute states back with ours
+      const muteActions$ = fromEvent(
+        widget.lazyActions,
+        ElementWidgetActions.DeviceMute,
+      ) as Observable<CustomEvent<IWidgetApiRequest>>;
+      muteActions$
+        .pipe(
+          withLatestFrom(
+            widgetApiState$,
+            this.audio.setEnabled$,
+            this.video.setEnabled$,
+          ),
+          this.scope.bind(),
+        )
+        .subscribe(([ev, state, setAudioEnabled, setVideoEnabled]) => {
+          // First copy the current state into our new state
+          const newState = { ...state };
+          // Update new state if there are any requested changes from the widget
+          // action in `ev.detail.data`.
+          if (
+            ev.detail.data.audio_enabled != null &&
+            typeof ev.detail.data.audio_enabled === "boolean" &&
+            setAudioEnabled !== null
+          ) {
+            newState.audio_enabled = ev.detail.data.audio_enabled;
+            setAudioEnabled(newState.audio_enabled);
+          }
+          if (
+            ev.detail.data.video_enabled != null &&
+            typeof ev.detail.data.video_enabled === "boolean" &&
+            setVideoEnabled !== null
+          ) {
+            newState.video_enabled = ev.detail.data.video_enabled;
+            setVideoEnabled(newState.video_enabled);
+          }
+          widget!.api.transport.reply(ev.detail, newState);
+        });
+    }
+  }
+}

src/state/ObservableScope.ts

@@ -9,6 +9,7 @@ import {
   BehaviorSubject,
   distinctUntilChanged,
   type Observable,
+  share,
   Subject,
   takeUntil,
 } from "rxjs";
@@ -35,6 +36,12 @@ export class ObservableScope {
     return this.bindImpl;
   }
 
+  private readonly shareImpl: MonoTypeOperator = share({ resetOnError: false, resetOnComplete: false, resetOnRefCountZero: false })
+  /**
+   * Shares (multicasts) the Observable as a hot Observable.
+   */
+  public readonly share: MonoTypeOperator = (input$) => input$.pipe(this.bindImpl, this.shareImpl)
+
   /**
    * Converts an Observable to a Behavior. If no initial value is specified, the
    * Observable must synchronously emit an initial value.

src/useCallViewKeyboardShortcuts.ts

@@ -29,9 +29,9 @@ const KeyToReactionMap: Record<string, ReactionOption> = Object.fromEntries(
 
 export function useCallViewKeyboardShortcuts(
   focusElement: RefObject<HTMLElement | null>,
-  toggleMicrophoneMuted: () => void,
-  toggleLocalVideoMuted: () => void,
-  setMicrophoneMuted: (muted: boolean) => void,
+  toggleAudio: (() => void) | null,
+  toggleVideo: (() => void) | null,
+  setAudioEnabled: ((enabled: boolean) => void) | null,
   sendReaction: (reaction: ReactionOption) => void,
   toggleHandRaised: () => void,
 ): void {
@@ -52,15 +52,15 @@ export function useCallViewKeyboardShortcuts(
 
         if (event.key === "m") {
           event.preventDefault();
-          toggleMicrophoneMuted();
-        } else if (event.key == "v") {
+          toggleAudio?.();
+        } else if (event.key === "v") {
           event.preventDefault();
-          toggleLocalVideoMuted();
+          toggleVideo?.();
         } else if (event.key === " ") {
           event.preventDefault();
           if (!spacebarHeld.current) {
             spacebarHeld.current = true;
-            setMicrophoneMuted(false);
+            setAudioEnabled?.(true);
           }
         } else if (event.key === "h") {
           event.preventDefault();
@@ -72,9 +72,9 @@ export function useCallViewKeyboardShortcuts(
       },
       [
         focusElement,
-        toggleLocalVideoMuted,
-        toggleMicrophoneMuted,
-        setMicrophoneMuted,
+        toggleVideo,
+        toggleAudio,
+        setAudioEnabled,
         sendReaction,
         toggleHandRaised,
       ],
@@ -95,10 +95,10 @@ export function useCallViewKeyboardShortcuts(
 
         if (event.key === " ") {
           spacebarHeld.current = false;
-          setMicrophoneMuted(true);
+          setAudioEnabled?.(false);
         }
       },
-      [focusElement, setMicrophoneMuted],
+      [focusElement, setAudioEnabled],
     ),
   );
 
@@ -108,8 +108,8 @@ export function useCallViewKeyboardShortcuts(
     useCallback(() => {
       if (spacebarHeld.current) {
         spacebarHeld.current = false;
-        setMicrophoneMuted(true);
+        setAudioEnabled?.(true);
       }
-    }, [setMicrophoneMuted, spacebarHeld]),
+    }, [setAudioEnabled, spacebarHeld]),
   );
 }

yarn.lock

@@ -10278,9 +10278,9 @@ __metadata:
   languageName: node
   linkType: hard
 
-"matrix-js-sdk@github:matrix-org/matrix-js-sdk#head=develop":
-  version: 37.13.0
-  resolution: "matrix-js-sdk@https://github.com/matrix-org/matrix-js-sdk.git#commit=3a33c658bbcb8ce8791ec066db899f2571f5c52f"
+"matrix-js-sdk@portal:/Users/timo/Projects/matrix-js-sdk::locator=element-call%40workspace%3A.":
+  version: 0.0.0-use.local
+  resolution: "matrix-js-sdk@portal:/Users/timo/Projects/matrix-js-sdk::locator=element-call%40workspace%3A."
   dependencies:
     "@babel/runtime": "npm:^7.12.5"
     "@matrix-org/matrix-sdk-crypto-wasm": "npm:^15.1.0"
@@ -10296,9 +10296,8 @@ __metadata:
     sdp-transform: "npm:^2.14.1"
     unhomoglyph: "npm:^1.0.6"
     uuid: "npm:11"
-  checksum: 10c0/1db0d39cfbe4f1c69c8acda0ea7580a4819fc47a7d4bff057382e33e72d9a610f8c03043a6c00bc647dfdc2815aa643c69d25022fb759342a92b77e1841524f1
   languageName: node
-  linkType: hard
+  linkType: soft
 
 "matrix-widget-api@npm:^1.10.0, matrix-widget-api@npm:^1.13.0":
   version: 1.13.1