Merge branch 'livekit' into toger5/delayed-event-delegation

src/state/CallViewModel/CallNotificationLifecycle.ts

@@ -81,7 +81,7 @@ export interface Props {
   localUser: { deviceId: string; userId: string };
 }
 /**
- * @returns {callPickupState$, autoLeave$}
+ * @returns two observables:
  * `callPickupState$` The current call pickup state of the call.
  *  - "unknown": The client has not yet sent the notification event. We don't know if it will because it first needs to send its own membership.
  *     Then we can conclude if we were the first one to join or not.

src/state/CallViewModel/CallViewModel.ts

@@ -82,7 +82,7 @@ import {
 } from "../../reactions";
 import { shallowEquals } from "../../utils/array";
 import { type MediaDevices } from "../MediaDevices";
-import { type Behavior } from "../Behavior";
+import { constant, type Behavior } from "../Behavior";
 import { E2eeType } from "../../e2ee/e2eeType";
 import { MatrixKeyProvider } from "../../e2ee/matrixKeyProvider";
 import { type MuteStates } from "../MuteStates";
@@ -119,6 +119,7 @@ import {
   createMatrixLivekitMembers$,
   type TaggedParticipant,
   type LocalMatrixLivekitMember,
+  type RemoteMatrixLivekitMember,
 } from "./remoteMembers/MatrixLivekitMembers.ts";
 import {
   type AutoLeaveReason,
@@ -158,7 +159,7 @@ export interface CallViewModelOptions {
   /** Optional behavior overriding the computed window size, mainly for testing purposes. */
   windowSize$?: Behavior<{ width: number; height: number }>;
   /** The version & compatibility mode of MatrixRTC that we should use. */
-  matrixRTCMode$: Behavior<MatrixRTCMode>;
+  matrixRTCMode$?: Behavior<MatrixRTCMode>;
 }
 
 // Do not play any sounds if the participant count has exceeded this
@@ -184,7 +185,7 @@ interface LayoutScanState {
 }
 
 type MediaItem = UserMedia | ScreenShare;
-type AudioLivekitItem = {
+export type LivekitRoomItem = {
   livekitRoom: LivekitRoom;
   participants: string[];
   url: string;
@@ -207,8 +208,11 @@ export interface CallViewModel {
   callPickupState$: Behavior<
     "unknown" | "ringing" | "timeout" | "decline" | "success" | null
   >;
+  /** Observable that emits when the user should leave the call (hangup pressed, widget action, error).
+   * THIS DOES NOT LEAVE THE CALL YET. The only way to leave the call (send the hangup event) is by ending the scope.
+   */
   leave$: Observable<"user" | AutoLeaveReason>;
-  /** Call to initiate hangup. Use in conbination with connectino state track the async hangup process. */
+  /** Call to initiate hangup. Use in conbination with reconnectino state track the async hangup process. */
   hangup: () => void;
 
   // joining
@@ -260,7 +264,11 @@ export interface CallViewModel {
    */
   participantCount$: Behavior<number>;
   /** Participants sorted by livekit room so they can be used in the audio rendering */
-  audioParticipants$: Behavior<AudioLivekitItem[]>;
+  livekitRoomItems$: Behavior<LivekitRoomItem[]>;
+  userMedia$: Behavior<UserMedia[]>;
+  /** use the layout instead, this is just for the sdk export. */
+  matrixLivekitMembers$: Behavior<RemoteMatrixLivekitMember[]>;
+  localMatrixLivekitMember$: Behavior<LocalMatrixLivekitMember | null>;
   /** List of participants raising their hand */
   handsRaised$: Behavior<Record<string, RaisedHandInfo>>;
   /** List of reactions. Keys are: membership.membershipId (currently predefined as: `${membershipEvent.userId}:${membershipEvent.deviceId}`)*/
@@ -343,17 +351,15 @@ export interface CallViewModel {
     switch: () => void;
   } | null>;
 
-  // connection state
   /**
-   * Whether various media/event sources should pretend to be disconnected from
-   * all network input, even if their connection still technically works.
+   * Whether the app is currently reconnecting to the LiveKit server and/or setting the matrix rtc room state.
    */
-  // We do this when the app is in the 'reconnecting' state, because it might be
-  // that the LiveKit connection is still functional while the homeserver is
-  // down, for example, and we want to avoid making people worry that the app is
-  // in a split-brained state.
-  // DISCUSSION own membership manager ALSO this probably can be simplifis
   reconnecting$: Behavior<boolean>;
+
+  /**
+   * Shortcut for not requireing to parse and combine connectionState.matrix and connectionState.livekit
+   */
+  connected$: Behavior<boolean>;
 }
 
 /**
@@ -386,6 +392,8 @@ export function createCallViewModel$(
     options.encryptionSystem,
     matrixRTCSession,
   );
+  const matrixRTCMode$ =
+    options.matrixRTCMode$ ?? constant(MatrixRTCMode.Legacy);
 
   // Each hbar seperates a block of input variables required for the CallViewModel to function.
   // The outputs of this block is written under the hbar.
@@ -420,7 +428,7 @@ export function createCallViewModel$(
   };
 
   const useOldJwtEndpoint$ = scope.behavior(
-    options.matrixRTCMode$.pipe(map((v) => v !== MatrixRTCMode.Matrix_2_0)),
+    matrixRTCMode$.pipe(map((v) => v !== MatrixRTCMode.Matrix_2_0)),
   );
   const localTransport$ = createLocalTransport$({
     scope: scope,
@@ -439,7 +447,7 @@ export function createCallViewModel$(
     roomId: matrixRoom.roomId,
     useOldJwtEndpoint$,
     useOldestMember$: scope.behavior(
-      options.matrixRTCMode$.pipe(map((v) => v === MatrixRTCMode.Legacy)),
+      matrixRTCMode$.pipe(map((v) => v === MatrixRTCMode.Legacy)),
     ),
   });
 
@@ -482,7 +490,7 @@ export function createCallViewModel$(
   });
 
   const connectOptions$ = scope.behavior(
-    options.matrixRTCMode$.pipe(
+    matrixRTCMode$.pipe(
       map((mode) => ({
         encryptMedia: livekitKeyProvider !== undefined,
         // TODO. This might need to get called again on each change of matrixRTCMode...
@@ -515,7 +523,7 @@ export function createCallViewModel$(
         muteStates,
         trackProcessorState$,
         logger.getChild(
-          "[Publisher" + connection.transport.livekit_service_url + "]",
+          "[Publisher " + connection.transport.livekit_service_url + "]",
         ),
       );
     },
@@ -614,7 +622,7 @@ export function createCallViewModel$(
     ),
   );
 
-  const audioParticipants$ = scope.behavior(
+  const livekitRoomItems$ = scope.behavior(
     matrixLivekitMembers$.pipe(
       switchMap((members) => {
         const a$ = combineLatest(
@@ -639,7 +647,7 @@ export function createCallViewModel$(
         return a$;
       }),
       map((members) =>
-        members.reduce<AudioLivekitItem[]>((acc, curr) => {
+        members.reduce<LivekitRoomItem[]>((acc, curr) => {
           if (!curr) return acc;
 
           const existing = acc.find((item) => item.url === curr.url);
@@ -1509,10 +1517,7 @@ export function createCallViewModel$(
       ),
       null,
     ),
-
     participantCount$: participantCount$,
-    audioParticipants$: audioParticipants$,
-
     handsRaised$: handsRaised$,
     reactions$: reactions$,
     joinSoundEffect$: joinSoundEffect$,
@@ -1531,6 +1536,16 @@ export function createCallViewModel$(
     spotlight$: spotlight$,
     pip$: pip$,
     layout$: layout$,
+    userMedia$,
+    localMatrixLivekitMember$,
+    matrixLivekitMembers$: scope.behavior(
+      matrixLivekitMembers$.pipe(
+        map((members) => members.value),
+        tap((v) => {
+          logger.debug("matrixLivekitMembers$ updated (exported)", v);
+        }),
+      ),
+    ),
     tileStoreGeneration$: tileStoreGeneration$,
     showSpotlightIndicators$: showSpotlightIndicators$,
     showSpeakingIndicators$: showSpeakingIndicators$,
@@ -1539,6 +1554,8 @@ export function createCallViewModel$(
     earpieceMode$: earpieceMode$,
     audioOutputSwitcher$: audioOutputSwitcher$,
     reconnecting$: localMembership.reconnecting$,
+    livekitRoomItems$,
+    connected$: localMembership.connected$,
   };
 }
 

src/state/CallViewModel/localMember/LocalMember.ts

@@ -139,7 +139,16 @@ interface Props {
  * We want
  *  - a publisher
  *  -
- * @param param0
+ * @param props The properties required to create the local membership.
+ * @param props.scope The observable scope to use.
+ * @param props.connectionManager The connection manager to get connections from.
+ * @param props.createPublisherFactory Factory to create a publisher once we have a connection.
+ * @param props.joinMatrixRTC Callback to join the matrix RTC session once we have a transport.
+ * @param props.homeserverConnected The homeserver connected state.
+ * @param props.localTransport$ The local transport to use for publishing.
+ * @param props.logger The logger to use.
+ * @param props.muteStates The mute states for video and audio.
+ * @param props.matrixRTCSession The matrix RTC session to join.
  * @returns
  *  - publisher: The handle to create tracks and publish them to the room.
  *  - connected$: the current connection state. Including matrix server and livekit server connection. (only considering the livekit server we are using for our own media publication)
@@ -178,14 +187,18 @@ export const createLocalMembership$ = ({
   // tracks$: Behavior<LocalTrack[]>;
   participant$: Behavior<LocalParticipant | null>;
   connection$: Behavior<Connection | null>;
-  /** Shorthand for homeserverConnected.rtcSession === Status.Reconnecting
-   * Direct translation to the js-sdk membership manager connection `Status`.
+  /**
+   * Tracks the homserver and livekit connected state and based on that computes reconnecting.
    */
   reconnecting$: Behavior<boolean>;
   /** Shorthand for homeserverConnected.rtcSession === Status.Disconnected
    * Direct translation to the js-sdk membership manager connection `Status`.
    */
   disconnected$: Behavior<boolean>;
+  /**
+   * Fully connected
+   */
+  connected$: Behavior<boolean>;
 } => {
   const logger = parentLogger.getChild("[LocalMembership]");
   logger.debug(`Creating local membership..`);
@@ -639,6 +652,7 @@ export const createLocalMembership$ = ({
     localMemberState$,
     participant$,
     reconnecting$,
+    connected$: matrixAndLivekitConnected$,
     disconnected$: scope.behavior(
       homeserverConnected.rtsSession$.pipe(
         map((state) => state === RTCSessionStatus.Disconnected),
@@ -672,9 +686,11 @@ interface EnterRTCSessionOptions {
  *      - Delay events management
  *      - Handles retries (fails only after several attempts)
  *
- * @param rtcSession
- * @param transport
- * @param options
+ * @param rtcSession - The MatrixRTCSession to join.
+ * @param transport - The LivekitTransport to use for this session.
+ * @param options - Options for entering the RTC session.
+ * @param options.encryptMedia - Whether to encrypt media.
+ * @param options.matrixRTCMode - The Matrix RTC mode to use.
  * @throws If the widget could not send ElementWidgetActions.JoinCall action.
  */
 // Exported for unit testing

src/state/CallViewModel/localMember/LocalTransport.test.ts

@@ -5,9 +5,18 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
 Please see LICENSE in the repository root for full details.
 */
 
-import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+import {
+  afterEach,
+  beforeEach,
+  describe,
+  expect,
+  it,
+  type MockedObject,
+  vi,
+} from "vitest";
 import { type CallMembership } from "matrix-js-sdk/lib/matrixrtc";
-import { BehaviorSubject } from "rxjs";
+import { BehaviorSubject, lastValueFrom } from "rxjs";
+import fetchMock from "fetch-mock";
 
 import { mockConfig, flushPromises, ownMemberMock } from "../../../utils/test";
 import { createLocalTransport$ } from "./LocalTransport";
@@ -18,8 +27,17 @@ import {
   FailToGetOpenIdToken,
 } from "../../../utils/errors";
 import * as openIDSFU from "../../../livekit/openIDSFU";
+import { customLivekitUrl } from "../../../settings/settings";
+import { testJWTToken } from "../../../utils/test-fixtures";
 
 describe("LocalTransport", () => {
+  const openIdResponse: openIDSFU.SFUConfig = {
+    url: "https://lk.example.org",
+    jwt: testJWTToken,
+    livekitAlias: "!example_room_id",
+    livekitIdentity: "@lk_user:ABCDEF",
+  };
+
   let scope: ObservableScope;
   beforeEach(() => (scope = new ObservableScope()));
   afterEach(() => scope.end());
@@ -65,13 +83,15 @@ describe("LocalTransport", () => {
     const errors: Error[] = [];
     const localTransport$ = createLocalTransport$({
       scope,
-      roomId: "!room:example.org",
+      roomId: "!example_room_id",
       useOldestMember$: constant(false),
       memberships$: constant(new Epoch<CallMembership[]>([])),
       client: {
         baseUrl: "https://lk.example.org",
         // Use empty domain to skip .well-known and use config directly
         getDomain: () => "",
+        // eslint-disable-next-line @typescript-eslint/naming-convention
+        _unstable_getRTCTransports: async () => Promise.resolve([]),
         getOpenIdToken: vi.fn(),
         getDeviceId: vi.fn(),
       },
@@ -145,11 +165,13 @@ describe("LocalTransport", () => {
 
     const localTransport$ = createLocalTransport$({
       scope,
-      roomId: "!room:example.org",
+      roomId: "!example_room_id",
       useOldestMember$: constant(true),
       memberships$,
       client: {
         getDomain: () => "",
+        // eslint-disable-next-line @typescript-eslint/naming-convention
+        _unstable_getRTCTransports: async () => Promise.resolve([]),
         getOpenIdToken: vi.fn(),
         getDeviceId: vi.fn(),
         baseUrl: "https://lk.example.org",
@@ -159,14 +181,159 @@ describe("LocalTransport", () => {
       delayId$: constant("delay_id_mock"),
     });
 
-    openIdResolver.resolve?.({ url: "https://lk.example.org", jwt: "jwt" });
+    openIdResolver.resolve?.(openIdResponse);
     expect(localTransport$.value).toBe(null);
     await flushPromises();
     // final
     expect(localTransport$.value).toStrictEqual({
-      livekit_alias: "!room:example.org",
+      livekit_alias: "!example_room_id",
       livekit_service_url: "https://lk.example.org",
       type: "livekit",
     });
   });
+
+  type LocalTransportProps = Parameters<typeof createLocalTransport$>[0];
+
+  describe("transport configuration mechanisms", () => {
+    let localTransportOpts: LocalTransportProps & {
+      client: MockedObject<LocalTransportProps["client"]>;
+    };
+    let openIdResolver: PromiseWithResolvers<openIDSFU.SFUConfig>;
+    beforeEach(() => {
+      mockConfig({});
+      customLivekitUrl.setValue(customLivekitUrl.defaultValue);
+      localTransportOpts = {
+        scope,
+        roomId: "!example_room_id",
+        useOldestMember$: constant(false),
+        memberships$: constant(new Epoch<CallMembership[]>([])),
+        client: {
+          getDomain: vi.fn().mockReturnValue(""),
+          // eslint-disable-next-line @typescript-eslint/naming-convention
+          _unstable_getRTCTransports: vi.fn().mockResolvedValue([]),
+          getOpenIdToken: vi.fn(),
+          getDeviceId: vi.fn(),
+        },
+      };
+      openIdResolver = Promise.withResolvers<openIDSFU.SFUConfig>();
+      vi.spyOn(openIDSFU, "getSFUConfigWithOpenID").mockReturnValue(
+        openIdResolver.promise,
+      );
+    });
+
+    afterEach(() => {
+      fetchMock.reset();
+    });
+
+    it("supports getting transport via application config", async () => {
+      mockConfig({
+        livekit: { livekit_service_url: "https://lk.example.org" },
+      });
+      const localTransport$ = createLocalTransport$(localTransportOpts);
+      openIdResolver.resolve?.(openIdResponse);
+      expect(localTransport$.value).toBe(null);
+      await flushPromises();
+      expect(localTransport$.value).toStrictEqual({
+        livekit_alias: "!example_room_id",
+        livekit_service_url: "https://lk.example.org",
+        type: "livekit",
+      });
+    });
+    it("supports getting transport via user settings", async () => {
+      customLivekitUrl.setValue("https://lk.example.org");
+      const localTransport$ = createLocalTransport$(localTransportOpts);
+      openIdResolver.resolve?.(openIdResponse);
+      expect(localTransport$.value).toBe(null);
+      await flushPromises();
+      expect(localTransport$.value).toStrictEqual({
+        livekit_alias: "!example_room_id",
+        livekit_service_url: "https://lk.example.org",
+        type: "livekit",
+      });
+    });
+    it("supports getting transport via backend", async () => {
+      localTransportOpts.client._unstable_getRTCTransports.mockResolvedValue([
+        { type: "livekit", livekit_service_url: "https://lk.example.org" },
+      ]);
+      const localTransport$ = createLocalTransport$(localTransportOpts);
+      openIdResolver.resolve?.(openIdResponse);
+      expect(localTransport$.value).toBe(null);
+      await flushPromises();
+      expect(localTransport$.value).toStrictEqual({
+        livekit_alias: "!example_room_id",
+        livekit_service_url: "https://lk.example.org",
+        type: "livekit",
+      });
+    });
+    it("fails fast if the openID request fails for backend config", async () => {
+      localTransportOpts.client._unstable_getRTCTransports.mockResolvedValue([
+        { type: "livekit", livekit_service_url: "https://lk.example.org" },
+      ]);
+      openIdResolver.reject(
+        new FailToGetOpenIdToken(new Error("Test driven error")),
+      );
+      try {
+        await lastValueFrom(createLocalTransport$(localTransportOpts));
+        throw Error("Expected test to throw");
+      } catch (ex) {
+        expect(ex).toBeInstanceOf(FailToGetOpenIdToken);
+      }
+    });
+    it("supports getting transport via well-known", async () => {
+      localTransportOpts.client.getDomain.mockReturnValue("example.org");
+      fetchMock.getOnce("https://example.org/.well-known/matrix/client", {
+        "org.matrix.msc4143.rtc_foci": [
+          { type: "livekit", livekit_service_url: "https://lk.example.org" },
+        ],
+      });
+      const localTransport$ = createLocalTransport$(localTransportOpts);
+      openIdResolver.resolve?.(openIdResponse);
+      expect(localTransport$.value).toBe(null);
+      await flushPromises();
+      expect(localTransport$.value).toStrictEqual({
+        livekit_alias: "!example_room_id",
+        livekit_service_url: "https://lk.example.org",
+        type: "livekit",
+      });
+      expect(fetchMock.done()).toEqual(true);
+    });
+    it("fails fast if the openId request fails for the well-known config", async () => {
+      localTransportOpts.client.getDomain.mockReturnValue("example.org");
+      fetchMock.getOnce("https://example.org/.well-known/matrix/client", {
+        "org.matrix.msc4143.rtc_foci": [
+          { type: "livekit", livekit_service_url: "https://lk.example.org" },
+        ],
+      });
+      openIdResolver.reject(
+        new FailToGetOpenIdToken(new Error("Test driven error")),
+      );
+      try {
+        await lastValueFrom(createLocalTransport$(localTransportOpts));
+        throw Error("Expected test to throw");
+      } catch (ex) {
+        expect(ex).toBeInstanceOf(FailToGetOpenIdToken);
+      }
+    });
+    it("throws if no options are available", async () => {
+      const localTransport$ = createLocalTransport$({
+        scope,
+        roomId: "!example_room_id",
+        useOldestMember$: constant(false),
+        memberships$: constant(new Epoch<CallMembership[]>([])),
+        client: {
+          getDomain: () => "",
+          // eslint-disable-next-line @typescript-eslint/naming-convention
+          _unstable_getRTCTransports: async () => Promise.resolve([]),
+          // These won't be called in this error path but satisfy the type
+          getOpenIdToken: vi.fn(),
+          getDeviceId: vi.fn(),
+        },
+      });
+      await flushPromises();
+
+      expect(() => localTransport$.value).toThrow(
+        new MatrixRTCTransportMissingError(""),
+      );
+    });
+  });
 });

src/state/CallViewModel/localMember/LocalTransport.ts

@@ -8,11 +8,11 @@ Please see LICENSE in the repository root for full details.
 import {
   type CallMembership,
   isLivekitTransport,
-  type LivekitTransportConfig,
   type LivekitTransport,
   isLivekitTransportConfig,
+  type Transport,
 } from "matrix-js-sdk/lib/matrixrtc";
-import { type MatrixClient } from "matrix-js-sdk";
+import { MatrixError, type MatrixClient } from "matrix-js-sdk";
 import {
   combineLatest,
   distinctUntilChanged,
@@ -28,7 +28,10 @@ import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/En
 import { type Behavior } from "../../Behavior.ts";
 import { type Epoch, type ObservableScope } from "../../ObservableScope.ts";
 import { Config } from "../../../config/Config.ts";
-import { MatrixRTCTransportMissingError } from "../../../utils/errors.ts";
+import {
+  FailToGetOpenIdToken,
+  MatrixRTCTransportMissingError,
+} from "../../../utils/errors.ts";
 import {
   getSFUConfigWithOpenID,
   type OpenIDClientParts,
@@ -47,7 +50,11 @@ interface Props {
   scope: ObservableScope;
   ownMembershipIdentity: CallMembershipIdentityParts;
   memberships$: Behavior<Epoch<CallMembership[]>>;
-  client: Pick<MatrixClient, "getDomain" | "baseUrl"> & OpenIDClientParts;
+  client: Pick<
+    MatrixClient,
+    "getDomain" | "baseUrl" | "_unstable_getRTCTransports"
+  > &
+    OpenIDClientParts;
   roomId: string;
   useOldestMember$: Behavior<boolean>;
   useOldJwtEndpoint$: Behavior<boolean>;
@@ -141,16 +148,30 @@ export const createLocalTransport$ = ({
 const FOCI_WK_KEY = "org.matrix.msc4143.rtc_foci";
 
 /**
+ * Determine the correct Transport for the current session, including
+ * validating auth against the service to ensure it's correct.
+ * Prefers in order:
  *
- * @param client
- * @param roomId
+
+ * 1. The `urlFromDevSettings` value. If this cannot be validated, the function will throw.
+ * 2. The transports returned via the homeserver.
+ * 3. The transports returned via .well-known.
+ * 4. The transport configured in Element Call's config.
+ *
+ * @param client The authenticated Matrix client for the current user
+ * @param roomId The ID of the room to be connected to.
+ * @param urlFromDevSettings Override URL provided by the user's local config.
  * @param useMatrix2 This implies using the matrix2 jwt endpoint (including delayed event delegation of the jwt token)
  * @param delayId
- * @returns
+ * @returns A fully validated transport config.
  * @throws MatrixRTCTransportMissingError | FailToGetOpenIdToken
  */
 async function makeTransport(
-  client: Pick<MatrixClient, "getDomain" | "baseUrl"> & OpenIDClientParts,
+  client: Pick<
+    MatrixClient,
+    "getDomain" | "baseUrl" | "_unstable_getRTCTransports"
+  > &
+    OpenIDClientParts,
   membership: CallMembershipIdentityParts,
   roomId: string,
   urlFromDevSettings: string | null,
@@ -159,51 +180,127 @@ async function makeTransport(
 ): Promise<LivekitTransport & { forceOldJwtEndpoint: boolean }> {
   let transport: LivekitTransport | undefined;
   logger.trace("Searching for a preferred transport");
-  //TODO refactor this to use the jwt service returned alias.
-  const livekitAlias = roomId;
+
+  // We will call `getSFUConfigWithOpenID` once per transport here as it's our
+  // only mechanism of valiation. This means we will also ask the
+  // homeserver for a OpenID token a few times. Since OpenID tokens are single
+  // use we don't want to risk any issues by re-using a token.
+  //
+  // If the OpenID request were to fail then it's acceptable for us to fail
+  // this function early, as we assume the homeserver has got some problems.
 
   // DEVTOOL: Highest priority: Load from devtool setting
   if (urlFromDevSettings !== null) {
-    const transportFromStorage: LivekitTransport = {
+    logger.info("Using LiveKit transport from dev tools: ", urlFromDevSettings);
+    // Validate that the SFU is up. Otherwise, we want to fail on this
+    // as we don't permit other SFUs.
+    const config = await getSFUConfigWithOpenID(
+      client,
+      urlFromDevSettings,
+      roomId,
+    );
+    return {
       type: "livekit",
       livekit_service_url: urlFromDevSettings,
-      livekit_alias: livekitAlias,
+      livekit_alias: config.livekitAlias,
     };
-    logger.info(
-      "Using LiveKit transport from dev tools: ",
-      transportFromStorage,
-    );
-    transport = transportFromStorage;
   }
 
-  // WELL_KNOWN: Prioritize the .well-known/matrix/client, if available, over the configured SFU
+  async function getFirstUsableTransport(
+    transports: Transport[],
+  ): Promise<LivekitTransport | null> {
+    for (const potentialTransport of transports) {
+      if (isLivekitTransportConfig(potentialTransport)) {
+        try {
+          const { livekitAlias } = await getSFUConfigWithOpenID(
+            client,
+            potentialTransport.livekit_service_url,
+            roomId,
+          );
+          return {
+            ...potentialTransport,
+            livekit_alias: livekitAlias,
+          };
+        } catch (ex) {
+          if (ex instanceof FailToGetOpenIdToken) {
+            // Explictly throw these
+            throw ex;
+          }
+          logger.debug(
+            `Could not use SFU service "${potentialTransport.livekit_service_url}" as SFU`,
+            ex,
+          );
+        }
+      }
+    }
+    return null;
+  }
+
+  // MSC4143: Attempt to fetch transports from backend.
+  if ("_unstable_getRTCTransports" in client) {
+    try {
+      const selectedTransport = await getFirstUsableTransport(
+        await client._unstable_getRTCTransports(),
+      );
+      if (selectedTransport) {
+        logger.info("Using backend-configured SFU", selectedTransport);
+        return selectedTransport;
+      }
+    } catch (ex) {
+      if (ex instanceof MatrixError && ex.httpStatus === 404) {
+        // Expected, this is an unstable endpoint and it's not required.
+        logger.debug("Backend does not provide any RTC transports", ex);
+      } else if (ex instanceof FailToGetOpenIdToken) {
+        throw ex;
+      } else {
+        // We got an error that wasn't just missing support for the feature, so log it loudly.
+        logger.error(
+          "Unexpected error fetching RTC transports from backend",
+          ex,
+        );
+      }
+    }
+  }
+
+  // Legacy MSC4143 (to be removed) WELL_KNOWN: Prioritize the .well-known/matrix/client, if available.
   const domain = client.getDomain();
-  if (domain && transport === undefined) {
+  if (domain) {
     // we use AutoDiscovery instead of relying on the MatrixClient having already
     // been fully configured and started
     const wellKnownFoci = (await AutoDiscovery.getRawClientConfig(domain))?.[
       FOCI_WK_KEY
     ];
-    if (Array.isArray(wellKnownFoci)) {
-      const wellKnownTransport: LivekitTransportConfig | undefined =
-        wellKnownFoci.find((f) => f && isLivekitTransportConfig(f));
-      if (wellKnownTransport !== undefined) {
-        logger.info("Using LiveKit transport from .well-known: ", transport);
-        transport = { ...wellKnownTransport, livekit_alias: livekitAlias };
-      }
+    const selectedTransport = Array.isArray(wellKnownFoci)
+      ? await getFirstUsableTransport(wellKnownFoci)
+      : null;
+    if (selectedTransport) {
+      logger.info("Using .well-known SFU", selectedTransport);
+      return selectedTransport;
     }
   }
 
   // CONFIG: Least prioritized; Load from config file
   const urlFromConf = Config.get().livekit?.livekit_service_url;
-  if (urlFromConf && transport === undefined) {
-    const transportFromConf: LivekitTransport = {
-      type: "livekit",
-      livekit_service_url: urlFromConf,
-      livekit_alias: livekitAlias,
-    };
-    logger.info("Using LiveKit transport from config: ", transportFromConf);
-    transport = transportFromConf;
+  if (urlFromConf) {
+    try {
+      const { livekitAlias } = await getSFUConfigWithOpenID(
+        client,
+        urlFromConf,
+        roomId,
+      );
+      const selectedTransport: LivekitTransport = {
+        type: "livekit",
+        livekit_service_url: urlFromConf,
+        livekit_alias: livekitAlias,
+      };
+      logger.info("Using config SFU", selectedTransport);
+      return selectedTransport;
+    } catch (ex) {
+      if (ex instanceof FailToGetOpenIdToken) {
+        throw ex;
+      }
+      logger.error("Failed to validate config SFU", ex);
+    }
   }
 
   if (!transport) throw new MatrixRTCTransportMissingError(domain ?? "");

src/state/CallViewModel/localMember/Publisher.ts

@@ -143,7 +143,7 @@ export class Publisher {
     this.logger.debug("createAndSetupTracks called");
     const lkRoom = this.connection.livekitRoom;
     // Observe mute state changes and update LiveKit microphone/camera states accordingly
-    this.observeMuteStates(this.scope);
+    this.observeMuteStates();
 
     // Check if audio and/or video is enabled. We only create tracks if enabled,
     // because it could prompt for permission, and we don't want to do that unnecessarily.
@@ -356,10 +356,9 @@ export class Publisher {
 
   /**
    * Observe changes in the mute states and update the LiveKit room accordingly.
-   * @param scope
    * @private
    */
-  private observeMuteStates(scope: ObservableScope): void {
+  private observeMuteStates(): void {
     const lkRoom = this.connection.livekitRoom;
     this.muteStates.audio.setHandler(async (enable) => {
       try {

src/state/CallViewModel/remoteMembers/Connection.test.ts

@@ -39,6 +39,7 @@ import {
   ElementCallError,
   FailToGetOpenIdToken,
 } from "../../../utils/errors.ts";
+import { testJWTToken } from "../../../utils/test-fixtures.ts";
 import { mockRemoteParticipant, ownMemberMock } from "../../../utils/test.ts";
 
 let testScope: ObservableScope;
@@ -122,7 +123,7 @@ function setupRemoteConnection(): Connection {
       status: 200,
       body: {
         url: "wss://matrix-rtc.m.localhost/livekit/sfu",
-        jwt: "ATOKEN",
+        jwt: testJWTToken,
       },
     };
   });
@@ -259,7 +260,7 @@ describe("Start connection states", () => {
       capturedState.cause instanceof Error
     ) {
       expect(capturedState.cause.message).toContain(
-        "SFU Config fetch failed with exception Error",
+        "SFU Config fetch failed with exception",
       );
       expect(connection.transport.livekit_alias).toEqual(
         livekitFocus.livekit_alias,
@@ -295,7 +296,7 @@ describe("Start connection states", () => {
         status: 200,
         body: {
           url: "wss://matrix-rtc.m.localhost/livekit/sfu",
-          jwt: "ATOKEN",
+          jwt: testJWTToken,
         },
       };
     });
@@ -393,7 +394,7 @@ describe("remote participants", () => {
     // livekitRoom and the rtc membership in order to publish the members that are publishing
     // on this connection.
 
-    const participants: RemoteParticipant[] = [
+    let participants: RemoteParticipant[] = [
       mockRemoteParticipant({ identity: "@alice:example.org:DEV000" }),
       mockRemoteParticipant({ identity: "@bob:example.org:DEV111" }),
       mockRemoteParticipant({ identity: "@carol:example.org:DEV222" }),
@@ -415,7 +416,22 @@ describe("remote participants", () => {
       fakeLivekitRoom.emit(RoomEvent.ParticipantConnected, p),
     );
 
-    // All remote participants should be present
+    // At this point there should be ~~no~~ publishers
+    // We do have publisher now, since we do not filter for publishers anymore (to also have participants with only data tracks)
+    // The filtering we do is just based on the matrixRTC member events.
+    expect(observedParticipants.pop()!.length).toEqual(4);
+
+    participants = [
+      mockRemoteParticipant({ identity: "@alice:example.org:DEV000" }),
+      mockRemoteParticipant({ identity: "@bob:example.org:DEV111" }),
+      mockRemoteParticipant({ identity: "@carol:example.org:DEV222" }),
+      mockRemoteParticipant({ identity: "@dan:example.org:DEV333" }),
+    ];
+    participants.forEach((p) =>
+      fakeLivekitRoom.emit(RoomEvent.ParticipantConnected, p),
+    );
+
+    // At this point there should be no publishers
     expect(observedParticipants.pop()!.length).toEqual(4);
   });
 

src/state/CallViewModel/remoteMembers/Connection.ts

@@ -228,7 +228,7 @@ export class Connection {
    *
    * @param opts - Connection options {@link ConnectionOpts}.
    *
-   * @param logger
+   * @param logger - The logger to use.
    */
   public constructor(
     opts: ConnectionOpts,
@@ -238,7 +238,7 @@ export class Connection {
     this.forceOldJwtEndpoint = opts.forceOldJwtEndpoint ?? false;
     this.logger = logger.getChild("[Connection]");
     this.logger.info(
-      `[Connection] Creating new connection to ${opts.transport.livekit_service_url} ${opts.transport.livekit_alias}`,
+      `Creating new connection to ${opts.transport.livekit_service_url} ${opts.transport.livekit_alias}`,
     );
     const { transport, client, scope } = opts;
 

src/state/CallViewModel/remoteMembers/ConnectionFactory.ts

@@ -13,7 +13,8 @@ import {
   type BaseE2EEManager,
 } from "livekit-client";
 import { type Logger } from "matrix-js-sdk/lib/logger";
-import E2EEWorker from "livekit-client/e2ee-worker?worker";
+// imported as inline to support worker when loaded from a cdn (cross domain)
+import E2EEWorker from "livekit-client/e2ee-worker?worker&inline";
 import { type CallMembershipIdentityParts } from "matrix-js-sdk/lib/matrixrtc/EncryptionManager";
 import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc/LivekitTransport";
 
@@ -45,11 +46,11 @@ export class ECConnectionFactory implements ConnectionFactory {
    * @param client - The OpenID client parts for authentication, needed to get openID and JWT tokens.
    * @param devices - Used for video/audio out/in capture options.
    * @param processorState$ - Effects like background blur (only for publishing connection?)
-   * @param livekitKeyProvider
+   * @param livekitKeyProvider - Optional key provider for end-to-end encryption.
    * @param controlledAudioDevices - Option to indicate whether audio output device is controlled externally (native mobile app).
+   * @param livekitRoomFactory - Optional factory function (for testing) to create LivekitRoom instances. If not provided, a default factory is used.
    * @param echoCancellation - Whether to enable echo cancellation for audio capture.
    * @param noiseSuppression - Whether to enable noise suppression for audio capture.
-   * @param livekitRoomFactory - Optional factory function (for testing) to create LivekitRoom instances. If not provided, a default factory is used.
    */
   public constructor(
     private client: OpenIDClientParts,

src/state/CallViewModel/remoteMembers/ConnectionManager.test.ts

@@ -293,47 +293,47 @@ describe("connectionManagerData$ stream", () => {
         a: expect.toSatisfy((e) => {
           const data: ConnectionManagerData = e.value;
           expect(data.getConnections().length).toBe(2);
-          expect(data.getParticipantForTransport(TRANSPORT_1).length).toBe(0);
-          expect(data.getParticipantForTransport(TRANSPORT_2).length).toBe(0);
+          expect(data.getParticipantsForTransport(TRANSPORT_1).length).toBe(0);
+          expect(data.getParticipantsForTransport(TRANSPORT_2).length).toBe(0);
           return true;
         }),
         b: expect.toSatisfy((e) => {
           const data: ConnectionManagerData = e.value;
           expect(data.getConnections().length).toBe(2);
-          expect(data.getParticipantForTransport(TRANSPORT_1).length).toBe(1);
-          expect(data.getParticipantForTransport(TRANSPORT_2).length).toBe(0);
-          expect(data.getParticipantForTransport(TRANSPORT_1)[0].identity).toBe(
-            "user1A",
-          );
+          expect(data.getParticipantsForTransport(TRANSPORT_1).length).toBe(1);
+          expect(data.getParticipantsForTransport(TRANSPORT_2).length).toBe(0);
+          expect(
+            data.getParticipantsForTransport(TRANSPORT_1)[0].identity,
+          ).toBe("user1A");
           return true;
         }),
         c: expect.toSatisfy((e) => {
           const data: ConnectionManagerData = e.value;
           expect(data.getConnections().length).toBe(2);
-          expect(data.getParticipantForTransport(TRANSPORT_1).length).toBe(1);
-          expect(data.getParticipantForTransport(TRANSPORT_2).length).toBe(1);
-          expect(data.getParticipantForTransport(TRANSPORT_1)[0].identity).toBe(
-            "user1A",
-          );
-          expect(data.getParticipantForTransport(TRANSPORT_2)[0].identity).toBe(
-            "user2A",
-          );
+          expect(data.getParticipantsForTransport(TRANSPORT_1).length).toBe(1);
+          expect(data.getParticipantsForTransport(TRANSPORT_2).length).toBe(1);
+          expect(
+            data.getParticipantsForTransport(TRANSPORT_1)[0].identity,
+          ).toBe("user1A");
+          expect(
+            data.getParticipantsForTransport(TRANSPORT_2)[0].identity,
+          ).toBe("user2A");
           return true;
         }),
         d: expect.toSatisfy((e) => {
           const data: ConnectionManagerData = e.value;
           expect(data.getConnections().length).toBe(2);
-          expect(data.getParticipantForTransport(TRANSPORT_1).length).toBe(2);
-          expect(data.getParticipantForTransport(TRANSPORT_2).length).toBe(1);
-          expect(data.getParticipantForTransport(TRANSPORT_1)[0].identity).toBe(
-            "user1A",
-          );
-          expect(data.getParticipantForTransport(TRANSPORT_1)[1].identity).toBe(
-            "user1B",
-          );
-          expect(data.getParticipantForTransport(TRANSPORT_2)[0].identity).toBe(
-            "user2A",
-          );
+          expect(data.getParticipantsForTransport(TRANSPORT_1).length).toBe(2);
+          expect(data.getParticipantsForTransport(TRANSPORT_2).length).toBe(1);
+          expect(
+            data.getParticipantsForTransport(TRANSPORT_1)[0].identity,
+          ).toBe("user1A");
+          expect(
+            data.getParticipantsForTransport(TRANSPORT_1)[1].identity,
+          ).toBe("user1B");
+          expect(
+            data.getParticipantsForTransport(TRANSPORT_2)[0].identity,
+          ).toBe("user2A");
           return true;
         }),
       });

src/state/CallViewModel/remoteMembers/ConnectionManager.ts

@@ -20,8 +20,10 @@ import { areLivekitTransportsEqual } from "./MatrixLivekitMembers.ts";
 import { type ConnectionFactory } from "./ConnectionFactory.ts";
 
 export class ConnectionManagerData {
-  private readonly store: Map<string, [Connection, RemoteParticipant[]]> =
-    new Map();
+  private readonly store: Map<
+    string,
+    { connection: Connection; participants: RemoteParticipant[] }
+  > = new Map();
 
   public constructor() {}
 
@@ -29,9 +31,9 @@ export class ConnectionManagerData {
     const key = this.getKey(connection.transport);
     const existing = this.store.get(key);
     if (!existing) {
-      this.store.set(key, [connection, participants]);
+      this.store.set(key, { connection, participants });
     } else {
-      existing[1].push(...participants);
+      existing.participants.push(...participants);
     }
   }
 
@@ -40,20 +42,24 @@ export class ConnectionManagerData {
   }
 
   public getConnections(): Connection[] {
-    return Array.from(this.store.values()).map(([connection]) => connection);
+    return Array.from(this.store.values()).map(({ connection }) => connection);
   }
 
   public getConnectionForTransport(
     transport: LivekitTransport,
   ): Connection | null {
-    return this.store.get(this.getKey(transport))?.[0] ?? null;
+    return this.store.get(this.getKey(transport))?.connection ?? null;
   }
 
-  public getParticipantForTransport(
+  public getParticipantsForTransport(
     transport: LivekitTransport,
   ): RemoteParticipant[] {
     const key = transport.livekit_service_url + "|" + transport.livekit_alias;
-    return this.store.get(key)?.[1] ?? [];
+    const existing = this.store.get(key);
+    if (existing) {
+      return existing.participants;
+    }
+    return [];
   }
 }
 
@@ -74,9 +80,11 @@ export interface IConnectionManager {
 
 /**
  * Crete a `ConnectionManager`
- * @param scope the observable scope used by this object.
- * @param connectionFactory used to create new connections.
- * @param _transportsSubscriptions$ A list of Behaviors each containing a LIST of LivekitTransport.
+ * @param props - Configuration object
+ * @param props.scope - The observable scope used by this object
+ * @param props.connectionFactory - Used to create new connections
+ * @param props.inputTransports$ - A list of Behaviors each containing a LIST of LivekitTransport.
+ * @param props.logger - The logger to use
  *   Each of these behaviors can be interpreted as subscribed list of transports.
  *
  *   Using `registerTransports` independent external modules can control what connections
@@ -207,6 +215,7 @@ export function createConnectionManager$({
         );
 
         // probably not required
+
         if (listOfConnectionsWithRemoteParticipants.length === 0) {
           return of(new Epoch(new ConnectionManagerData(), epoch));
         }

src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.test.ts

@@ -249,7 +249,7 @@ describe("Publication edge case", () => {
       constant(connectionWithPublisher),
     );
 
-    const matrixLivekitMember$ = createMatrixLivekitMembers$({
+    const matrixLivekitMembers$ = createMatrixLivekitMembers$({
       scope: testScope,
       membershipsWithTransport$: testScope.behavior(membershipsWithTransport$),
       connectionManager: {
@@ -257,7 +257,7 @@ describe("Publication edge case", () => {
       } as unknown as IConnectionManager,
     });
     await flushPromises();
-    expect(matrixLivekitMember$.value.value).toSatisfy(
+    expect(matrixLivekitMembers$.value.value).toSatisfy(
       (data: RemoteMatrixLivekitMember[]) => {
         expect(data.length).toEqual(2);
         expect(data[0].membership$.value).toBe(bobMembership);

src/state/CallViewModel/remoteMembers/MatrixLivekitMembers.ts

@@ -100,7 +100,7 @@ export function createMatrixLivekitMembers$({
         function* ([membershipsWithTransport, managerData]) {
           for (const { membership, transport } of membershipsWithTransport) {
             const participants = transport
-              ? managerData.getParticipantForTransport(transport)
+              ? managerData.getParticipantsForTransport(transport)
               : [];
             const participant =
               participants.find(

src/state/CallViewModel/remoteMembers/integration.test.ts

@@ -13,7 +13,11 @@ import fetchMock from "fetch-mock";
 import { type LivekitTransport } from "matrix-js-sdk/lib/matrixrtc";
 import { logger } from "matrix-js-sdk/lib/logger";
 
-import { type Epoch, ObservableScope, trackEpoch } from "../../ObservableScope.ts";
+import {
+  type Epoch,
+  ObservableScope,
+  trackEpoch,
+} from "../../ObservableScope.ts";
 import { ECConnectionFactory } from "./ConnectionFactory.ts";
 import { type OpenIDClientParts } from "../../../livekit/openIDSFU.ts";
 import {
@@ -31,6 +35,7 @@ import {
 import { createConnectionManager$ } from "./ConnectionManager.ts";
 import { membershipsAndTransports$ } from "../../SessionBehaviors.ts";
 import { constant } from "../../Behavior.ts";
+import { testJWTToken } from "../../../utils/test-fixtures.ts";
 
 // Test the integration of ConnectionManager and MatrixLivekitMerger
 
@@ -83,7 +88,7 @@ beforeEach(() => {
       status: 200,
       body: {
         url: `wss://${domain}/livekit/sfu`,
-        jwt: "ATOKEN",
+        jwt: testJWTToken,
       },
     };
   });
@@ -124,14 +129,14 @@ test("bob, carl, then bob joining no tracks yet", () => {
       ownMembershipIdentity: ownMemberMock,
     });
 
-    const matrixLivekitItems$ = createMatrixLivekitMembers$({
+    const matrixLivekitMembers$ = createMatrixLivekitMembers$({
       scope: testScope,
       membershipsWithTransport$:
         membershipsAndTransports.membershipsWithTransport$,
       connectionManager,
     });
 
-    expectObservable(matrixLivekitItems$).toBe(vMarble, {
+    expectObservable(matrixLivekitMembers$).toBe(vMarble, {
       a: expect.toSatisfy((e: Epoch<RemoteMatrixLivekitMember[]>) => {
         const items = e.value;
         expect(items.length).toBe(1);